blk-mq.c 98.1 KB
Newer Older
1
// SPDX-License-Identifier: GPL-2.0
2 3 4 5 6 7
/*
 * Block multiqueue core code
 *
 * Copyright (C) 2013-2014 Jens Axboe
 * Copyright (C) 2013-2014 Christoph Hellwig
 */
8 9 10 11 12
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
13
#include <linux/kmemleak.h>
14 15 16 17 18 19 20 21 22 23
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/llist.h>
#include <linux/list_sort.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
24
#include <linux/sched/topology.h>
25
#include <linux/sched/signal.h>
26
#include <linux/delay.h>
27
#include <linux/crash_dump.h>
28
#include <linux/prefetch.h>
29
#include <linux/blk-crypto.h>
30 31 32 33

#include <trace/events/block.h>

#include <linux/blk-mq.h>
34
#include <linux/t10-pi.h>
35 36
#include "blk.h"
#include "blk-mq.h"
37
#include "blk-mq-debugfs.h"
38
#include "blk-mq-tag.h"
39
#include "blk-pm.h"
40
#include "blk-stat.h"
41
#include "blk-mq-sched.h"
42
#include "blk-rq-qos.h"
43

44
static DEFINE_PER_CPU(struct llist_head, blk_cpu_done);
45

46 47 48
static void blk_mq_poll_stats_start(struct request_queue *q);
static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);

49 50
static int blk_mq_poll_stats_bkt(const struct request *rq)
{
51
	int ddir, sectors, bucket;
52

J
Jens Axboe 已提交
53
	ddir = rq_data_dir(rq);
54
	sectors = blk_rq_stats_sectors(rq);
55

56
	bucket = ddir + 2 * ilog2(sectors);
57 58 59 60 61 62 63 64 65

	if (bucket < 0)
		return -1;
	else if (bucket >= BLK_MQ_POLL_STATS_BKTS)
		return ddir + BLK_MQ_POLL_STATS_BKTS - 2;

	return bucket;
}

66
/*
67 68
 * Check if any of the ctx, dispatch list or elevator
 * have pending work in this hardware queue.
69
 */
70
static bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
71
{
72 73
	return !list_empty_careful(&hctx->dispatch) ||
		sbitmap_any_bit_set(&hctx->ctx_map) ||
74
			blk_mq_sched_has_work(hctx);
75 76
}

77 78 79 80 81 82
/*
 * Mark this ctx as having pending work in this hardware queue
 */
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
				     struct blk_mq_ctx *ctx)
{
83 84 85 86
	const int bit = ctx->index_hw[hctx->type];

	if (!sbitmap_test_bit(&hctx->ctx_map, bit))
		sbitmap_set_bit(&hctx->ctx_map, bit);
87 88 89 90 91
}

static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
				      struct blk_mq_ctx *ctx)
{
92 93 94
	const int bit = ctx->index_hw[hctx->type];

	sbitmap_clear_bit(&hctx->ctx_map, bit);
95 96
}

97
struct mq_inflight {
98
	struct block_device *part;
99
	unsigned int inflight[2];
100 101
};

102
static bool blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,
103 104 105 106 107
				  struct request *rq, void *priv,
				  bool reserved)
{
	struct mq_inflight *mi = priv;

108 109
	if ((!mi->part->bd_partno || rq->part == mi->part) &&
	    blk_mq_rq_state(rq) == MQ_RQ_IN_FLIGHT)
110
		mi->inflight[rq_data_dir(rq)]++;
111 112

	return true;
113 114
}

115 116
unsigned int blk_mq_in_flight(struct request_queue *q,
		struct block_device *part)
117
{
118
	struct mq_inflight mi = { .part = part };
119 120

	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
121

122
	return mi.inflight[0] + mi.inflight[1];
123 124
}

125 126
void blk_mq_in_flight_rw(struct request_queue *q, struct block_device *part,
		unsigned int inflight[2])
127
{
128
	struct mq_inflight mi = { .part = part };
129

130
	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
131 132
	inflight[0] = mi.inflight[0];
	inflight[1] = mi.inflight[1];
133 134
}

135
void blk_freeze_queue_start(struct request_queue *q)
136
{
137 138
	mutex_lock(&q->mq_freeze_lock);
	if (++q->mq_freeze_depth == 1) {
139
		percpu_ref_kill(&q->q_usage_counter);
140
		mutex_unlock(&q->mq_freeze_lock);
J
Jens Axboe 已提交
141
		if (queue_is_mq(q))
142
			blk_mq_run_hw_queues(q, false);
143 144
	} else {
		mutex_unlock(&q->mq_freeze_lock);
145
	}
146
}
147
EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
148

149
void blk_mq_freeze_queue_wait(struct request_queue *q)
150
{
151
	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
152
}
153
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
154

155 156 157 158 159 160 161 162
int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
				     unsigned long timeout)
{
	return wait_event_timeout(q->mq_freeze_wq,
					percpu_ref_is_zero(&q->q_usage_counter),
					timeout);
}
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
163

164 165 166 167
/*
 * Guarantee no request is in use, so we can change any data structure of
 * the queue afterward.
 */
168
void blk_freeze_queue(struct request_queue *q)
169
{
170 171 172 173 174 175 176
	/*
	 * In the !blk_mq case we are only calling this to kill the
	 * q_usage_counter, otherwise this increases the freeze depth
	 * and waits for it to return to zero.  For this reason there is
	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
	 * exported to drivers as the only user for unfreeze is blk_mq.
	 */
177
	blk_freeze_queue_start(q);
178 179
	blk_mq_freeze_queue_wait(q);
}
180 181 182 183 184 185 186 187 188

void blk_mq_freeze_queue(struct request_queue *q)
{
	/*
	 * ...just an alias to keep freeze and unfreeze actions balanced
	 * in the blk_mq_* namespace
	 */
	blk_freeze_queue(q);
}
189
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
190

191
void blk_mq_unfreeze_queue(struct request_queue *q)
192
{
193 194 195 196
	mutex_lock(&q->mq_freeze_lock);
	q->mq_freeze_depth--;
	WARN_ON_ONCE(q->mq_freeze_depth < 0);
	if (!q->mq_freeze_depth) {
197
		percpu_ref_resurrect(&q->q_usage_counter);
198
		wake_up_all(&q->mq_freeze_wq);
199
	}
200
	mutex_unlock(&q->mq_freeze_lock);
201
}
202
EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
203

204 205 206 207 208 209
/*
 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
 * mpt3sas driver such that this function can be removed.
 */
void blk_mq_quiesce_queue_nowait(struct request_queue *q)
{
210
	blk_queue_flag_set(QUEUE_FLAG_QUIESCED, q);
211 212 213
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);

214
/**
215
 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
216 217 218
 * @q: request queue.
 *
 * Note: this function does not prevent that the struct request end_io()
219 220 221
 * callback function is invoked. Once this function is returned, we make
 * sure no dispatch can happen until the queue is unquiesced via
 * blk_mq_unquiesce_queue().
222 223 224 225 226 227 228
 */
void blk_mq_quiesce_queue(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;
	bool rcu = false;

229
	blk_mq_quiesce_queue_nowait(q);
230

231 232
	queue_for_each_hw_ctx(q, hctx, i) {
		if (hctx->flags & BLK_MQ_F_BLOCKING)
233
			synchronize_srcu(hctx->srcu);
234 235 236 237 238 239 240 241
		else
			rcu = true;
	}
	if (rcu)
		synchronize_rcu();
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);

242 243 244 245 246 247 248 249 250
/*
 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
 * @q: request queue.
 *
 * This function recovers queue into the state before quiescing
 * which is done by blk_mq_quiesce_queue.
 */
void blk_mq_unquiesce_queue(struct request_queue *q)
{
251
	blk_queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
252

253 254
	/* dispatch requests which are inserted during quiescing */
	blk_mq_run_hw_queues(q, true);
255 256 257
}
EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);

258 259 260 261 262 263 264 265 266 267
void blk_mq_wake_waiters(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hw_queue_mapped(hctx))
			blk_mq_tag_wakeup_all(hctx->tags, true);
}

268
/*
269 270
 * Only need start/end time stamping if we have iostat or
 * blk stats enabled, or using an IO scheduler.
271 272 273
 */
static inline bool blk_mq_need_time_stamp(struct request *rq)
{
274
	return (rq->rq_flags & (RQF_IO_STAT | RQF_STATS)) || rq->q->elevator;
275 276
}

277
static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
278
		unsigned int tag, u64 alloc_time_ns)
279
{
280 281
	struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
	struct request *rq = tags->static_rqs[tag];
282

283
	if (data->q->elevator) {
284
		rq->tag = BLK_MQ_NO_TAG;
285 286 287
		rq->internal_tag = tag;
	} else {
		rq->tag = tag;
288
		rq->internal_tag = BLK_MQ_NO_TAG;
289 290
	}

291
	/* csd/requeue_work/fifo_time is initialized before use */
292 293
	rq->q = data->q;
	rq->mq_ctx = data->ctx;
294
	rq->mq_hctx = data->hctx;
295
	rq->rq_flags = 0;
296
	rq->cmd_flags = data->cmd_flags;
297 298
	if (data->flags & BLK_MQ_REQ_PM)
		rq->rq_flags |= RQF_PM;
299
	if (blk_queue_io_stat(data->q))
300
		rq->rq_flags |= RQF_IO_STAT;
301
	INIT_LIST_HEAD(&rq->queuelist);
302 303 304 305
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
	rq->rq_disk = NULL;
	rq->part = NULL;
306 307 308
#ifdef CONFIG_BLK_RQ_ALLOC_TIME
	rq->alloc_time_ns = alloc_time_ns;
#endif
309 310 311 312
	if (blk_mq_need_time_stamp(rq))
		rq->start_time_ns = ktime_get_ns();
	else
		rq->start_time_ns = 0;
313
	rq->io_start_time_ns = 0;
314
	rq->stats_sectors = 0;
315 316 317 318
	rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	rq->nr_integrity_segments = 0;
#endif
319
	blk_crypto_rq_set_defaults(rq);
320
	/* tag was already set */
321
	WRITE_ONCE(rq->deadline, 0);
322

323 324
	rq->timeout = 0;

325 326 327
	rq->end_io = NULL;
	rq->end_io_data = NULL;

328
	data->ctx->rq_dispatched[op_is_sync(data->cmd_flags)]++;
K
Keith Busch 已提交
329
	refcount_set(&rq->ref, 1);
330 331 332 333 334 335 336 337 338 339 340 341 342 343 344

	if (!op_is_flush(data->cmd_flags)) {
		struct elevator_queue *e = data->q->elevator;

		rq->elv.icq = NULL;
		if (e && e->type->ops.prepare_request) {
			if (e->type->icq_cache)
				blk_mq_sched_assign_ioc(rq);

			e->type->ops.prepare_request(rq);
			rq->rq_flags |= RQF_ELVPRIV;
		}
	}

	data->hctx->queued++;
345
	return rq;
346 347
}

348
static struct request *__blk_mq_alloc_request(struct blk_mq_alloc_data *data)
349
{
350
	struct request_queue *q = data->q;
351
	struct elevator_queue *e = q->elevator;
352
	u64 alloc_time_ns = 0;
353
	unsigned int tag;
354

355 356 357 358
	/* alloc_time includes depth and tag waits */
	if (blk_queue_rq_alloc_time(q))
		alloc_time_ns = ktime_get_ns();

359
	if (data->cmd_flags & REQ_NOWAIT)
360
		data->flags |= BLK_MQ_REQ_NOWAIT;
361 362 363

	if (e) {
		/*
364
		 * Flush/passthrough requests are special and go directly to the
365 366
		 * dispatch list. Don't include reserved tags in the
		 * limiting, as it isn't useful.
367
		 */
368
		if (!op_is_flush(data->cmd_flags) &&
369
		    !blk_op_is_passthrough(data->cmd_flags) &&
370
		    e->type->ops.limit_depth &&
371
		    !(data->flags & BLK_MQ_REQ_RESERVED))
372
			e->type->ops.limit_depth(data->cmd_flags, data);
373 374
	}

375
retry:
376 377
	data->ctx = blk_mq_get_ctx(q);
	data->hctx = blk_mq_map_queue(q, data->cmd_flags, data->ctx);
378
	if (!e)
379 380
		blk_mq_tag_busy(data->hctx);

381 382 383 384 385
	/*
	 * Waiting allocations only fail because of an inactive hctx.  In that
	 * case just retry the hctx assignment and tag allocation as CPU hotplug
	 * should have migrated us to an online CPU by now.
	 */
386
	tag = blk_mq_get_tag(data);
387 388 389 390 391 392 393
	if (tag == BLK_MQ_NO_TAG) {
		if (data->flags & BLK_MQ_REQ_NOWAIT)
			return NULL;

		/*
		 * Give up the CPU and sleep for a random short time to ensure
		 * that thread using a realtime scheduling class are migrated
394
		 * off the CPU, and thus off the hctx that is going away.
395 396 397 398
		 */
		msleep(3);
		goto retry;
	}
399
	return blk_mq_rq_ctx_init(data, tag, alloc_time_ns);
400 401
}

402
struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
403
		blk_mq_req_flags_t flags)
404
{
405 406 407 408 409
	struct blk_mq_alloc_data data = {
		.q		= q,
		.flags		= flags,
		.cmd_flags	= op,
	};
410
	struct request *rq;
411
	int ret;
412

413
	ret = blk_queue_enter(q, flags);
414 415
	if (ret)
		return ERR_PTR(ret);
416

417
	rq = __blk_mq_alloc_request(&data);
418
	if (!rq)
419
		goto out_queue_exit;
420 421 422
	rq->__data_len = 0;
	rq->__sector = (sector_t) -1;
	rq->bio = rq->biotail = NULL;
423
	return rq;
424 425 426
out_queue_exit:
	blk_queue_exit(q);
	return ERR_PTR(-EWOULDBLOCK);
427
}
428
EXPORT_SYMBOL(blk_mq_alloc_request);
429

430
struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
431
	unsigned int op, blk_mq_req_flags_t flags, unsigned int hctx_idx)
M
Ming Lin 已提交
432
{
433 434 435 436 437
	struct blk_mq_alloc_data data = {
		.q		= q,
		.flags		= flags,
		.cmd_flags	= op,
	};
438
	u64 alloc_time_ns = 0;
439
	unsigned int cpu;
440
	unsigned int tag;
M
Ming Lin 已提交
441 442
	int ret;

443 444 445 446
	/* alloc_time includes depth and tag waits */
	if (blk_queue_rq_alloc_time(q))
		alloc_time_ns = ktime_get_ns();

M
Ming Lin 已提交
447 448 449 450 451 452
	/*
	 * If the tag allocator sleeps we could get an allocation for a
	 * different hardware context.  No need to complicate the low level
	 * allocator for this for the rare use case of a command tied to
	 * a specific queue.
	 */
453
	if (WARN_ON_ONCE(!(flags & (BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED))))
M
Ming Lin 已提交
454 455 456 457 458
		return ERR_PTR(-EINVAL);

	if (hctx_idx >= q->nr_hw_queues)
		return ERR_PTR(-EIO);

459
	ret = blk_queue_enter(q, flags);
M
Ming Lin 已提交
460 461 462
	if (ret)
		return ERR_PTR(ret);

463 464 465 466
	/*
	 * Check if the hardware context is actually mapped to anything.
	 * If not tell the caller that it should skip this queue.
	 */
467
	ret = -EXDEV;
468 469
	data.hctx = q->queue_hw_ctx[hctx_idx];
	if (!blk_mq_hw_queue_mapped(data.hctx))
470
		goto out_queue_exit;
471 472
	cpu = cpumask_first_and(data.hctx->cpumask, cpu_online_mask);
	data.ctx = __blk_mq_get_ctx(q, cpu);
M
Ming Lin 已提交
473

474
	if (!q->elevator)
475 476
		blk_mq_tag_busy(data.hctx);

477
	ret = -EWOULDBLOCK;
478 479
	tag = blk_mq_get_tag(&data);
	if (tag == BLK_MQ_NO_TAG)
480
		goto out_queue_exit;
481 482
	return blk_mq_rq_ctx_init(&data, tag, alloc_time_ns);

483 484 485
out_queue_exit:
	blk_queue_exit(q);
	return ERR_PTR(ret);
M
Ming Lin 已提交
486 487 488
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);

K
Keith Busch 已提交
489 490 491 492
static void __blk_mq_free_request(struct request *rq)
{
	struct request_queue *q = rq->q;
	struct blk_mq_ctx *ctx = rq->mq_ctx;
493
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
K
Keith Busch 已提交
494 495
	const int sched_tag = rq->internal_tag;

496
	blk_crypto_free_request(rq);
497
	blk_pm_mark_last_busy(rq);
498
	rq->mq_hctx = NULL;
499
	if (rq->tag != BLK_MQ_NO_TAG)
500
		blk_mq_put_tag(hctx->tags, ctx, rq->tag);
501
	if (sched_tag != BLK_MQ_NO_TAG)
502
		blk_mq_put_tag(hctx->sched_tags, ctx, sched_tag);
K
Keith Busch 已提交
503 504 505 506
	blk_mq_sched_restart(hctx);
	blk_queue_exit(q);
}

507
void blk_mq_free_request(struct request *rq)
508 509
{
	struct request_queue *q = rq->q;
510 511
	struct elevator_queue *e = q->elevator;
	struct blk_mq_ctx *ctx = rq->mq_ctx;
512
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
513

514
	if (rq->rq_flags & RQF_ELVPRIV) {
515 516
		if (e && e->type->ops.finish_request)
			e->type->ops.finish_request(rq);
517 518 519 520 521
		if (rq->elv.icq) {
			put_io_context(rq->elv.icq->ioc);
			rq->elv.icq = NULL;
		}
	}
522

523
	ctx->rq_completed[rq_is_sync(rq)]++;
524
	if (rq->rq_flags & RQF_MQ_INFLIGHT)
525
		__blk_mq_dec_active_requests(hctx);
J
Jens Axboe 已提交
526

527 528 529
	if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
		laptop_io_completion(q->backing_dev_info);

530
	rq_qos_done(q, rq);
531

K
Keith Busch 已提交
532 533 534
	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
	if (refcount_dec_and_test(&rq->ref))
		__blk_mq_free_request(rq);
535
}
J
Jens Axboe 已提交
536
EXPORT_SYMBOL_GPL(blk_mq_free_request);
537

538
inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
539
{
540 541 542 543
	u64 now = 0;

	if (blk_mq_need_time_stamp(rq))
		now = ktime_get_ns();
544

545 546
	if (rq->rq_flags & RQF_STATS) {
		blk_mq_poll_stats_start(rq->q);
547
		blk_stat_add(rq, now);
548 549
	}

550
	blk_mq_sched_completed_request(rq, now);
551

552
	blk_account_io_done(rq, now);
M
Ming Lei 已提交
553

C
Christoph Hellwig 已提交
554
	if (rq->end_io) {
555
		rq_qos_done(rq->q, rq);
556
		rq->end_io(rq, error);
C
Christoph Hellwig 已提交
557
	} else {
558
		blk_mq_free_request(rq);
C
Christoph Hellwig 已提交
559
	}
560
}
561
EXPORT_SYMBOL(__blk_mq_end_request);
562

563
void blk_mq_end_request(struct request *rq, blk_status_t error)
564 565 566
{
	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
		BUG();
567
	__blk_mq_end_request(rq, error);
568
}
569
EXPORT_SYMBOL(blk_mq_end_request);
570

571
static void blk_complete_reqs(struct llist_head *list)
572
{
573 574
	struct llist_node *entry = llist_reverse_order(llist_del_all(list));
	struct request *rq, *next;
575

576
	llist_for_each_entry_safe(rq, next, entry, ipi_list)
577
		rq->q->mq_ops->complete(rq);
578 579
}

580
static __latent_entropy void blk_done_softirq(struct softirq_action *h)
581
{
582
	blk_complete_reqs(this_cpu_ptr(&blk_cpu_done));
583 584
}

585 586
static int blk_softirq_cpu_dead(unsigned int cpu)
{
587
	blk_complete_reqs(&per_cpu(blk_cpu_done, cpu));
588 589 590
	return 0;
}

591
static void __blk_mq_complete_request_remote(void *data)
592
{
593
	__raise_softirq_irqoff(BLOCK_SOFTIRQ);
594 595
}

596 597 598 599 600 601 602
static inline bool blk_mq_complete_need_ipi(struct request *rq)
{
	int cpu = raw_smp_processor_id();

	if (!IS_ENABLED(CONFIG_SMP) ||
	    !test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags))
		return false;
603 604 605 606 607 608 609 610
	/*
	 * With force threaded interrupts enabled, raising softirq from an SMP
	 * function call will always result in waking the ksoftirqd thread.
	 * This is probably worse than completing the request on a different
	 * cache domain.
	 */
	if (force_irqthreads)
		return false;
611 612 613 614 615 616 617 618 619 620 621

	/* same CPU or cache domain?  Complete locally */
	if (cpu == rq->mq_ctx->cpu ||
	    (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags) &&
	     cpus_share_cache(cpu, rq->mq_ctx->cpu)))
		return false;

	/* don't try to IPI to an offline CPU */
	return cpu_online(rq->mq_ctx->cpu);
}

622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
static void blk_mq_complete_send_ipi(struct request *rq)
{
	struct llist_head *list;
	unsigned int cpu;

	cpu = rq->mq_ctx->cpu;
	list = &per_cpu(blk_cpu_done, cpu);
	if (llist_add(&rq->ipi_list, list)) {
		INIT_CSD(&rq->csd, __blk_mq_complete_request_remote, rq);
		smp_call_function_single_async(cpu, &rq->csd);
	}
}

static void blk_mq_raise_softirq(struct request *rq)
{
	struct llist_head *list;

	preempt_disable();
	list = this_cpu_ptr(&blk_cpu_done);
	if (llist_add(&rq->ipi_list, list))
		raise_softirq(BLOCK_SOFTIRQ);
	preempt_enable();
}

646
bool blk_mq_complete_request_remote(struct request *rq)
647
{
648
	WRITE_ONCE(rq->state, MQ_RQ_COMPLETE);
649

650 651 652 653
	/*
	 * For a polled request, always complete locallly, it's pointless
	 * to redirect the completion.
	 */
654 655
	if (rq->cmd_flags & REQ_HIPRI)
		return false;
C
Christoph Hellwig 已提交
656

657
	if (blk_mq_complete_need_ipi(rq)) {
658 659
		blk_mq_complete_send_ipi(rq);
		return true;
660
	}
661

662 663 664 665 666
	if (rq->q->nr_hw_queues == 1) {
		blk_mq_raise_softirq(rq);
		return true;
	}
	return false;
667 668 669 670 671 672 673 674 675 676 677 678 679 680
}
EXPORT_SYMBOL_GPL(blk_mq_complete_request_remote);

/**
 * blk_mq_complete_request - end I/O on a request
 * @rq:		the request being processed
 *
 * Description:
 *	Complete a request by scheduling the ->complete_rq operation.
 **/
void blk_mq_complete_request(struct request *rq)
{
	if (!blk_mq_complete_request_remote(rq))
		rq->q->mq_ops->complete(rq);
681
}
682
EXPORT_SYMBOL(blk_mq_complete_request);
683

684
static void hctx_unlock(struct blk_mq_hw_ctx *hctx, int srcu_idx)
685
	__releases(hctx->srcu)
686 687 688 689
{
	if (!(hctx->flags & BLK_MQ_F_BLOCKING))
		rcu_read_unlock();
	else
690
		srcu_read_unlock(hctx->srcu, srcu_idx);
691 692 693
}

static void hctx_lock(struct blk_mq_hw_ctx *hctx, int *srcu_idx)
694
	__acquires(hctx->srcu)
695
{
696 697 698
	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
		/* shut up gcc false positive */
		*srcu_idx = 0;
699
		rcu_read_lock();
700
	} else
701
		*srcu_idx = srcu_read_lock(hctx->srcu);
702 703
}

704 705 706 707 708 709 710 711
/**
 * blk_mq_start_request - Start processing a request
 * @rq: Pointer to request to be started
 *
 * Function used by device drivers to notify the block layer that a request
 * is going to be processed now, so blk layer can do proper initializations
 * such as starting the timeout timer.
 */
712
void blk_mq_start_request(struct request *rq)
713 714 715
{
	struct request_queue *q = rq->q;

716
	trace_block_rq_issue(rq);
717

718
	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
719
		rq->io_start_time_ns = ktime_get_ns();
720
		rq->stats_sectors = blk_rq_sectors(rq);
721
		rq->rq_flags |= RQF_STATS;
722
		rq_qos_issue(q, rq);
723 724
	}

725
	WARN_ON_ONCE(blk_mq_rq_state(rq) != MQ_RQ_IDLE);
726

727
	blk_add_timer(rq);
K
Keith Busch 已提交
728
	WRITE_ONCE(rq->state, MQ_RQ_IN_FLIGHT);
729

730 731 732 733
#ifdef CONFIG_BLK_DEV_INTEGRITY
	if (blk_integrity_rq(rq) && req_op(rq) == REQ_OP_WRITE)
		q->integrity.profile->prepare_fn(rq);
#endif
734
}
735
EXPORT_SYMBOL(blk_mq_start_request);
736

737
static void __blk_mq_requeue_request(struct request *rq)
738 739 740
{
	struct request_queue *q = rq->q;

741 742
	blk_mq_put_driver_tag(rq);

743
	trace_block_rq_requeue(rq);
744
	rq_qos_requeue(q, rq);
745

K
Keith Busch 已提交
746 747
	if (blk_mq_request_started(rq)) {
		WRITE_ONCE(rq->state, MQ_RQ_IDLE);
748
		rq->rq_flags &= ~RQF_TIMED_OUT;
749
	}
750 751
}

752
void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
753 754 755
{
	__blk_mq_requeue_request(rq);

756 757 758
	/* this request will be re-inserted to io scheduler queue */
	blk_mq_sched_requeue_request(rq);

J
Jens Axboe 已提交
759
	BUG_ON(!list_empty(&rq->queuelist));
760
	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
761 762 763
}
EXPORT_SYMBOL(blk_mq_requeue_request);

764 765 766
static void blk_mq_requeue_work(struct work_struct *work)
{
	struct request_queue *q =
767
		container_of(work, struct request_queue, requeue_work.work);
768 769 770
	LIST_HEAD(rq_list);
	struct request *rq, *next;

771
	spin_lock_irq(&q->requeue_lock);
772
	list_splice_init(&q->requeue_list, &rq_list);
773
	spin_unlock_irq(&q->requeue_lock);
774 775

	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
776
		if (!(rq->rq_flags & (RQF_SOFTBARRIER | RQF_DONTPREP)))
777 778
			continue;

779
		rq->rq_flags &= ~RQF_SOFTBARRIER;
780
		list_del_init(&rq->queuelist);
781 782 783 784 785 786
		/*
		 * If RQF_DONTPREP, rq has contained some driver specific
		 * data, so insert it to hctx dispatch list to avoid any
		 * merge.
		 */
		if (rq->rq_flags & RQF_DONTPREP)
787
			blk_mq_request_bypass_insert(rq, false, false);
788 789
		else
			blk_mq_sched_insert_request(rq, true, false, false);
790 791 792 793 794
	}

	while (!list_empty(&rq_list)) {
		rq = list_entry(rq_list.next, struct request, queuelist);
		list_del_init(&rq->queuelist);
795
		blk_mq_sched_insert_request(rq, false, false, false);
796 797
	}

798
	blk_mq_run_hw_queues(q, false);
799 800
}

801 802
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
				bool kick_requeue_list)
803 804 805 806 807 808
{
	struct request_queue *q = rq->q;
	unsigned long flags;

	/*
	 * We abuse this flag that is otherwise used by the I/O scheduler to
809
	 * request head insertion from the workqueue.
810
	 */
811
	BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
812 813 814

	spin_lock_irqsave(&q->requeue_lock, flags);
	if (at_head) {
815
		rq->rq_flags |= RQF_SOFTBARRIER;
816 817 818 819 820
		list_add(&rq->queuelist, &q->requeue_list);
	} else {
		list_add_tail(&rq->queuelist, &q->requeue_list);
	}
	spin_unlock_irqrestore(&q->requeue_lock, flags);
821 822 823

	if (kick_requeue_list)
		blk_mq_kick_requeue_list(q);
824 825 826 827
}

void blk_mq_kick_requeue_list(struct request_queue *q)
{
828
	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work, 0);
829 830 831
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);

832 833 834
void blk_mq_delay_kick_requeue_list(struct request_queue *q,
				    unsigned long msecs)
{
835 836
	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
				    msecs_to_jiffies(msecs));
837 838 839
}
EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);

840 841
struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
{
842 843
	if (tag < tags->nr_tags) {
		prefetch(tags->rqs[tag]);
844
		return tags->rqs[tag];
845
	}
846 847

	return NULL;
848 849 850
}
EXPORT_SYMBOL(blk_mq_tag_to_rq);

851 852
static bool blk_mq_rq_inflight(struct blk_mq_hw_ctx *hctx, struct request *rq,
			       void *priv, bool reserved)
853 854
{
	/*
855
	 * If we find a request that isn't idle and the queue matches,
856
	 * we know the queue is busy. Return false to stop the iteration.
857
	 */
858
	if (blk_mq_request_started(rq) && rq->q == hctx->queue) {
859 860 861 862 863 864 865 866 867
		bool *busy = priv;

		*busy = true;
		return false;
	}

	return true;
}

868
bool blk_mq_queue_inflight(struct request_queue *q)
869 870 871
{
	bool busy = false;

872
	blk_mq_queue_tag_busy_iter(q, blk_mq_rq_inflight, &busy);
873 874
	return busy;
}
875
EXPORT_SYMBOL_GPL(blk_mq_queue_inflight);
876

877
static void blk_mq_rq_timed_out(struct request *req, bool reserved)
878
{
879
	req->rq_flags |= RQF_TIMED_OUT;
880 881 882 883 884 885 886
	if (req->q->mq_ops->timeout) {
		enum blk_eh_timer_return ret;

		ret = req->q->mq_ops->timeout(req, reserved);
		if (ret == BLK_EH_DONE)
			return;
		WARN_ON_ONCE(ret != BLK_EH_RESET_TIMER);
887
	}
888 889

	blk_add_timer(req);
890
}
891

K
Keith Busch 已提交
892
static bool blk_mq_req_expired(struct request *rq, unsigned long *next)
893
{
K
Keith Busch 已提交
894
	unsigned long deadline;
895

K
Keith Busch 已提交
896 897
	if (blk_mq_rq_state(rq) != MQ_RQ_IN_FLIGHT)
		return false;
898 899
	if (rq->rq_flags & RQF_TIMED_OUT)
		return false;
900

901
	deadline = READ_ONCE(rq->deadline);
K
Keith Busch 已提交
902 903
	if (time_after_eq(jiffies, deadline))
		return true;
904

K
Keith Busch 已提交
905 906 907 908 909
	if (*next == 0)
		*next = deadline;
	else if (time_after(*next, deadline))
		*next = deadline;
	return false;
910 911
}

912
static bool blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
913 914
		struct request *rq, void *priv, bool reserved)
{
K
Keith Busch 已提交
915 916 917 918 919 920 921
	unsigned long *next = priv;

	/*
	 * Just do a quick check if it is expired before locking the request in
	 * so we're not unnecessarilly synchronizing across CPUs.
	 */
	if (!blk_mq_req_expired(rq, next))
922
		return true;
K
Keith Busch 已提交
923 924 925 926 927 928 929 930 931 932 933

	/*
	 * We have reason to believe the request may be expired. Take a
	 * reference on the request to lock this request lifetime into its
	 * currently allocated context to prevent it from being reallocated in
	 * the event the completion by-passes this timeout handler.
	 *
	 * If the reference was already released, then the driver beat the
	 * timeout handler to posting a natural completion.
	 */
	if (!refcount_inc_not_zero(&rq->ref))
934
		return true;
K
Keith Busch 已提交
935

936
	/*
K
Keith Busch 已提交
937 938 939 940
	 * The request is now locked and cannot be reallocated underneath the
	 * timeout handler's processing. Re-verify this exact request is truly
	 * expired; if it is not expired, then the request was completed and
	 * reallocated as a new request.
941
	 */
K
Keith Busch 已提交
942
	if (blk_mq_req_expired(rq, next))
943
		blk_mq_rq_timed_out(rq, reserved);
944 945 946 947

	if (is_flush_rq(rq, hctx))
		rq->end_io(rq, 0);
	else if (refcount_dec_and_test(&rq->ref))
K
Keith Busch 已提交
948
		__blk_mq_free_request(rq);
949 950

	return true;
951 952
}

953
static void blk_mq_timeout_work(struct work_struct *work)
954
{
955 956
	struct request_queue *q =
		container_of(work, struct request_queue, timeout_work);
K
Keith Busch 已提交
957
	unsigned long next = 0;
958
	struct blk_mq_hw_ctx *hctx;
959
	int i;
960

961 962 963 964 965 966 967 968 969
	/* A deadlock might occur if a request is stuck requiring a
	 * timeout at the same time a queue freeze is waiting
	 * completion, since the timeout code would not be able to
	 * acquire the queue reference here.
	 *
	 * That's why we don't use blk_queue_enter here; instead, we use
	 * percpu_ref_tryget directly, because we need to be able to
	 * obtain a reference even in the short window between the queue
	 * starting to freeze, by dropping the first reference in
970
	 * blk_freeze_queue_start, and the moment the last request is
971 972 973 974
	 * consumed, marked by the instant q_usage_counter reaches
	 * zero.
	 */
	if (!percpu_ref_tryget(&q->q_usage_counter))
975 976
		return;

K
Keith Busch 已提交
977
	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &next);
978

K
Keith Busch 已提交
979 980
	if (next != 0) {
		mod_timer(&q->timeout, next);
981
	} else {
982 983 984 985 986 987
		/*
		 * Request timeouts are handled as a forward rolling timer. If
		 * we end up here it means that no requests are pending and
		 * also that no request has been pending for a while. Mark
		 * each hctx as idle.
		 */
988 989 990 991 992
		queue_for_each_hw_ctx(q, hctx, i) {
			/* the hctx may be unmapped, so check it here */
			if (blk_mq_hw_queue_mapped(hctx))
				blk_mq_tag_idle(hctx);
		}
993
	}
994
	blk_queue_exit(q);
995 996
}

997 998 999 1000 1001 1002 1003 1004 1005 1006
struct flush_busy_ctx_data {
	struct blk_mq_hw_ctx *hctx;
	struct list_head *list;
};

static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
{
	struct flush_busy_ctx_data *flush_data = data;
	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
M
Ming Lei 已提交
1007
	enum hctx_type type = hctx->type;
1008 1009

	spin_lock(&ctx->lock);
M
Ming Lei 已提交
1010
	list_splice_tail_init(&ctx->rq_lists[type], flush_data->list);
1011
	sbitmap_clear_bit(sb, bitnr);
1012 1013 1014 1015
	spin_unlock(&ctx->lock);
	return true;
}

1016 1017 1018 1019
/*
 * Process software queues that have been marked busy, splicing them
 * to the for-dispatch
 */
1020
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
1021
{
1022 1023 1024 1025
	struct flush_busy_ctx_data data = {
		.hctx = hctx,
		.list = list,
	};
1026

1027
	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
1028
}
1029
EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
1030

1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041
struct dispatch_rq_data {
	struct blk_mq_hw_ctx *hctx;
	struct request *rq;
};

static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
		void *data)
{
	struct dispatch_rq_data *dispatch_data = data;
	struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];
M
Ming Lei 已提交
1042
	enum hctx_type type = hctx->type;
1043 1044

	spin_lock(&ctx->lock);
M
Ming Lei 已提交
1045 1046
	if (!list_empty(&ctx->rq_lists[type])) {
		dispatch_data->rq = list_entry_rq(ctx->rq_lists[type].next);
1047
		list_del_init(&dispatch_data->rq->queuelist);
M
Ming Lei 已提交
1048
		if (list_empty(&ctx->rq_lists[type]))
1049 1050 1051 1052 1053 1054 1055 1056 1057 1058
			sbitmap_clear_bit(sb, bitnr);
	}
	spin_unlock(&ctx->lock);

	return !dispatch_data->rq;
}

struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
					struct blk_mq_ctx *start)
{
1059
	unsigned off = start ? start->index_hw[hctx->type] : 0;
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070
	struct dispatch_rq_data data = {
		.hctx = hctx,
		.rq   = NULL,
	};

	__sbitmap_for_each_set(&hctx->ctx_map, off,
			       dispatch_rq_from_ctx, &data);

	return data.rq;
}

1071 1072 1073 1074
static inline unsigned int queued_to_index(unsigned int queued)
{
	if (!queued)
		return 0;
1075

1076
	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
1077 1078
}

1079 1080
static bool __blk_mq_get_driver_tag(struct request *rq)
{
1081
	struct sbitmap_queue *bt = rq->mq_hctx->tags->bitmap_tags;
1082 1083 1084
	unsigned int tag_offset = rq->mq_hctx->tags->nr_reserved_tags;
	int tag;

1085 1086
	blk_mq_tag_busy(rq->mq_hctx);

1087
	if (blk_mq_tag_is_reserved(rq->mq_hctx->sched_tags, rq->internal_tag)) {
1088
		bt = rq->mq_hctx->tags->breserved_tags;
1089
		tag_offset = 0;
1090 1091 1092
	} else {
		if (!hctx_may_queue(rq->mq_hctx, bt))
			return false;
1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104
	}

	tag = __sbitmap_queue_get(bt);
	if (tag == BLK_MQ_NO_TAG)
		return false;

	rq->tag = tag + tag_offset;
	return true;
}

static bool blk_mq_get_driver_tag(struct request *rq)
{
1105 1106 1107 1108 1109
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;

	if (rq->tag == BLK_MQ_NO_TAG && !__blk_mq_get_driver_tag(rq))
		return false;

1110
	if ((hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED) &&
1111 1112
			!(rq->rq_flags & RQF_MQ_INFLIGHT)) {
		rq->rq_flags |= RQF_MQ_INFLIGHT;
1113
		__blk_mq_inc_active_requests(hctx);
1114 1115 1116
	}
	hctx->tags->rqs[rq->tag] = rq;
	return true;
1117 1118
}

1119 1120
static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
				int flags, void *key)
1121 1122 1123 1124 1125
{
	struct blk_mq_hw_ctx *hctx;

	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);

1126
	spin_lock(&hctx->dispatch_wait_lock);
1127 1128 1129 1130
	if (!list_empty(&wait->entry)) {
		struct sbitmap_queue *sbq;

		list_del_init(&wait->entry);
1131
		sbq = hctx->tags->bitmap_tags;
1132 1133
		atomic_dec(&sbq->ws_active);
	}
1134 1135
	spin_unlock(&hctx->dispatch_wait_lock);

1136 1137 1138 1139
	blk_mq_run_hw_queue(hctx, true);
	return 1;
}

1140 1141
/*
 * Mark us waiting for a tag. For shared tags, this involves hooking us into
1142 1143
 * the tag wakeups. For non-shared tags, we can simply mark us needing a
 * restart. For both cases, take care to check the condition again after
1144 1145
 * marking us as waiting.
 */
1146
static bool blk_mq_mark_tag_wait(struct blk_mq_hw_ctx *hctx,
1147
				 struct request *rq)
1148
{
1149
	struct sbitmap_queue *sbq = hctx->tags->bitmap_tags;
1150
	struct wait_queue_head *wq;
1151 1152
	wait_queue_entry_t *wait;
	bool ret;
1153

1154
	if (!(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
1155
		blk_mq_sched_mark_restart_hctx(hctx);
1156

1157 1158 1159 1160 1161 1162 1163 1164
		/*
		 * It's possible that a tag was freed in the window between the
		 * allocation failure and adding the hardware queue to the wait
		 * queue.
		 *
		 * Don't clear RESTART here, someone else could have set it.
		 * At most this will cost an extra queue run.
		 */
1165
		return blk_mq_get_driver_tag(rq);
1166 1167
	}

1168
	wait = &hctx->dispatch_wait;
1169 1170 1171
	if (!list_empty_careful(&wait->entry))
		return false;

1172
	wq = &bt_wait_ptr(sbq, hctx)->wait;
1173 1174 1175

	spin_lock_irq(&wq->lock);
	spin_lock(&hctx->dispatch_wait_lock);
1176
	if (!list_empty(&wait->entry)) {
1177 1178
		spin_unlock(&hctx->dispatch_wait_lock);
		spin_unlock_irq(&wq->lock);
1179
		return false;
1180 1181
	}

1182
	atomic_inc(&sbq->ws_active);
1183 1184
	wait->flags &= ~WQ_FLAG_EXCLUSIVE;
	__add_wait_queue(wq, wait);
1185

1186
	/*
1187 1188 1189
	 * It's possible that a tag was freed in the window between the
	 * allocation failure and adding the hardware queue to the wait
	 * queue.
1190
	 */
1191
	ret = blk_mq_get_driver_tag(rq);
1192
	if (!ret) {
1193 1194
		spin_unlock(&hctx->dispatch_wait_lock);
		spin_unlock_irq(&wq->lock);
1195
		return false;
1196
	}
1197 1198 1199 1200 1201 1202

	/*
	 * We got a tag, remove ourselves from the wait queue to ensure
	 * someone else gets the wakeup.
	 */
	list_del_init(&wait->entry);
1203
	atomic_dec(&sbq->ws_active);
1204 1205
	spin_unlock(&hctx->dispatch_wait_lock);
	spin_unlock_irq(&wq->lock);
1206 1207

	return true;
1208 1209
}

1210 1211 1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
#define BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT  8
#define BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR  4
/*
 * Update dispatch busy with the Exponential Weighted Moving Average(EWMA):
 * - EWMA is one simple way to compute running average value
 * - weight(7/8 and 1/8) is applied so that it can decrease exponentially
 * - take 4 as factor for avoiding to get too small(0) result, and this
 *   factor doesn't matter because EWMA decreases exponentially
 */
static void blk_mq_update_dispatch_busy(struct blk_mq_hw_ctx *hctx, bool busy)
{
	unsigned int ewma;

	if (hctx->queue->elevator)
		return;

	ewma = hctx->dispatch_busy;

	if (!ewma && !busy)
		return;

	ewma *= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT - 1;
	if (busy)
		ewma += 1 << BLK_MQ_DISPATCH_BUSY_EWMA_FACTOR;
	ewma /= BLK_MQ_DISPATCH_BUSY_EWMA_WEIGHT;

	hctx->dispatch_busy = ewma;
}

1239 1240
#define BLK_MQ_RESOURCE_DELAY	3		/* ms units */

1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257
static void blk_mq_handle_dev_resource(struct request *rq,
				       struct list_head *list)
{
	struct request *next =
		list_first_entry_or_null(list, struct request, queuelist);

	/*
	 * If an I/O scheduler has been configured and we got a driver tag for
	 * the next request already, free it.
	 */
	if (next)
		blk_mq_put_driver_tag(next);

	list_add(&rq->queuelist, list);
	__blk_mq_requeue_request(rq);
}

1258 1259 1260 1261 1262 1263 1264 1265 1266 1267 1268 1269 1270
static void blk_mq_handle_zone_resource(struct request *rq,
					struct list_head *zone_list)
{
	/*
	 * If we end up here it is because we cannot dispatch a request to a
	 * specific zone due to LLD level zone-write locking or other zone
	 * related resource not being available. In this case, set the request
	 * aside in zone_list for retrying it later.
	 */
	list_add(&rq->queuelist, zone_list);
	__blk_mq_requeue_request(rq);
}

1271 1272 1273 1274 1275 1276 1277 1278 1279 1280
enum prep_dispatch {
	PREP_DISPATCH_OK,
	PREP_DISPATCH_NO_TAG,
	PREP_DISPATCH_NO_BUDGET,
};

static enum prep_dispatch blk_mq_prep_dispatch_rq(struct request *rq,
						  bool need_budget)
{
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1281
	int budget_token = -1;
1282

1283 1284 1285 1286 1287 1288 1289
	if (need_budget) {
		budget_token = blk_mq_get_dispatch_budget(rq->q);
		if (budget_token < 0) {
			blk_mq_put_driver_tag(rq);
			return PREP_DISPATCH_NO_BUDGET;
		}
		blk_mq_set_rq_budget_token(rq, budget_token);
1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300
	}

	if (!blk_mq_get_driver_tag(rq)) {
		/*
		 * The initial allocation attempt failed, so we need to
		 * rerun the hardware queue when a tag is freed. The
		 * waitqueue takes care of that. If the queue is run
		 * before we add this entry back on the dispatch list,
		 * we'll re-run it below.
		 */
		if (!blk_mq_mark_tag_wait(hctx, rq)) {
1301 1302 1303 1304 1305
			/*
			 * All budgets not got from this function will be put
			 * together during handling partial dispatch
			 */
			if (need_budget)
1306
				blk_mq_put_dispatch_budget(rq->q, budget_token);
1307 1308 1309 1310 1311 1312 1313
			return PREP_DISPATCH_NO_TAG;
		}
	}

	return PREP_DISPATCH_OK;
}

1314 1315
/* release all allocated budgets before calling to blk_mq_dispatch_rq_list */
static void blk_mq_release_budgets(struct request_queue *q,
1316
		struct list_head *list)
1317
{
1318
	struct request *rq;
1319

1320 1321
	list_for_each_entry(rq, list, queuelist) {
		int budget_token = blk_mq_get_rq_budget_token(rq);
1322

1323 1324 1325
		if (budget_token >= 0)
			blk_mq_put_dispatch_budget(q, budget_token);
	}
1326 1327
}

1328 1329 1330
/*
 * Returns true if we did some work AND can potentially do more.
 */
1331
bool blk_mq_dispatch_rq_list(struct blk_mq_hw_ctx *hctx, struct list_head *list,
1332
			     unsigned int nr_budgets)
1333
{
1334
	enum prep_dispatch prep;
1335
	struct request_queue *q = hctx->queue;
1336
	struct request *rq, *nxt;
1337
	int errors, queued;
1338
	blk_status_t ret = BLK_STS_OK;
1339
	LIST_HEAD(zone_list);
1340

1341 1342 1343
	if (list_empty(list))
		return false;

1344 1345 1346
	/*
	 * Now process all the entries, sending them to the driver.
	 */
1347
	errors = queued = 0;
1348
	do {
1349
		struct blk_mq_queue_data bd;
1350

1351
		rq = list_first_entry(list, struct request, queuelist);
1352

1353
		WARN_ON_ONCE(hctx != rq->mq_hctx);
1354
		prep = blk_mq_prep_dispatch_rq(rq, !nr_budgets);
1355
		if (prep != PREP_DISPATCH_OK)
1356
			break;
1357

1358 1359
		list_del_init(&rq->queuelist);

1360
		bd.rq = rq;
1361 1362 1363 1364 1365 1366 1367 1368 1369

		/*
		 * Flag last if we have no more requests, or if we have more
		 * but can't assign a driver tag to it.
		 */
		if (list_empty(list))
			bd.last = true;
		else {
			nxt = list_first_entry(list, struct request, queuelist);
1370
			bd.last = !blk_mq_get_driver_tag(nxt);
1371
		}
1372

1373 1374 1375 1376 1377 1378
		/*
		 * once the request is queued to lld, no need to cover the
		 * budget any more
		 */
		if (nr_budgets)
			nr_budgets--;
1379
		ret = q->mq_ops->queue_rq(hctx, &bd);
1380 1381 1382
		switch (ret) {
		case BLK_STS_OK:
			queued++;
1383
			break;
1384 1385 1386 1387 1388
		case BLK_STS_RESOURCE:
		case BLK_STS_DEV_RESOURCE:
			blk_mq_handle_dev_resource(rq, list);
			goto out;
		case BLK_STS_ZONE_RESOURCE:
1389 1390 1391 1392 1393 1394
			/*
			 * Move the request to zone_list and keep going through
			 * the dispatch list to find more requests the drive can
			 * accept.
			 */
			blk_mq_handle_zone_resource(rq, &zone_list);
1395 1396
			break;
		default:
1397
			errors++;
1398
			blk_mq_end_request(rq, ret);
1399
		}
1400
	} while (!list_empty(list));
1401
out:
1402 1403 1404
	if (!list_empty(&zone_list))
		list_splice_tail_init(&zone_list, list);

1405
	hctx->dispatched[queued_to_index(queued)]++;
1406

1407 1408 1409 1410 1411
	/* If we didn't flush the entire list, we could have told the driver
	 * there was more coming, but that turned out to be a lie.
	 */
	if ((!list_empty(list) || errors) && q->mq_ops->commit_rqs && queued)
		q->mq_ops->commit_rqs(hctx);
1412 1413 1414 1415
	/*
	 * Any items that need requeuing? Stuff them into hctx->dispatch,
	 * that is where we will continue on next queue run.
	 */
1416
	if (!list_empty(list)) {
1417
		bool needs_restart;
1418 1419
		/* For non-shared tags, the RESTART check will suffice */
		bool no_tag = prep == PREP_DISPATCH_NO_TAG &&
1420
			(hctx->flags & BLK_MQ_F_TAG_QUEUE_SHARED);
1421
		bool no_budget_avail = prep == PREP_DISPATCH_NO_BUDGET;
1422

1423 1424
		if (nr_budgets)
			blk_mq_release_budgets(q, list);
1425

1426
		spin_lock(&hctx->lock);
1427
		list_splice_tail_init(list, &hctx->dispatch);
1428
		spin_unlock(&hctx->lock);
1429

1430 1431 1432 1433 1434 1435 1436 1437 1438
		/*
		 * Order adding requests to hctx->dispatch and checking
		 * SCHED_RESTART flag. The pair of this smp_mb() is the one
		 * in blk_mq_sched_restart(). Avoid restart code path to
		 * miss the new added requests to hctx->dispatch, meantime
		 * SCHED_RESTART is observed here.
		 */
		smp_mb();

1439
		/*
1440 1441 1442
		 * If SCHED_RESTART was set by the caller of this function and
		 * it is no longer set that means that it was cleared by another
		 * thread and hence that a queue rerun is needed.
1443
		 *
1444 1445 1446 1447
		 * If 'no_tag' is set, that means that we failed getting
		 * a driver tag with an I/O scheduler attached. If our dispatch
		 * waitqueue is no longer active, ensure that we run the queue
		 * AFTER adding our entries back to the list.
1448
		 *
1449 1450 1451 1452 1453 1454 1455
		 * If no I/O scheduler has been configured it is possible that
		 * the hardware queue got stopped and restarted before requests
		 * were pushed back onto the dispatch list. Rerun the queue to
		 * avoid starvation. Notes:
		 * - blk_mq_run_hw_queue() checks whether or not a queue has
		 *   been stopped before rerunning a queue.
		 * - Some but not all block drivers stop a queue before
1456
		 *   returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
1457
		 *   and dm-rq.
1458 1459 1460
		 *
		 * If driver returns BLK_STS_RESOURCE and SCHED_RESTART
		 * bit is set, run queue after a delay to avoid IO stalls
1461 1462
		 * that could otherwise occur if the queue is idle.  We'll do
		 * similar if we couldn't get budget and SCHED_RESTART is set.
1463
		 */
1464 1465
		needs_restart = blk_mq_sched_needs_restart(hctx);
		if (!needs_restart ||
1466
		    (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
1467
			blk_mq_run_hw_queue(hctx, true);
1468 1469
		else if (needs_restart && (ret == BLK_STS_RESOURCE ||
					   no_budget_avail))
1470
			blk_mq_delay_run_hw_queue(hctx, BLK_MQ_RESOURCE_DELAY);
1471

1472
		blk_mq_update_dispatch_busy(hctx, true);
1473
		return false;
1474 1475
	} else
		blk_mq_update_dispatch_busy(hctx, false);
1476

1477
	return (queued + errors) != 0;
1478 1479
}

1480 1481 1482 1483 1484 1485
/**
 * __blk_mq_run_hw_queue - Run a hardware queue.
 * @hctx: Pointer to the hardware queue to run.
 *
 * Send pending requests to the hardware.
 */
1486 1487 1488 1489
static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	int srcu_idx;

1490 1491 1492 1493 1494 1495
	/*
	 * We can't run the queue inline with ints disabled. Ensure that
	 * we catch bad users of this early.
	 */
	WARN_ON_ONCE(in_interrupt());

1496
	might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);
1497

1498 1499 1500
	hctx_lock(hctx, &srcu_idx);
	blk_mq_sched_dispatch_requests(hctx);
	hctx_unlock(hctx, srcu_idx);
1501 1502
}

1503 1504 1505 1506 1507 1508 1509 1510 1511
static inline int blk_mq_first_mapped_cpu(struct blk_mq_hw_ctx *hctx)
{
	int cpu = cpumask_first_and(hctx->cpumask, cpu_online_mask);

	if (cpu >= nr_cpu_ids)
		cpu = cpumask_first(hctx->cpumask);
	return cpu;
}

1512 1513 1514 1515 1516 1517 1518 1519
/*
 * It'd be great if the workqueue API had a way to pass
 * in a mask and had some smarts for more clever placement.
 * For now we just round-robin here, switching for every
 * BLK_MQ_CPU_WORK_BATCH queued items.
 */
static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
{
1520
	bool tried = false;
1521
	int next_cpu = hctx->next_cpu;
1522

1523 1524
	if (hctx->queue->nr_hw_queues == 1)
		return WORK_CPU_UNBOUND;
1525 1526

	if (--hctx->next_cpu_batch <= 0) {
1527
select_cpu:
1528
		next_cpu = cpumask_next_and(next_cpu, hctx->cpumask,
1529
				cpu_online_mask);
1530
		if (next_cpu >= nr_cpu_ids)
1531
			next_cpu = blk_mq_first_mapped_cpu(hctx);
1532 1533 1534
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}

1535 1536 1537 1538
	/*
	 * Do unbound schedule if we can't find a online CPU for this hctx,
	 * and it should only happen in the path of handling CPU DEAD.
	 */
1539
	if (!cpu_online(next_cpu)) {
1540 1541 1542 1543 1544 1545 1546 1547 1548
		if (!tried) {
			tried = true;
			goto select_cpu;
		}

		/*
		 * Make sure to re-select CPU next time once after CPUs
		 * in hctx->cpumask become online again.
		 */
1549
		hctx->next_cpu = next_cpu;
1550 1551 1552
		hctx->next_cpu_batch = 1;
		return WORK_CPU_UNBOUND;
	}
1553 1554 1555

	hctx->next_cpu = next_cpu;
	return next_cpu;
1556 1557
}

1558 1559 1560 1561
/**
 * __blk_mq_delay_run_hw_queue - Run (or schedule to run) a hardware queue.
 * @hctx: Pointer to the hardware queue to run.
 * @async: If we want to run the queue asynchronously.
1562
 * @msecs: Milliseconds of delay to wait before running the queue.
1563 1564 1565 1566
 *
 * If !@async, try to run the queue now. Else, run the queue asynchronously and
 * with a delay of @msecs.
 */
1567 1568
static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,
					unsigned long msecs)
1569
{
1570
	if (unlikely(blk_mq_hctx_stopped(hctx)))
1571 1572
		return;

1573
	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1574 1575
		int cpu = get_cpu();
		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
1576
			__blk_mq_run_hw_queue(hctx);
1577
			put_cpu();
1578 1579
			return;
		}
1580

1581
		put_cpu();
1582
	}
1583

1584 1585
	kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx), &hctx->run_work,
				    msecs_to_jiffies(msecs));
1586 1587
}

1588 1589 1590
/**
 * blk_mq_delay_run_hw_queue - Run a hardware queue asynchronously.
 * @hctx: Pointer to the hardware queue to run.
1591
 * @msecs: Milliseconds of delay to wait before running the queue.
1592 1593 1594
 *
 * Run a hardware queue asynchronously with a delay of @msecs.
 */
1595 1596 1597 1598 1599 1600
void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
	__blk_mq_delay_run_hw_queue(hctx, true, msecs);
}
EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);

1601 1602 1603 1604 1605 1606 1607 1608 1609
/**
 * blk_mq_run_hw_queue - Start to run a hardware queue.
 * @hctx: Pointer to the hardware queue to run.
 * @async: If we want to run the queue asynchronously.
 *
 * Check if the request queue is not in a quiesced state and if there are
 * pending requests to be sent. If this is true, run the queue to send requests
 * to hardware.
 */
1610
void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
1611
{
1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622
	int srcu_idx;
	bool need_run;

	/*
	 * When queue is quiesced, we may be switching io scheduler, or
	 * updating nr_hw_queues, or other things, and we can't run queue
	 * any more, even __blk_mq_hctx_has_pending() can't be called safely.
	 *
	 * And queue will be rerun in blk_mq_unquiesce_queue() if it is
	 * quiesced.
	 */
1623 1624 1625 1626
	hctx_lock(hctx, &srcu_idx);
	need_run = !blk_queue_quiesced(hctx->queue) &&
		blk_mq_hctx_has_pending(hctx);
	hctx_unlock(hctx, srcu_idx);
1627

1628
	if (need_run)
1629
		__blk_mq_delay_run_hw_queue(hctx, async, 0);
1630
}
O
Omar Sandoval 已提交
1631
EXPORT_SYMBOL(blk_mq_run_hw_queue);
1632

1633 1634 1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
/*
 * Is the request queue handled by an IO scheduler that does not respect
 * hardware queues when dispatching?
 */
static bool blk_mq_has_sqsched(struct request_queue *q)
{
	struct elevator_queue *e = q->elevator;

	if (e && e->type->ops.dispatch_request &&
	    !(e->type->elevator_features & ELEVATOR_F_MQ_AWARE))
		return true;
	return false;
}

/*
 * Return prefered queue to dispatch from (if any) for non-mq aware IO
 * scheduler.
 */
static struct blk_mq_hw_ctx *blk_mq_get_sq_hctx(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;

	/*
	 * If the IO scheduler does not respect hardware queues when
	 * dispatching, we just don't bother with multiple HW queues and
	 * dispatch from hctx for the current CPU since running multiple queues
	 * just causes lock contention inside the scheduler and pointless cache
	 * bouncing.
	 */
	hctx = blk_mq_map_queue_type(q, HCTX_TYPE_DEFAULT,
				     raw_smp_processor_id());
	if (!blk_mq_hctx_stopped(hctx))
		return hctx;
	return NULL;
}

1669
/**
1670
 * blk_mq_run_hw_queues - Run all hardware queues in a request queue.
1671 1672 1673
 * @q: Pointer to the request queue to run.
 * @async: If we want to run the queue asynchronously.
 */
1674
void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1675
{
1676
	struct blk_mq_hw_ctx *hctx, *sq_hctx;
1677 1678
	int i;

1679 1680 1681
	sq_hctx = NULL;
	if (blk_mq_has_sqsched(q))
		sq_hctx = blk_mq_get_sq_hctx(q);
1682
	queue_for_each_hw_ctx(q, hctx, i) {
1683
		if (blk_mq_hctx_stopped(hctx))
1684
			continue;
1685 1686 1687 1688 1689 1690 1691 1692
		/*
		 * Dispatch from this hctx either if there's no hctx preferred
		 * by IO scheduler or if it has requests that bypass the
		 * scheduler.
		 */
		if (!sq_hctx || sq_hctx == hctx ||
		    !list_empty_careful(&hctx->dispatch))
			blk_mq_run_hw_queue(hctx, async);
1693 1694
	}
}
1695
EXPORT_SYMBOL(blk_mq_run_hw_queues);
1696

1697 1698 1699
/**
 * blk_mq_delay_run_hw_queues - Run all hardware queues asynchronously.
 * @q: Pointer to the request queue to run.
1700
 * @msecs: Milliseconds of delay to wait before running the queues.
1701 1702 1703
 */
void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs)
{
1704
	struct blk_mq_hw_ctx *hctx, *sq_hctx;
1705 1706
	int i;

1707 1708 1709
	sq_hctx = NULL;
	if (blk_mq_has_sqsched(q))
		sq_hctx = blk_mq_get_sq_hctx(q);
1710 1711 1712
	queue_for_each_hw_ctx(q, hctx, i) {
		if (blk_mq_hctx_stopped(hctx))
			continue;
1713 1714 1715 1716 1717 1718 1719 1720
		/*
		 * Dispatch from this hctx either if there's no hctx preferred
		 * by IO scheduler or if it has requests that bypass the
		 * scheduler.
		 */
		if (!sq_hctx || sq_hctx == hctx ||
		    !list_empty_careful(&hctx->dispatch))
			blk_mq_delay_run_hw_queue(hctx, msecs);
1721 1722 1723 1724
	}
}
EXPORT_SYMBOL(blk_mq_delay_run_hw_queues);

1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744
/**
 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
 * @q: request queue.
 *
 * The caller is responsible for serializing this function against
 * blk_mq_{start,stop}_hw_queue().
 */
bool blk_mq_queue_stopped(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hctx_stopped(hctx))
			return true;

	return false;
}
EXPORT_SYMBOL(blk_mq_queue_stopped);

1745 1746 1747
/*
 * This function is often used for pausing .queue_rq() by driver when
 * there isn't enough resource or some conditions aren't satisfied, and
1748
 * BLK_STS_RESOURCE is usually returned.
1749 1750 1751 1752 1753
 *
 * We do not guarantee that dispatch can be drained or blocked
 * after blk_mq_stop_hw_queue() returns. Please use
 * blk_mq_quiesce_queue() for that requirement.
 */
1754 1755
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
{
1756
	cancel_delayed_work(&hctx->run_work);
1757

1758
	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
1759
}
1760
EXPORT_SYMBOL(blk_mq_stop_hw_queue);
1761

1762 1763 1764
/*
 * This function is often used for pausing .queue_rq() by driver when
 * there isn't enough resource or some conditions aren't satisfied, and
1765
 * BLK_STS_RESOURCE is usually returned.
1766 1767 1768 1769 1770
 *
 * We do not guarantee that dispatch can be drained or blocked
 * after blk_mq_stop_hw_queues() returns. Please use
 * blk_mq_quiesce_queue() for that requirement.
 */
1771 1772
void blk_mq_stop_hw_queues(struct request_queue *q)
{
1773 1774 1775 1776 1777
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_stop_hw_queue(hctx);
1778 1779 1780
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);

1781 1782 1783
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1784

1785
	blk_mq_run_hw_queue(hctx, false);
1786 1787 1788
}
EXPORT_SYMBOL(blk_mq_start_hw_queue);

1789 1790 1791 1792 1793 1794 1795 1796 1797 1798
void blk_mq_start_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_start_hw_queues);

1799 1800 1801 1802 1803 1804 1805 1806 1807 1808
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
	if (!blk_mq_hctx_stopped(hctx))
		return;

	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
	blk_mq_run_hw_queue(hctx, async);
}
EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);

1809
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
1810 1811 1812 1813
{
	struct blk_mq_hw_ctx *hctx;
	int i;

1814 1815
	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_stopped_hw_queue(hctx, async);
1816 1817 1818
}
EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);

1819
static void blk_mq_run_work_fn(struct work_struct *work)
1820 1821 1822
{
	struct blk_mq_hw_ctx *hctx;

1823
	hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
1824

1825
	/*
M
Ming Lei 已提交
1826
	 * If we are stopped, don't run the queue.
1827
	 */
1828
	if (blk_mq_hctx_stopped(hctx))
1829
		return;
1830 1831 1832 1833

	__blk_mq_run_hw_queue(hctx);
}

1834 1835 1836
static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
					    struct request *rq,
					    bool at_head)
1837
{
J
Jens Axboe 已提交
1838
	struct blk_mq_ctx *ctx = rq->mq_ctx;
M
Ming Lei 已提交
1839
	enum hctx_type type = hctx->type;
J
Jens Axboe 已提交
1840

1841 1842
	lockdep_assert_held(&ctx->lock);

1843
	trace_block_rq_insert(rq);
1844

1845
	if (at_head)
M
Ming Lei 已提交
1846
		list_add(&rq->queuelist, &ctx->rq_lists[type]);
1847
	else
M
Ming Lei 已提交
1848
		list_add_tail(&rq->queuelist, &ctx->rq_lists[type]);
1849
}
1850

1851 1852
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
			     bool at_head)
1853 1854 1855
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

1856 1857
	lockdep_assert_held(&ctx->lock);

J
Jens Axboe 已提交
1858
	__blk_mq_insert_req_list(hctx, rq, at_head);
1859 1860 1861
	blk_mq_hctx_mark_pending(hctx, ctx);
}

1862 1863 1864
/**
 * blk_mq_request_bypass_insert - Insert a request at dispatch list.
 * @rq: Pointer to request to be inserted.
1865
 * @at_head: true if the request should be inserted at the head of the list.
1866 1867
 * @run_queue: If we should run the hardware queue after inserting the request.
 *
1868 1869 1870
 * Should only be used carefully, when the caller knows we want to
 * bypass a potential IO scheduler on the target device.
 */
1871 1872
void blk_mq_request_bypass_insert(struct request *rq, bool at_head,
				  bool run_queue)
1873
{
1874
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
1875 1876

	spin_lock(&hctx->lock);
1877 1878 1879 1880
	if (at_head)
		list_add(&rq->queuelist, &hctx->dispatch);
	else
		list_add_tail(&rq->queuelist, &hctx->dispatch);
1881 1882
	spin_unlock(&hctx->lock);

1883 1884
	if (run_queue)
		blk_mq_run_hw_queue(hctx, false);
1885 1886
}

1887 1888
void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			    struct list_head *list)
1889 1890

{
1891
	struct request *rq;
M
Ming Lei 已提交
1892
	enum hctx_type type = hctx->type;
1893

1894 1895 1896 1897
	/*
	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
	 * offline now
	 */
1898
	list_for_each_entry(rq, list, queuelist) {
J
Jens Axboe 已提交
1899
		BUG_ON(rq->mq_ctx != ctx);
1900
		trace_block_rq_insert(rq);
1901
	}
1902 1903

	spin_lock(&ctx->lock);
M
Ming Lei 已提交
1904
	list_splice_tail_init(list, &ctx->rq_lists[type]);
1905
	blk_mq_hctx_mark_pending(hctx, ctx);
1906 1907 1908
	spin_unlock(&ctx->lock);
}

1909 1910
static int plug_rq_cmp(void *priv, const struct list_head *a,
		       const struct list_head *b)
1911 1912 1913 1914
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

P
Pavel Begunkov 已提交
1915 1916 1917 1918
	if (rqa->mq_ctx != rqb->mq_ctx)
		return rqa->mq_ctx > rqb->mq_ctx;
	if (rqa->mq_hctx != rqb->mq_hctx)
		return rqa->mq_hctx > rqb->mq_hctx;
J
Jens Axboe 已提交
1919 1920

	return blk_rq_pos(rqa) > blk_rq_pos(rqb);
1921 1922 1923 1924 1925 1926
}

void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
	LIST_HEAD(list);

1927 1928
	if (list_empty(&plug->mq_list))
		return;
1929 1930
	list_splice_init(&plug->mq_list, &list);

1931 1932
	if (plug->rq_count > 2 && plug->multiple_queues)
		list_sort(NULL, &list, plug_rq_cmp);
1933

1934 1935
	plug->rq_count = 0;

1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949
	do {
		struct list_head rq_list;
		struct request *rq, *head_rq = list_entry_rq(list.next);
		struct list_head *pos = &head_rq->queuelist; /* skip first */
		struct blk_mq_hw_ctx *this_hctx = head_rq->mq_hctx;
		struct blk_mq_ctx *this_ctx = head_rq->mq_ctx;
		unsigned int depth = 1;

		list_for_each_continue(pos, &list) {
			rq = list_entry_rq(pos);
			BUG_ON(!rq->q);
			if (rq->mq_hctx != this_hctx || rq->mq_ctx != this_ctx)
				break;
			depth++;
1950 1951
		}

1952 1953
		list_cut_before(&rq_list, &list, pos);
		trace_block_unplug(head_rq->q, depth, !from_schedule);
1954
		blk_mq_sched_insert_requests(this_hctx, this_ctx, &rq_list,
1955
						from_schedule);
1956
	} while(!list_empty(&list));
1957 1958
}

1959 1960
static void blk_mq_bio_to_request(struct request *rq, struct bio *bio,
		unsigned int nr_segs)
1961
{
1962 1963
	int err;

1964 1965 1966 1967 1968
	if (bio->bi_opf & REQ_RAHEAD)
		rq->cmd_flags |= REQ_FAILFAST_MASK;

	rq->__sector = bio->bi_iter.bi_sector;
	rq->write_hint = bio->bi_write_hint;
1969
	blk_rq_bio_prep(rq, bio, nr_segs);
1970 1971 1972 1973

	/* This can't fail, since GFP_NOIO includes __GFP_DIRECT_RECLAIM. */
	err = blk_crypto_rq_bio_prep(rq, bio, GFP_NOIO);
	WARN_ON_ONCE(err);
1974

1975
	blk_account_io_start(rq);
1976 1977
}

1978 1979
static blk_status_t __blk_mq_issue_directly(struct blk_mq_hw_ctx *hctx,
					    struct request *rq,
1980
					    blk_qc_t *cookie, bool last)
1981 1982 1983 1984
{
	struct request_queue *q = rq->q;
	struct blk_mq_queue_data bd = {
		.rq = rq,
1985
		.last = last,
1986
	};
1987
	blk_qc_t new_cookie;
1988
	blk_status_t ret;
1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999

	new_cookie = request_to_qc_t(hctx, rq);

	/*
	 * For OK queue, we are done. For error, caller may kill it.
	 * Any other error (busy), just add it to our list as we
	 * previously would have done.
	 */
	ret = q->mq_ops->queue_rq(hctx, &bd);
	switch (ret) {
	case BLK_STS_OK:
2000
		blk_mq_update_dispatch_busy(hctx, false);
2001 2002 2003
		*cookie = new_cookie;
		break;
	case BLK_STS_RESOURCE:
2004
	case BLK_STS_DEV_RESOURCE:
2005
		blk_mq_update_dispatch_busy(hctx, true);
2006 2007 2008
		__blk_mq_requeue_request(rq);
		break;
	default:
2009
		blk_mq_update_dispatch_busy(hctx, false);
2010 2011 2012 2013 2014 2015 2016
		*cookie = BLK_QC_T_NONE;
		break;
	}

	return ret;
}

2017
static blk_status_t __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
2018
						struct request *rq,
2019
						blk_qc_t *cookie,
2020
						bool bypass_insert, bool last)
2021 2022
{
	struct request_queue *q = rq->q;
M
Ming Lei 已提交
2023
	bool run_queue = true;
2024
	int budget_token;
M
Ming Lei 已提交
2025

2026
	/*
2027
	 * RCU or SRCU read lock is needed before checking quiesced flag.
2028
	 *
2029 2030 2031
	 * When queue is stopped or quiesced, ignore 'bypass_insert' from
	 * blk_mq_request_issue_directly(), and return BLK_STS_OK to caller,
	 * and avoid driver to try to dispatch again.
2032
	 */
2033
	if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) {
M
Ming Lei 已提交
2034
		run_queue = false;
2035 2036
		bypass_insert = false;
		goto insert;
M
Ming Lei 已提交
2037
	}
2038

2039 2040
	if (q->elevator && !bypass_insert)
		goto insert;
2041

2042 2043
	budget_token = blk_mq_get_dispatch_budget(q);
	if (budget_token < 0)
2044
		goto insert;
2045

2046 2047
	blk_mq_set_rq_budget_token(rq, budget_token);

2048
	if (!blk_mq_get_driver_tag(rq)) {
2049
		blk_mq_put_dispatch_budget(q, budget_token);
2050
		goto insert;
2051
	}
2052

2053 2054 2055 2056 2057
	return __blk_mq_issue_directly(hctx, rq, cookie, last);
insert:
	if (bypass_insert)
		return BLK_STS_RESOURCE;

2058 2059
	blk_mq_sched_insert_request(rq, false, run_queue, false);

2060 2061 2062
	return BLK_STS_OK;
}

2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073
/**
 * blk_mq_try_issue_directly - Try to send a request directly to device driver.
 * @hctx: Pointer of the associated hardware queue.
 * @rq: Pointer to request to be sent.
 * @cookie: Request queue cookie.
 *
 * If the device has enough resources to accept a new request now, send the
 * request directly to device driver. Else, insert at hctx->dispatch queue, so
 * we can try send it another time in the future. Requests inserted at this
 * queue have higher priority.
 */
2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085
static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
		struct request *rq, blk_qc_t *cookie)
{
	blk_status_t ret;
	int srcu_idx;

	might_sleep_if(hctx->flags & BLK_MQ_F_BLOCKING);

	hctx_lock(hctx, &srcu_idx);

	ret = __blk_mq_try_issue_directly(hctx, rq, cookie, false, true);
	if (ret == BLK_STS_RESOURCE || ret == BLK_STS_DEV_RESOURCE)
2086
		blk_mq_request_bypass_insert(rq, false, true);
2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101
	else if (ret != BLK_STS_OK)
		blk_mq_end_request(rq, ret);

	hctx_unlock(hctx, srcu_idx);
}

blk_status_t blk_mq_request_issue_directly(struct request *rq, bool last)
{
	blk_status_t ret;
	int srcu_idx;
	blk_qc_t unused_cookie;
	struct blk_mq_hw_ctx *hctx = rq->mq_hctx;

	hctx_lock(hctx, &srcu_idx);
	ret = __blk_mq_try_issue_directly(hctx, rq, &unused_cookie, true, last);
2102
	hctx_unlock(hctx, srcu_idx);
2103 2104

	return ret;
2105 2106
}

2107 2108 2109
void blk_mq_try_issue_list_directly(struct blk_mq_hw_ctx *hctx,
		struct list_head *list)
{
2110
	int queued = 0;
2111
	int errors = 0;
2112

2113
	while (!list_empty(list)) {
2114
		blk_status_t ret;
2115 2116 2117 2118
		struct request *rq = list_first_entry(list, struct request,
				queuelist);

		list_del_init(&rq->queuelist);
2119 2120 2121 2122
		ret = blk_mq_request_issue_directly(rq, list_empty(list));
		if (ret != BLK_STS_OK) {
			if (ret == BLK_STS_RESOURCE ||
					ret == BLK_STS_DEV_RESOURCE) {
2123
				blk_mq_request_bypass_insert(rq, false,
2124
							list_empty(list));
2125 2126 2127
				break;
			}
			blk_mq_end_request(rq, ret);
2128
			errors++;
2129 2130
		} else
			queued++;
2131
	}
J
Jens Axboe 已提交
2132 2133 2134 2135 2136 2137

	/*
	 * If we didn't flush the entire list, we could have told
	 * the driver there was more coming, but that turned out to
	 * be a lie.
	 */
2138 2139
	if ((!list_empty(list) || errors) &&
	     hctx->queue->mq_ops->commit_rqs && queued)
J
Jens Axboe 已提交
2140
		hctx->queue->mq_ops->commit_rqs(hctx);
2141 2142
}

2143 2144 2145 2146 2147 2148 2149 2150 2151 2152 2153 2154 2155 2156
static void blk_add_rq_to_plug(struct blk_plug *plug, struct request *rq)
{
	list_add_tail(&rq->queuelist, &plug->mq_list);
	plug->rq_count++;
	if (!plug->multiple_queues && !list_is_singular(&plug->mq_list)) {
		struct request *tmp;

		tmp = list_first_entry(&plug->mq_list, struct request,
						queuelist);
		if (tmp->q != rq->q)
			plug->multiple_queues = true;
	}
}

2157
/**
2158
 * blk_mq_submit_bio - Create and send a request to block device.
2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170 2171
 * @bio: Bio pointer.
 *
 * Builds up a request structure from @q and @bio and send to the device. The
 * request may not be queued directly to hardware if:
 * * This request can be merged with another one
 * * We want to place request at plug queue for possible future merging
 * * There is an IO scheduler active at this queue
 *
 * It will not queue the request if there is an error with the bio, or at the
 * request creation.
 *
 * Returns: Request queue cookie.
 */
2172
blk_qc_t blk_mq_submit_bio(struct bio *bio)
2173
{
2174
	struct request_queue *q = bio->bi_bdev->bd_disk->queue;
2175
	const int is_sync = op_is_sync(bio->bi_opf);
2176
	const int is_flush_fua = op_is_flush(bio->bi_opf);
2177 2178 2179
	struct blk_mq_alloc_data data = {
		.q		= q,
	};
2180
	struct request *rq;
2181
	struct blk_plug *plug;
2182
	struct request *same_queue_rq = NULL;
2183
	unsigned int nr_segs;
2184
	blk_qc_t cookie;
2185
	blk_status_t ret;
J
Jeffle Xu 已提交
2186
	bool hipri;
2187 2188

	blk_queue_bounce(q, &bio);
2189
	__blk_queue_split(&bio, &nr_segs);
2190

2191
	if (!bio_integrity_prep(bio))
2192
		goto queue_exit;
2193

2194
	if (!is_flush_fua && !blk_queue_nomerges(q) &&
2195
	    blk_attempt_plug_merge(q, bio, nr_segs, &same_queue_rq))
2196
		goto queue_exit;
2197

2198
	if (blk_mq_sched_bio_merge(q, bio, nr_segs))
2199
		goto queue_exit;
2200

2201
	rq_qos_throttle(q, bio);
J
Jens Axboe 已提交
2202

J
Jeffle Xu 已提交
2203 2204
	hipri = bio->bi_opf & REQ_HIPRI;

2205
	data.cmd_flags = bio->bi_opf;
2206
	rq = __blk_mq_alloc_request(&data);
J
Jens Axboe 已提交
2207
	if (unlikely(!rq)) {
2208
		rq_qos_cleanup(q, bio);
J
Jens Axboe 已提交
2209
		if (bio->bi_opf & REQ_NOWAIT)
2210
			bio_wouldblock_error(bio);
2211
		goto queue_exit;
J
Jens Axboe 已提交
2212 2213
	}

2214
	trace_block_getrq(bio);
2215

2216
	rq_qos_track(q, rq, bio);
2217

2218
	cookie = request_to_qc_t(data.hctx, rq);
2219

2220 2221
	blk_mq_bio_to_request(rq, bio, nr_segs);

2222 2223 2224 2225 2226 2227 2228 2229
	ret = blk_crypto_init_request(rq);
	if (ret != BLK_STS_OK) {
		bio->bi_status = ret;
		bio_endio(bio);
		blk_mq_free_request(rq);
		return BLK_QC_T_NONE;
	}

2230
	plug = blk_mq_plug(q, bio);
2231
	if (unlikely(is_flush_fua)) {
2232
		/* Bypass scheduler for flush requests */
2233 2234
		blk_insert_flush(rq);
		blk_mq_run_hw_queue(data.hctx, true);
2235 2236 2237
	} else if (plug && (q->nr_hw_queues == 1 ||
		   blk_mq_is_sbitmap_shared(rq->mq_hctx->flags) ||
		   q->mq_ops->commit_rqs || !blk_queue_nonrot(q))) {
2238 2239 2240
		/*
		 * Use plugging if we have a ->commit_rqs() hook as well, as
		 * we know the driver uses bd->last in a smart fashion.
M
Ming Lei 已提交
2241 2242 2243
		 *
		 * Use normal plugging if this disk is slow HDD, as sequential
		 * IO may benefit a lot from plug merging.
2244
		 */
2245
		unsigned int request_count = plug->rq_count;
2246 2247
		struct request *last = NULL;

M
Ming Lei 已提交
2248
		if (!request_count)
2249
			trace_block_plug(q);
2250 2251
		else
			last = list_entry_rq(plug->mq_list.prev);
2252

2253 2254
		if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
2255 2256
			blk_flush_plug_list(plug, false);
			trace_block_plug(q);
2257
		}
2258

2259
		blk_add_rq_to_plug(plug, rq);
2260
	} else if (q->elevator) {
2261
		/* Insert the request at the IO scheduler queue */
2262
		blk_mq_sched_insert_request(rq, false, true, true);
2263
	} else if (plug && !blk_queue_nomerges(q)) {
2264
		/*
2265
		 * We do limited plugging. If the bio can be merged, do that.
2266 2267
		 * Otherwise the existing request in the plug list will be
		 * issued. So the plug list will have one request at most
2268 2269
		 * The plug list might get flushed before this. If that happens,
		 * the plug list is empty, and same_queue_rq is invalid.
2270
		 */
2271 2272
		if (list_empty(&plug->mq_list))
			same_queue_rq = NULL;
2273
		if (same_queue_rq) {
2274
			list_del_init(&same_queue_rq->queuelist);
2275 2276
			plug->rq_count--;
		}
2277
		blk_add_rq_to_plug(plug, rq);
2278
		trace_block_plug(q);
2279

2280
		if (same_queue_rq) {
2281
			data.hctx = same_queue_rq->mq_hctx;
2282
			trace_block_unplug(q, 1, true);
2283
			blk_mq_try_issue_directly(data.hctx, same_queue_rq,
2284
					&cookie);
2285
		}
2286 2287
	} else if ((q->nr_hw_queues > 1 && is_sync) ||
			!data.hctx->dispatch_busy) {
2288 2289 2290 2291
		/*
		 * There is no scheduler and we can try to send directly
		 * to the hardware.
		 */
2292
		blk_mq_try_issue_directly(data.hctx, rq, &cookie);
2293
	} else {
2294
		/* Default case. */
2295
		blk_mq_sched_insert_request(rq, false, true, true);
2296
	}
2297

J
Jeffle Xu 已提交
2298 2299
	if (!hipri)
		return BLK_QC_T_NONE;
2300
	return cookie;
2301 2302 2303
queue_exit:
	blk_queue_exit(q);
	return BLK_QC_T_NONE;
2304 2305
}

2306 2307
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx)
2308
{
2309
	struct page *page;
2310

2311
	if (tags->rqs && set->ops->exit_request) {
2312
		int i;
2313

2314
		for (i = 0; i < tags->nr_tags; i++) {
J
Jens Axboe 已提交
2315 2316 2317
			struct request *rq = tags->static_rqs[i];

			if (!rq)
2318
				continue;
2319
			set->ops->exit_request(set, rq, hctx_idx);
J
Jens Axboe 已提交
2320
			tags->static_rqs[i] = NULL;
2321
		}
2322 2323
	}

2324 2325
	while (!list_empty(&tags->page_list)) {
		page = list_first_entry(&tags->page_list, struct page, lru);
2326
		list_del_init(&page->lru);
2327 2328
		/*
		 * Remove kmemleak object previously allocated in
2329
		 * blk_mq_alloc_rqs().
2330 2331
		 */
		kmemleak_free(page_address(page));
2332 2333
		__free_pages(page, page->private);
	}
2334
}
2335

2336
void blk_mq_free_rq_map(struct blk_mq_tags *tags, unsigned int flags)
2337
{
2338
	kfree(tags->rqs);
2339
	tags->rqs = NULL;
J
Jens Axboe 已提交
2340 2341
	kfree(tags->static_rqs);
	tags->static_rqs = NULL;
2342

2343
	blk_mq_free_tags(tags, flags);
2344 2345
}

2346 2347 2348
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
					unsigned int hctx_idx,
					unsigned int nr_tags,
2349 2350
					unsigned int reserved_tags,
					unsigned int flags)
2351
{
2352
	struct blk_mq_tags *tags;
2353
	int node;
2354

2355
	node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
2356 2357 2358
	if (node == NUMA_NO_NODE)
		node = set->numa_node;

2359
	tags = blk_mq_init_tags(nr_tags, reserved_tags, node, flags);
2360 2361
	if (!tags)
		return NULL;
2362

2363
	tags->rqs = kcalloc_node(nr_tags, sizeof(struct request *),
2364
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
2365
				 node);
2366
	if (!tags->rqs) {
2367
		blk_mq_free_tags(tags, flags);
2368 2369
		return NULL;
	}
2370

2371 2372 2373
	tags->static_rqs = kcalloc_node(nr_tags, sizeof(struct request *),
					GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
					node);
J
Jens Axboe 已提交
2374 2375
	if (!tags->static_rqs) {
		kfree(tags->rqs);
2376
		blk_mq_free_tags(tags, flags);
J
Jens Axboe 已提交
2377 2378 2379
		return NULL;
	}

2380 2381 2382 2383 2384 2385 2386 2387
	return tags;
}

static size_t order_to_size(unsigned int order)
{
	return (size_t)PAGE_SIZE << order;
}

2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398
static int blk_mq_init_request(struct blk_mq_tag_set *set, struct request *rq,
			       unsigned int hctx_idx, int node)
{
	int ret;

	if (set->ops->init_request) {
		ret = set->ops->init_request(set, rq, hctx_idx, node);
		if (ret)
			return ret;
	}

K
Keith Busch 已提交
2399
	WRITE_ONCE(rq->state, MQ_RQ_IDLE);
2400 2401 2402
	return 0;
}

2403 2404 2405 2406 2407
int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx, unsigned int depth)
{
	unsigned int i, j, entries_per_page, max_order = 4;
	size_t rq_size, left;
2408 2409
	int node;

2410
	node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], hctx_idx);
2411 2412
	if (node == NUMA_NO_NODE)
		node = set->numa_node;
2413 2414 2415

	INIT_LIST_HEAD(&tags->page_list);

2416 2417 2418 2419
	/*
	 * rq_size is the size of the request plus driver payload, rounded
	 * to the cacheline size
	 */
2420
	rq_size = round_up(sizeof(struct request) + set->cmd_size,
2421
				cache_line_size());
2422
	left = rq_size * depth;
2423

2424
	for (i = 0; i < depth; ) {
2425 2426 2427 2428 2429
		int this_order = max_order;
		struct page *page;
		int to_do;
		void *p;

2430
		while (this_order && left < order_to_size(this_order - 1))
2431 2432 2433
			this_order--;

		do {
2434
			page = alloc_pages_node(node,
2435
				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
2436
				this_order);
2437 2438 2439 2440 2441 2442 2443 2444 2445
			if (page)
				break;
			if (!this_order--)
				break;
			if (order_to_size(this_order) < rq_size)
				break;
		} while (1);

		if (!page)
2446
			goto fail;
2447 2448

		page->private = this_order;
2449
		list_add_tail(&page->lru, &tags->page_list);
2450 2451

		p = page_address(page);
2452 2453 2454 2455
		/*
		 * Allow kmemleak to scan these pages as they contain pointers
		 * to additional allocations like via ops->init_request().
		 */
2456
		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
2457
		entries_per_page = order_to_size(this_order) / rq_size;
2458
		to_do = min(entries_per_page, depth - i);
2459 2460
		left -= to_do * rq_size;
		for (j = 0; j < to_do; j++) {
J
Jens Axboe 已提交
2461 2462 2463
			struct request *rq = p;

			tags->static_rqs[i] = rq;
2464 2465 2466
			if (blk_mq_init_request(set, rq, hctx_idx, node)) {
				tags->static_rqs[i] = NULL;
				goto fail;
2467 2468
			}

2469 2470 2471 2472
			p += rq_size;
			i++;
		}
	}
2473
	return 0;
2474

2475
fail:
2476 2477
	blk_mq_free_rqs(set, tags, hctx_idx);
	return -ENOMEM;
2478 2479
}

2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559
struct rq_iter_data {
	struct blk_mq_hw_ctx *hctx;
	bool has_rq;
};

static bool blk_mq_has_request(struct request *rq, void *data, bool reserved)
{
	struct rq_iter_data *iter_data = data;

	if (rq->mq_hctx != iter_data->hctx)
		return true;
	iter_data->has_rq = true;
	return false;
}

static bool blk_mq_hctx_has_requests(struct blk_mq_hw_ctx *hctx)
{
	struct blk_mq_tags *tags = hctx->sched_tags ?
			hctx->sched_tags : hctx->tags;
	struct rq_iter_data data = {
		.hctx	= hctx,
	};

	blk_mq_all_tag_iter(tags, blk_mq_has_request, &data);
	return data.has_rq;
}

static inline bool blk_mq_last_cpu_in_hctx(unsigned int cpu,
		struct blk_mq_hw_ctx *hctx)
{
	if (cpumask_next_and(-1, hctx->cpumask, cpu_online_mask) != cpu)
		return false;
	if (cpumask_next_and(cpu, hctx->cpumask, cpu_online_mask) < nr_cpu_ids)
		return false;
	return true;
}

static int blk_mq_hctx_notify_offline(unsigned int cpu, struct hlist_node *node)
{
	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
			struct blk_mq_hw_ctx, cpuhp_online);

	if (!cpumask_test_cpu(cpu, hctx->cpumask) ||
	    !blk_mq_last_cpu_in_hctx(cpu, hctx))
		return 0;

	/*
	 * Prevent new request from being allocated on the current hctx.
	 *
	 * The smp_mb__after_atomic() Pairs with the implied barrier in
	 * test_and_set_bit_lock in sbitmap_get().  Ensures the inactive flag is
	 * seen once we return from the tag allocator.
	 */
	set_bit(BLK_MQ_S_INACTIVE, &hctx->state);
	smp_mb__after_atomic();

	/*
	 * Try to grab a reference to the queue and wait for any outstanding
	 * requests.  If we could not grab a reference the queue has been
	 * frozen and there are no requests.
	 */
	if (percpu_ref_tryget(&hctx->queue->q_usage_counter)) {
		while (blk_mq_hctx_has_requests(hctx))
			msleep(5);
		percpu_ref_put(&hctx->queue->q_usage_counter);
	}

	return 0;
}

static int blk_mq_hctx_notify_online(unsigned int cpu, struct hlist_node *node)
{
	struct blk_mq_hw_ctx *hctx = hlist_entry_safe(node,
			struct blk_mq_hw_ctx, cpuhp_online);

	if (cpumask_test_cpu(cpu, hctx->cpumask))
		clear_bit(BLK_MQ_S_INACTIVE, &hctx->state);
	return 0;
}

J
Jens Axboe 已提交
2560 2561 2562 2563 2564
/*
 * 'cpu' is going away. splice any existing rq_list entries from this
 * software queue to the hw queue dispatch list, and ensure that it
 * gets run.
 */
2565
static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
2566
{
2567
	struct blk_mq_hw_ctx *hctx;
2568 2569
	struct blk_mq_ctx *ctx;
	LIST_HEAD(tmp);
M
Ming Lei 已提交
2570
	enum hctx_type type;
2571

2572
	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
2573 2574 2575
	if (!cpumask_test_cpu(cpu, hctx->cpumask))
		return 0;

J
Jens Axboe 已提交
2576
	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
M
Ming Lei 已提交
2577
	type = hctx->type;
2578 2579

	spin_lock(&ctx->lock);
M
Ming Lei 已提交
2580 2581
	if (!list_empty(&ctx->rq_lists[type])) {
		list_splice_init(&ctx->rq_lists[type], &tmp);
2582 2583 2584 2585 2586
		blk_mq_hctx_clear_pending(hctx, ctx);
	}
	spin_unlock(&ctx->lock);

	if (list_empty(&tmp))
2587
		return 0;
2588

J
Jens Axboe 已提交
2589 2590 2591
	spin_lock(&hctx->lock);
	list_splice_tail_init(&tmp, &hctx->dispatch);
	spin_unlock(&hctx->lock);
2592 2593

	blk_mq_run_hw_queue(hctx, true);
2594
	return 0;
2595 2596
}

2597
static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
2598
{
2599 2600 2601
	if (!(hctx->flags & BLK_MQ_F_STACKING))
		cpuhp_state_remove_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
						    &hctx->cpuhp_online);
2602 2603
	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
					    &hctx->cpuhp_dead);
2604 2605
}

2606
/* hctx->ctxs will be freed in queue's release handler */
2607 2608 2609 2610
static void blk_mq_exit_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
2611 2612
	if (blk_mq_hw_queue_mapped(hctx))
		blk_mq_tag_idle(hctx);
2613

2614
	if (set->ops->exit_request)
2615
		set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
2616

2617 2618 2619
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);

2620
	blk_mq_remove_cpuhp(hctx);
2621 2622 2623 2624

	spin_lock(&q->unused_hctx_lock);
	list_add(&hctx->hctx_list, &q->unused_hctx_list);
	spin_unlock(&q->unused_hctx_lock);
2625 2626
}

M
Ming Lei 已提交
2627 2628 2629 2630 2631 2632 2633 2634 2635
static void blk_mq_exit_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set, int nr_queue)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (i == nr_queue)
			break;
2636
		blk_mq_debugfs_unregister_hctx(hctx);
2637
		blk_mq_exit_hctx(q, set, hctx, i);
M
Ming Lei 已提交
2638 2639 2640
	}
}

2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654
static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set)
{
	int hw_ctx_size = sizeof(struct blk_mq_hw_ctx);

	BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, srcu),
			   __alignof__(struct blk_mq_hw_ctx)) !=
		     sizeof(struct blk_mq_hw_ctx));

	if (tag_set->flags & BLK_MQ_F_BLOCKING)
		hw_ctx_size += sizeof(struct srcu_struct);

	return hw_ctx_size;
}

2655 2656 2657
static int blk_mq_init_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
2658
{
2659 2660
	hctx->queue_num = hctx_idx;

2661 2662 2663
	if (!(hctx->flags & BLK_MQ_F_STACKING))
		cpuhp_state_add_instance_nocalls(CPUHP_AP_BLK_MQ_ONLINE,
				&hctx->cpuhp_online);
2664 2665 2666 2667 2668 2669 2670
	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);

	hctx->tags = set->tags[hctx_idx];

	if (set->ops->init_hctx &&
	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
		goto unregister_cpu_notifier;
2671

2672 2673 2674 2675 2676 2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699
	if (blk_mq_init_request(set, hctx->fq->flush_rq, hctx_idx,
				hctx->numa_node))
		goto exit_hctx;
	return 0;

 exit_hctx:
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);
 unregister_cpu_notifier:
	blk_mq_remove_cpuhp(hctx);
	return -1;
}

static struct blk_mq_hw_ctx *
blk_mq_alloc_hctx(struct request_queue *q, struct blk_mq_tag_set *set,
		int node)
{
	struct blk_mq_hw_ctx *hctx;
	gfp_t gfp = GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY;

	hctx = kzalloc_node(blk_mq_hw_ctx_size(set), gfp, node);
	if (!hctx)
		goto fail_alloc_hctx;

	if (!zalloc_cpumask_var_node(&hctx->cpumask, gfp, node))
		goto free_hctx;

	atomic_set(&hctx->nr_active, 0);
2700
	if (node == NUMA_NO_NODE)
2701 2702
		node = set->numa_node;
	hctx->numa_node = node;
2703

2704
	INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
2705 2706 2707
	spin_lock_init(&hctx->lock);
	INIT_LIST_HEAD(&hctx->dispatch);
	hctx->queue = q;
2708
	hctx->flags = set->flags & ~BLK_MQ_F_TAG_QUEUE_SHARED;
2709

2710 2711
	INIT_LIST_HEAD(&hctx->hctx_list);

2712
	/*
2713 2714
	 * Allocate space for all possible cpus to avoid allocation at
	 * runtime
2715
	 */
2716
	hctx->ctxs = kmalloc_array_node(nr_cpu_ids, sizeof(void *),
2717
			gfp, node);
2718
	if (!hctx->ctxs)
2719
		goto free_cpumask;
2720

2721
	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8),
2722
				gfp, node, false, false))
2723 2724
		goto free_ctxs;
	hctx->nr_ctx = 0;
2725

2726
	spin_lock_init(&hctx->dispatch_wait_lock);
2727 2728 2729
	init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
	INIT_LIST_HEAD(&hctx->dispatch_wait.entry);

2730
	hctx->fq = blk_alloc_flush_queue(hctx->numa_node, set->cmd_size, gfp);
2731
	if (!hctx->fq)
2732
		goto free_bitmap;
2733

2734
	if (hctx->flags & BLK_MQ_F_BLOCKING)
2735
		init_srcu_struct(hctx->srcu);
2736
	blk_mq_hctx_kobj_init(hctx);
2737

2738
	return hctx;
2739

2740
 free_bitmap:
2741
	sbitmap_free(&hctx->ctx_map);
2742 2743
 free_ctxs:
	kfree(hctx->ctxs);
2744 2745 2746 2747 2748 2749
 free_cpumask:
	free_cpumask_var(hctx->cpumask);
 free_hctx:
	kfree(hctx);
 fail_alloc_hctx:
	return NULL;
2750
}
2751 2752 2753 2754

static void blk_mq_init_cpu_queues(struct request_queue *q,
				   unsigned int nr_hw_queues)
{
J
Jens Axboe 已提交
2755 2756
	struct blk_mq_tag_set *set = q->tag_set;
	unsigned int i, j;
2757 2758 2759 2760

	for_each_possible_cpu(i) {
		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
		struct blk_mq_hw_ctx *hctx;
M
Ming Lei 已提交
2761
		int k;
2762 2763 2764

		__ctx->cpu = i;
		spin_lock_init(&__ctx->lock);
M
Ming Lei 已提交
2765 2766 2767
		for (k = HCTX_TYPE_DEFAULT; k < HCTX_MAX_TYPES; k++)
			INIT_LIST_HEAD(&__ctx->rq_lists[k]);

2768 2769 2770 2771 2772 2773
		__ctx->queue = q;

		/*
		 * Set local node, IFF we have more than one hw queue. If
		 * not, we remain on the home node of the device
		 */
J
Jens Axboe 已提交
2774 2775 2776
		for (j = 0; j < set->nr_maps; j++) {
			hctx = blk_mq_map_queue_type(q, j, i);
			if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
2777
				hctx->numa_node = cpu_to_node(i);
J
Jens Axboe 已提交
2778
		}
2779 2780 2781
	}
}

2782 2783
static bool __blk_mq_alloc_map_and_request(struct blk_mq_tag_set *set,
					int hctx_idx)
2784
{
2785
	unsigned int flags = set->flags;
2786 2787 2788
	int ret = 0;

	set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
2789
					set->queue_depth, set->reserved_tags, flags);
2790 2791 2792 2793 2794 2795 2796 2797
	if (!set->tags[hctx_idx])
		return false;

	ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
				set->queue_depth);
	if (!ret)
		return true;

2798
	blk_mq_free_rq_map(set->tags[hctx_idx], flags);
2799 2800 2801 2802 2803 2804 2805
	set->tags[hctx_idx] = NULL;
	return false;
}

static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
					 unsigned int hctx_idx)
{
2806 2807
	unsigned int flags = set->flags;

2808
	if (set->tags && set->tags[hctx_idx]) {
2809
		blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
2810
		blk_mq_free_rq_map(set->tags[hctx_idx], flags);
2811 2812
		set->tags[hctx_idx] = NULL;
	}
2813 2814
}

2815
static void blk_mq_map_swqueue(struct request_queue *q)
2816
{
J
Jens Axboe 已提交
2817
	unsigned int i, j, hctx_idx;
2818 2819
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
M
Ming Lei 已提交
2820
	struct blk_mq_tag_set *set = q->tag_set;
2821 2822

	queue_for_each_hw_ctx(q, hctx, i) {
2823
		cpumask_clear(hctx->cpumask);
2824
		hctx->nr_ctx = 0;
2825
		hctx->dispatch_from = NULL;
2826 2827 2828
	}

	/*
2829
	 * Map software to hardware queues.
2830 2831
	 *
	 * If the cpu isn't present, the cpu is mapped to first hctx.
2832
	 */
2833
	for_each_possible_cpu(i) {
2834

2835
		ctx = per_cpu_ptr(q->queue_ctx, i);
J
Jens Axboe 已提交
2836
		for (j = 0; j < set->nr_maps; j++) {
2837 2838 2839
			if (!set->map[j].nr_queues) {
				ctx->hctxs[j] = blk_mq_map_queue_type(q,
						HCTX_TYPE_DEFAULT, i);
2840
				continue;
2841
			}
2842 2843 2844
			hctx_idx = set->map[j].mq_map[i];
			/* unmapped hw queue can be remapped after CPU topo changed */
			if (!set->tags[hctx_idx] &&
2845
			    !__blk_mq_alloc_map_and_request(set, hctx_idx)) {
2846 2847 2848 2849 2850 2851 2852 2853
				/*
				 * If tags initialization fail for some hctx,
				 * that hctx won't be brought online.  In this
				 * case, remap the current ctx to hctx[0] which
				 * is guaranteed to always have tags allocated
				 */
				set->map[j].mq_map[i] = 0;
			}
2854

J
Jens Axboe 已提交
2855
			hctx = blk_mq_map_queue_type(q, j, i);
2856
			ctx->hctxs[j] = hctx;
J
Jens Axboe 已提交
2857 2858 2859 2860 2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875
			/*
			 * If the CPU is already set in the mask, then we've
			 * mapped this one already. This can happen if
			 * devices share queues across queue maps.
			 */
			if (cpumask_test_cpu(i, hctx->cpumask))
				continue;

			cpumask_set_cpu(i, hctx->cpumask);
			hctx->type = j;
			ctx->index_hw[hctx->type] = hctx->nr_ctx;
			hctx->ctxs[hctx->nr_ctx++] = ctx;

			/*
			 * If the nr_ctx type overflows, we have exceeded the
			 * amount of sw queues we can support.
			 */
			BUG_ON(!hctx->nr_ctx);
		}
2876 2877 2878 2879

		for (; j < HCTX_MAX_TYPES; j++)
			ctx->hctxs[j] = blk_mq_map_queue_type(q,
					HCTX_TYPE_DEFAULT, i);
2880
	}
2881 2882

	queue_for_each_hw_ctx(q, hctx, i) {
2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897
		/*
		 * If no software queues are mapped to this hardware queue,
		 * disable it and free the request entries.
		 */
		if (!hctx->nr_ctx) {
			/* Never unmap queue 0.  We need it as a
			 * fallback in case of a new remap fails
			 * allocation
			 */
			if (i && set->tags[i])
				blk_mq_free_map_and_requests(set, i);

			hctx->tags = NULL;
			continue;
		}
2898

M
Ming Lei 已提交
2899 2900 2901
		hctx->tags = set->tags[i];
		WARN_ON(!hctx->tags);

2902 2903 2904 2905 2906
		/*
		 * Set the map size to the number of mapped software queues.
		 * This is more accurate and more efficient than looping
		 * over all possibly mapped software queues.
		 */
2907
		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
2908

2909 2910 2911
		/*
		 * Initialize batch roundrobin counts
		 */
2912
		hctx->next_cpu = blk_mq_first_mapped_cpu(hctx);
2913 2914
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}
2915 2916
}

2917 2918 2919 2920
/*
 * Caller needs to ensure that we're either frozen/quiesced, or that
 * the queue isn't live yet.
 */
2921
static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2922 2923 2924 2925
{
	struct blk_mq_hw_ctx *hctx;
	int i;

2926
	queue_for_each_hw_ctx(q, hctx, i) {
2927
		if (shared)
2928
			hctx->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
2929
		else
2930
			hctx->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
2931 2932 2933
	}
}

2934 2935
static void blk_mq_update_tag_set_shared(struct blk_mq_tag_set *set,
					 bool shared)
2936 2937
{
	struct request_queue *q;
2938

2939 2940
	lockdep_assert_held(&set->tag_list_lock);

2941 2942
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_freeze_queue(q);
2943
		queue_set_hctx_shared(q, shared);
2944 2945 2946 2947 2948 2949 2950 2951 2952
		blk_mq_unfreeze_queue(q);
	}
}

static void blk_mq_del_queue_tag_set(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;

	mutex_lock(&set->tag_list_lock);
2953
	list_del(&q->tag_set_list);
2954 2955
	if (list_is_singular(&set->tag_list)) {
		/* just transitioned to unshared */
2956
		set->flags &= ~BLK_MQ_F_TAG_QUEUE_SHARED;
2957
		/* update existing queue */
2958
		blk_mq_update_tag_set_shared(set, false);
2959
	}
2960
	mutex_unlock(&set->tag_list_lock);
2961
	INIT_LIST_HEAD(&q->tag_set_list);
2962 2963 2964 2965 2966 2967
}

static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
				     struct request_queue *q)
{
	mutex_lock(&set->tag_list_lock);
2968

2969 2970 2971 2972
	/*
	 * Check to see if we're transitioning to shared (from 1 to 2 queues).
	 */
	if (!list_empty(&set->tag_list) &&
2973 2974
	    !(set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)) {
		set->flags |= BLK_MQ_F_TAG_QUEUE_SHARED;
2975
		/* update existing queue */
2976
		blk_mq_update_tag_set_shared(set, true);
2977
	}
2978
	if (set->flags & BLK_MQ_F_TAG_QUEUE_SHARED)
2979
		queue_set_hctx_shared(q, true);
2980
	list_add_tail(&q->tag_set_list, &set->tag_list);
2981

2982 2983 2984
	mutex_unlock(&set->tag_list_lock);
}

2985 2986 2987 2988 2989 2990 2991 2992 2993 2994 2995 2996 2997 2998 2999 3000 3001 3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012
/* All allocations will be freed in release handler of q->mq_kobj */
static int blk_mq_alloc_ctxs(struct request_queue *q)
{
	struct blk_mq_ctxs *ctxs;
	int cpu;

	ctxs = kzalloc(sizeof(*ctxs), GFP_KERNEL);
	if (!ctxs)
		return -ENOMEM;

	ctxs->queue_ctx = alloc_percpu(struct blk_mq_ctx);
	if (!ctxs->queue_ctx)
		goto fail;

	for_each_possible_cpu(cpu) {
		struct blk_mq_ctx *ctx = per_cpu_ptr(ctxs->queue_ctx, cpu);
		ctx->ctxs = ctxs;
	}

	q->mq_kobj = &ctxs->kobj;
	q->queue_ctx = ctxs->queue_ctx;

	return 0;
 fail:
	kfree(ctxs);
	return -ENOMEM;
}

3013 3014 3015 3016 3017 3018 3019 3020
/*
 * It is the actual release handler for mq, but we do it from
 * request queue's release handler for avoiding use-after-free
 * and headache because q->mq_kobj shouldn't have been introduced,
 * but we can't group ctx/kctx kobj without it.
 */
void blk_mq_release(struct request_queue *q)
{
3021 3022
	struct blk_mq_hw_ctx *hctx, *next;
	int i;
3023

3024 3025 3026 3027 3028 3029
	queue_for_each_hw_ctx(q, hctx, i)
		WARN_ON_ONCE(hctx && list_empty(&hctx->hctx_list));

	/* all hctx are in .unused_hctx_list now */
	list_for_each_entry_safe(hctx, next, &q->unused_hctx_list, hctx_list) {
		list_del_init(&hctx->hctx_list);
3030
		kobject_put(&hctx->kobj);
3031
	}
3032 3033 3034

	kfree(q->queue_hw_ctx);

3035 3036 3037 3038 3039
	/*
	 * release .mq_kobj and sw queue's kobject now because
	 * both share lifetime with request queue.
	 */
	blk_mq_sysfs_deinit(q);
3040 3041
}

3042 3043
struct request_queue *blk_mq_init_queue_data(struct blk_mq_tag_set *set,
		void *queuedata)
3044 3045 3046
{
	struct request_queue *uninit_q, *q;

3047
	uninit_q = blk_alloc_queue(set->numa_node);
3048 3049
	if (!uninit_q)
		return ERR_PTR(-ENOMEM);
3050
	uninit_q->queuedata = queuedata;
3051

3052 3053 3054 3055 3056
	/*
	 * Initialize the queue without an elevator. device_add_disk() will do
	 * the initialization.
	 */
	q = blk_mq_init_allocated_queue(set, uninit_q, false);
3057 3058 3059 3060 3061
	if (IS_ERR(q))
		blk_cleanup_queue(uninit_q);

	return q;
}
3062 3063 3064 3065 3066 3067
EXPORT_SYMBOL_GPL(blk_mq_init_queue_data);

struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
{
	return blk_mq_init_queue_data(set, NULL);
}
3068 3069
EXPORT_SYMBOL(blk_mq_init_queue);

3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084
/*
 * Helper for setting up a queue with mq ops, given queue depth, and
 * the passed in mq ops flags.
 */
struct request_queue *blk_mq_init_sq_queue(struct blk_mq_tag_set *set,
					   const struct blk_mq_ops *ops,
					   unsigned int queue_depth,
					   unsigned int set_flags)
{
	struct request_queue *q;
	int ret;

	memset(set, 0, sizeof(*set));
	set->ops = ops;
	set->nr_hw_queues = 1;
J
Jens Axboe 已提交
3085
	set->nr_maps = 1;
3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103
	set->queue_depth = queue_depth;
	set->numa_node = NUMA_NO_NODE;
	set->flags = set_flags;

	ret = blk_mq_alloc_tag_set(set);
	if (ret)
		return ERR_PTR(ret);

	q = blk_mq_init_queue(set);
	if (IS_ERR(q)) {
		blk_mq_free_tag_set(set);
		return q;
	}

	return q;
}
EXPORT_SYMBOL(blk_mq_init_sq_queue);

3104 3105 3106 3107
static struct blk_mq_hw_ctx *blk_mq_alloc_and_init_hctx(
		struct blk_mq_tag_set *set, struct request_queue *q,
		int hctx_idx, int node)
{
3108
	struct blk_mq_hw_ctx *hctx = NULL, *tmp;
3109

3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
	/* reuse dead hctx first */
	spin_lock(&q->unused_hctx_lock);
	list_for_each_entry(tmp, &q->unused_hctx_list, hctx_list) {
		if (tmp->numa_node == node) {
			hctx = tmp;
			break;
		}
	}
	if (hctx)
		list_del_init(&hctx->hctx_list);
	spin_unlock(&q->unused_hctx_lock);

	if (!hctx)
		hctx = blk_mq_alloc_hctx(q, set, node);
3124
	if (!hctx)
3125
		goto fail;
3126

3127 3128
	if (blk_mq_init_hctx(q, set, hctx, hctx_idx))
		goto free_hctx;
3129 3130

	return hctx;
3131 3132 3133 3134 3135

 free_hctx:
	kobject_put(&hctx->kobj);
 fail:
	return NULL;
3136 3137
}

K
Keith Busch 已提交
3138 3139
static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
						struct request_queue *q)
3140
{
3141
	int i, j, end;
K
Keith Busch 已提交
3142
	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
3143

3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159
	if (q->nr_hw_queues < set->nr_hw_queues) {
		struct blk_mq_hw_ctx **new_hctxs;

		new_hctxs = kcalloc_node(set->nr_hw_queues,
				       sizeof(*new_hctxs), GFP_KERNEL,
				       set->numa_node);
		if (!new_hctxs)
			return;
		if (hctxs)
			memcpy(new_hctxs, hctxs, q->nr_hw_queues *
			       sizeof(*hctxs));
		q->queue_hw_ctx = new_hctxs;
		kfree(hctxs);
		hctxs = new_hctxs;
	}

3160 3161
	/* protect against switching io scheduler  */
	mutex_lock(&q->sysfs_lock);
3162
	for (i = 0; i < set->nr_hw_queues; i++) {
K
Keith Busch 已提交
3163
		int node;
3164
		struct blk_mq_hw_ctx *hctx;
K
Keith Busch 已提交
3165

3166
		node = blk_mq_hw_queue_to_node(&set->map[HCTX_TYPE_DEFAULT], i);
3167 3168 3169 3170 3171 3172 3173
		/*
		 * If the hw queue has been mapped to another numa node,
		 * we need to realloc the hctx. If allocation fails, fallback
		 * to use the previous one.
		 */
		if (hctxs[i] && (hctxs[i]->numa_node == node))
			continue;
K
Keith Busch 已提交
3174

3175 3176
		hctx = blk_mq_alloc_and_init_hctx(set, q, i, node);
		if (hctx) {
3177
			if (hctxs[i])
3178 3179 3180 3181 3182 3183 3184 3185 3186
				blk_mq_exit_hctx(q, set, hctxs[i], i);
			hctxs[i] = hctx;
		} else {
			if (hctxs[i])
				pr_warn("Allocate new hctx on node %d fails,\
						fallback to previous one on node %d\n",
						node, hctxs[i]->numa_node);
			else
				break;
K
Keith Busch 已提交
3187
		}
3188
	}
3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200
	/*
	 * Increasing nr_hw_queues fails. Free the newly allocated
	 * hctxs and keep the previous q->nr_hw_queues.
	 */
	if (i != set->nr_hw_queues) {
		j = q->nr_hw_queues;
		end = i;
	} else {
		j = i;
		end = q->nr_hw_queues;
		q->nr_hw_queues = set->nr_hw_queues;
	}
3201

3202
	for (; j < end; j++) {
K
Keith Busch 已提交
3203 3204 3205
		struct blk_mq_hw_ctx *hctx = hctxs[j];

		if (hctx) {
3206 3207
			if (hctx->tags)
				blk_mq_free_map_and_requests(set, j);
K
Keith Busch 已提交
3208 3209 3210 3211
			blk_mq_exit_hctx(q, set, hctx, j);
			hctxs[j] = NULL;
		}
	}
3212
	mutex_unlock(&q->sysfs_lock);
K
Keith Busch 已提交
3213 3214 3215
}

struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
3216 3217
						  struct request_queue *q,
						  bool elevator_init)
K
Keith Busch 已提交
3218
{
M
Ming Lei 已提交
3219 3220 3221
	/* mark the queue as mq asap */
	q->mq_ops = set->ops;

3222
	q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn,
3223 3224
					     blk_mq_poll_stats_bkt,
					     BLK_MQ_POLL_STATS_BKTS, q);
3225 3226 3227
	if (!q->poll_cb)
		goto err_exit;

3228
	if (blk_mq_alloc_ctxs(q))
3229
		goto err_poll;
K
Keith Busch 已提交
3230

3231 3232 3233
	/* init q->mq_kobj and sw queues' kobjects */
	blk_mq_sysfs_init(q);

3234 3235 3236
	INIT_LIST_HEAD(&q->unused_hctx_list);
	spin_lock_init(&q->unused_hctx_lock);

K
Keith Busch 已提交
3237 3238 3239
	blk_mq_realloc_hw_ctxs(set, q);
	if (!q->nr_hw_queues)
		goto err_hctxs;
3240

3241
	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
3242
	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
3243

J
Jens Axboe 已提交
3244
	q->tag_set = set;
3245

3246
	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
3247 3248
	if (set->nr_maps > HCTX_TYPE_POLL &&
	    set->map[HCTX_TYPE_POLL].nr_queues)
3249
		blk_queue_flag_set(QUEUE_FLAG_POLL, q);
3250

3251 3252
	q->sg_reserved_size = INT_MAX;

3253
	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
3254 3255 3256
	INIT_LIST_HEAD(&q->requeue_list);
	spin_lock_init(&q->requeue_lock);

3257 3258
	q->nr_requests = set->queue_depth;

3259 3260 3261
	/*
	 * Default to classic polling
	 */
3262
	q->poll_nsec = BLK_MQ_POLL_CLASSIC;
3263

3264
	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
3265
	blk_mq_add_queue_tag_set(set, q);
3266
	blk_mq_map_swqueue(q);
3267

3268 3269
	if (elevator_init)
		elevator_init_mq(q);
3270

3271
	return q;
3272

3273
err_hctxs:
K
Keith Busch 已提交
3274
	kfree(q->queue_hw_ctx);
3275
	q->nr_hw_queues = 0;
3276
	blk_mq_sysfs_deinit(q);
3277 3278 3279
err_poll:
	blk_stat_free_callback(q->poll_cb);
	q->poll_cb = NULL;
M
Ming Lin 已提交
3280 3281
err_exit:
	q->mq_ops = NULL;
3282 3283
	return ERR_PTR(-ENOMEM);
}
3284
EXPORT_SYMBOL(blk_mq_init_allocated_queue);
3285

3286 3287
/* tags can _not_ be used after returning from blk_mq_exit_queue */
void blk_mq_exit_queue(struct request_queue *q)
3288
{
M
Ming Lei 已提交
3289
	struct blk_mq_tag_set	*set = q->tag_set;
3290

3291
	blk_mq_del_queue_tag_set(q);
M
Ming Lei 已提交
3292
	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
3293 3294
}

3295 3296 3297 3298
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	int i;

3299
	for (i = 0; i < set->nr_hw_queues; i++) {
3300
		if (!__blk_mq_alloc_map_and_request(set, i))
3301
			goto out_unwind;
3302 3303
		cond_resched();
	}
3304 3305 3306 3307 3308

	return 0;

out_unwind:
	while (--i >= 0)
3309
		blk_mq_free_map_and_requests(set, i);
3310 3311 3312 3313 3314 3315 3316 3317 3318

	return -ENOMEM;
}

/*
 * Allocate the request maps associated with this tag_set. Note that this
 * may reduce the depth asked for, if memory is tight. set->queue_depth
 * will be updated to reflect the allocated depth.
 */
3319
static int blk_mq_alloc_map_and_requests(struct blk_mq_tag_set *set)
3320 3321 3322 3323 3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338 3339 3340 3341 3342 3343 3344 3345 3346 3347 3348
{
	unsigned int depth;
	int err;

	depth = set->queue_depth;
	do {
		err = __blk_mq_alloc_rq_maps(set);
		if (!err)
			break;

		set->queue_depth >>= 1;
		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
			err = -ENOMEM;
			break;
		}
	} while (set->queue_depth);

	if (!set->queue_depth || err) {
		pr_err("blk-mq: failed to allocate request map\n");
		return -ENOMEM;
	}

	if (depth != set->queue_depth)
		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
						depth, set->queue_depth);

	return 0;
}

3349 3350
static int blk_mq_update_queue_map(struct blk_mq_tag_set *set)
{
3351 3352 3353 3354 3355 3356 3357 3358
	/*
	 * blk_mq_map_queues() and multiple .map_queues() implementations
	 * expect that set->map[HCTX_TYPE_DEFAULT].nr_queues is set to the
	 * number of hardware queues.
	 */
	if (set->nr_maps == 1)
		set->map[HCTX_TYPE_DEFAULT].nr_queues = set->nr_hw_queues;

3359
	if (set->ops->map_queues && !is_kdump_kernel()) {
J
Jens Axboe 已提交
3360 3361
		int i;

3362 3363 3364 3365 3366 3367 3368
		/*
		 * transport .map_queues is usually done in the following
		 * way:
		 *
		 * for (queue = 0; queue < set->nr_hw_queues; queue++) {
		 * 	mask = get_cpu_mask(queue)
		 * 	for_each_cpu(cpu, mask)
J
Jens Axboe 已提交
3369
		 * 		set->map[x].mq_map[cpu] = queue;
3370 3371 3372 3373 3374 3375
		 * }
		 *
		 * When we need to remap, the table has to be cleared for
		 * killing stale mapping since one CPU may not be mapped
		 * to any hw queue.
		 */
J
Jens Axboe 已提交
3376 3377
		for (i = 0; i < set->nr_maps; i++)
			blk_mq_clear_mq_map(&set->map[i]);
3378

3379
		return set->ops->map_queues(set);
J
Jens Axboe 已提交
3380 3381
	} else {
		BUG_ON(set->nr_maps > 1);
3382
		return blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
J
Jens Axboe 已提交
3383
	}
3384 3385
}

3386 3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406 3407 3408
static int blk_mq_realloc_tag_set_tags(struct blk_mq_tag_set *set,
				  int cur_nr_hw_queues, int new_nr_hw_queues)
{
	struct blk_mq_tags **new_tags;

	if (cur_nr_hw_queues >= new_nr_hw_queues)
		return 0;

	new_tags = kcalloc_node(new_nr_hw_queues, sizeof(struct blk_mq_tags *),
				GFP_KERNEL, set->numa_node);
	if (!new_tags)
		return -ENOMEM;

	if (set->tags)
		memcpy(new_tags, set->tags, cur_nr_hw_queues *
		       sizeof(*set->tags));
	kfree(set->tags);
	set->tags = new_tags;
	set->nr_hw_queues = new_nr_hw_queues;

	return 0;
}

3409 3410 3411 3412 3413 3414
static int blk_mq_alloc_tag_set_tags(struct blk_mq_tag_set *set,
				int new_nr_hw_queues)
{
	return blk_mq_realloc_tag_set_tags(set, 0, new_nr_hw_queues);
}

3415 3416 3417
/*
 * Alloc a tag set to be associated with one or more request queues.
 * May fail with EINVAL for various error conditions. May adjust the
3418
 * requested depth down, if it's too large. In that case, the set
3419 3420
 * value will be stored in set->queue_depth.
 */
3421 3422
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
{
J
Jens Axboe 已提交
3423
	int i, ret;
3424

B
Bart Van Assche 已提交
3425 3426
	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);

3427 3428
	if (!set->nr_hw_queues)
		return -EINVAL;
3429
	if (!set->queue_depth)
3430 3431 3432 3433
		return -EINVAL;
	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
		return -EINVAL;

C
Christoph Hellwig 已提交
3434
	if (!set->ops->queue_rq)
3435 3436
		return -EINVAL;

3437 3438 3439
	if (!set->ops->get_budget ^ !set->ops->put_budget)
		return -EINVAL;

3440 3441 3442 3443 3444
	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
		pr_info("blk-mq: reduced tag depth to %u\n",
			BLK_MQ_MAX_DEPTH);
		set->queue_depth = BLK_MQ_MAX_DEPTH;
	}
3445

J
Jens Axboe 已提交
3446 3447 3448 3449 3450
	if (!set->nr_maps)
		set->nr_maps = 1;
	else if (set->nr_maps > HCTX_MAX_TYPES)
		return -EINVAL;

3451 3452 3453 3454 3455 3456 3457
	/*
	 * If a crashdump is active, then we are potentially in a very
	 * memory constrained environment. Limit us to 1 queue and
	 * 64 tags to prevent using too much memory.
	 */
	if (is_kdump_kernel()) {
		set->nr_hw_queues = 1;
3458
		set->nr_maps = 1;
3459 3460
		set->queue_depth = min(64U, set->queue_depth);
	}
K
Keith Busch 已提交
3461
	/*
3462 3463
	 * There is no use for more h/w queues than cpus if we just have
	 * a single map
K
Keith Busch 已提交
3464
	 */
3465
	if (set->nr_maps == 1 && set->nr_hw_queues > nr_cpu_ids)
K
Keith Busch 已提交
3466
		set->nr_hw_queues = nr_cpu_ids;
3467

3468
	if (blk_mq_alloc_tag_set_tags(set, set->nr_hw_queues) < 0)
3469
		return -ENOMEM;
3470

3471
	ret = -ENOMEM;
J
Jens Axboe 已提交
3472 3473
	for (i = 0; i < set->nr_maps; i++) {
		set->map[i].mq_map = kcalloc_node(nr_cpu_ids,
3474
						  sizeof(set->map[i].mq_map[0]),
J
Jens Axboe 已提交
3475 3476 3477
						  GFP_KERNEL, set->numa_node);
		if (!set->map[i].mq_map)
			goto out_free_mq_map;
3478
		set->map[i].nr_queues = is_kdump_kernel() ? 1 : set->nr_hw_queues;
J
Jens Axboe 已提交
3479
	}
3480

3481
	ret = blk_mq_update_queue_map(set);
3482 3483 3484
	if (ret)
		goto out_free_mq_map;

3485
	ret = blk_mq_alloc_map_and_requests(set);
3486
	if (ret)
3487
		goto out_free_mq_map;
3488

3489
	if (blk_mq_is_sbitmap_shared(set->flags)) {
3490 3491
		atomic_set(&set->active_queues_shared_sbitmap, 0);

3492 3493 3494 3495 3496 3497
		if (blk_mq_init_shared_sbitmap(set, set->flags)) {
			ret = -ENOMEM;
			goto out_free_mq_rq_maps;
		}
	}

3498 3499 3500
	mutex_init(&set->tag_list_lock);
	INIT_LIST_HEAD(&set->tag_list);

3501
	return 0;
3502

3503 3504 3505
out_free_mq_rq_maps:
	for (i = 0; i < set->nr_hw_queues; i++)
		blk_mq_free_map_and_requests(set, i);
3506
out_free_mq_map:
J
Jens Axboe 已提交
3507 3508 3509 3510
	for (i = 0; i < set->nr_maps; i++) {
		kfree(set->map[i].mq_map);
		set->map[i].mq_map = NULL;
	}
3511 3512
	kfree(set->tags);
	set->tags = NULL;
3513
	return ret;
3514 3515 3516 3517 3518
}
EXPORT_SYMBOL(blk_mq_alloc_tag_set);

void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{
J
Jens Axboe 已提交
3519
	int i, j;
3520

3521
	for (i = 0; i < set->nr_hw_queues; i++)
3522
		blk_mq_free_map_and_requests(set, i);
3523

3524 3525 3526
	if (blk_mq_is_sbitmap_shared(set->flags))
		blk_mq_exit_shared_sbitmap(set);

J
Jens Axboe 已提交
3527 3528 3529 3530
	for (j = 0; j < set->nr_maps; j++) {
		kfree(set->map[j].mq_map);
		set->map[j].mq_map = NULL;
	}
3531

M
Ming Lei 已提交
3532
	kfree(set->tags);
3533
	set->tags = NULL;
3534 3535 3536
}
EXPORT_SYMBOL(blk_mq_free_tag_set);

3537 3538 3539 3540 3541 3542
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i, ret;

3543
	if (!set)
3544 3545
		return -EINVAL;

3546 3547 3548
	if (q->nr_requests == nr)
		return 0;

3549
	blk_mq_freeze_queue(q);
3550
	blk_mq_quiesce_queue(q);
3551

3552 3553
	ret = 0;
	queue_for_each_hw_ctx(q, hctx, i) {
3554 3555
		if (!hctx->tags)
			continue;
3556 3557 3558 3559
		/*
		 * If we're using an MQ scheduler, just update the scheduler
		 * queue depth. This is similar to what the old code would do.
		 */
3560
		if (!hctx->sched_tags) {
3561
			ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
3562
							false);
3563 3564
			if (!ret && blk_mq_is_sbitmap_shared(set->flags))
				blk_mq_tag_resize_shared_sbitmap(set, nr);
3565 3566 3567 3568
		} else {
			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
							nr, true);
		}
3569 3570
		if (ret)
			break;
3571 3572
		if (q->elevator && q->elevator->type->ops.depth_updated)
			q->elevator->type->ops.depth_updated(hctx);
3573 3574 3575 3576 3577
	}

	if (!ret)
		q->nr_requests = nr;

3578
	blk_mq_unquiesce_queue(q);
3579 3580
	blk_mq_unfreeze_queue(q);

3581 3582 3583
	return ret;
}

3584 3585 3586 3587 3588 3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620 3621 3622 3623 3624 3625 3626 3627 3628 3629 3630 3631 3632 3633 3634 3635 3636 3637 3638 3639 3640 3641 3642 3643 3644 3645 3646 3647 3648 3649 3650 3651 3652 3653
/*
 * request_queue and elevator_type pair.
 * It is just used by __blk_mq_update_nr_hw_queues to cache
 * the elevator_type associated with a request_queue.
 */
struct blk_mq_qe_pair {
	struct list_head node;
	struct request_queue *q;
	struct elevator_type *type;
};

/*
 * Cache the elevator_type in qe pair list and switch the
 * io scheduler to 'none'
 */
static bool blk_mq_elv_switch_none(struct list_head *head,
		struct request_queue *q)
{
	struct blk_mq_qe_pair *qe;

	if (!q->elevator)
		return true;

	qe = kmalloc(sizeof(*qe), GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY);
	if (!qe)
		return false;

	INIT_LIST_HEAD(&qe->node);
	qe->q = q;
	qe->type = q->elevator->type;
	list_add(&qe->node, head);

	mutex_lock(&q->sysfs_lock);
	/*
	 * After elevator_switch_mq, the previous elevator_queue will be
	 * released by elevator_release. The reference of the io scheduler
	 * module get by elevator_get will also be put. So we need to get
	 * a reference of the io scheduler module here to prevent it to be
	 * removed.
	 */
	__module_get(qe->type->elevator_owner);
	elevator_switch_mq(q, NULL);
	mutex_unlock(&q->sysfs_lock);

	return true;
}

static void blk_mq_elv_switch_back(struct list_head *head,
		struct request_queue *q)
{
	struct blk_mq_qe_pair *qe;
	struct elevator_type *t = NULL;

	list_for_each_entry(qe, head, node)
		if (qe->q == q) {
			t = qe->type;
			break;
		}

	if (!t)
		return;

	list_del(&qe->node);
	kfree(qe);

	mutex_lock(&q->sysfs_lock);
	elevator_switch_mq(q, t);
	mutex_unlock(&q->sysfs_lock);
}

3654 3655
static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
							int nr_hw_queues)
K
Keith Busch 已提交
3656 3657
{
	struct request_queue *q;
3658
	LIST_HEAD(head);
3659
	int prev_nr_hw_queues;
K
Keith Busch 已提交
3660

3661 3662
	lockdep_assert_held(&set->tag_list_lock);

3663
	if (set->nr_maps == 1 && nr_hw_queues > nr_cpu_ids)
K
Keith Busch 已提交
3664
		nr_hw_queues = nr_cpu_ids;
3665 3666 3667
	if (nr_hw_queues < 1)
		return;
	if (set->nr_maps == 1 && nr_hw_queues == set->nr_hw_queues)
K
Keith Busch 已提交
3668 3669 3670 3671
		return;

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_freeze_queue(q);
3672 3673 3674 3675 3676 3677 3678 3679
	/*
	 * Switch IO scheduler to 'none', cleaning up the data associated
	 * with the previous scheduler. We will switch back once we are done
	 * updating the new sw to hw queue mappings.
	 */
	list_for_each_entry(q, &set->tag_list, tag_set_list)
		if (!blk_mq_elv_switch_none(&head, q))
			goto switch_back;
K
Keith Busch 已提交
3680

3681 3682 3683 3684 3685
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_debugfs_unregister_hctxs(q);
		blk_mq_sysfs_unregister(q);
	}

3686
	prev_nr_hw_queues = set->nr_hw_queues;
3687 3688 3689 3690
	if (blk_mq_realloc_tag_set_tags(set, set->nr_hw_queues, nr_hw_queues) <
	    0)
		goto reregister;

K
Keith Busch 已提交
3691
	set->nr_hw_queues = nr_hw_queues;
3692
fallback:
3693
	blk_mq_update_queue_map(set);
K
Keith Busch 已提交
3694 3695
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_realloc_hw_ctxs(set, q);
3696 3697 3698 3699
		if (q->nr_hw_queues != set->nr_hw_queues) {
			pr_warn("Increasing nr_hw_queues to %d fails, fallback to %d\n",
					nr_hw_queues, prev_nr_hw_queues);
			set->nr_hw_queues = prev_nr_hw_queues;
3700
			blk_mq_map_queues(&set->map[HCTX_TYPE_DEFAULT]);
3701 3702
			goto fallback;
		}
3703 3704 3705
		blk_mq_map_swqueue(q);
	}

3706
reregister:
3707 3708 3709
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_sysfs_register(q);
		blk_mq_debugfs_register_hctxs(q);
K
Keith Busch 已提交
3710 3711
	}

3712 3713 3714 3715
switch_back:
	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_elv_switch_back(&head, q);

K
Keith Busch 已提交
3716 3717 3718
	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_unfreeze_queue(q);
}
3719 3720 3721 3722 3723 3724 3725

void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
{
	mutex_lock(&set->tag_list_lock);
	__blk_mq_update_nr_hw_queues(set, nr_hw_queues);
	mutex_unlock(&set->tag_list_lock);
}
K
Keith Busch 已提交
3726 3727
EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);

3728 3729 3730 3731
/* Enable polling stats and return whether they were already enabled. */
static bool blk_poll_stats_enable(struct request_queue *q)
{
	if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
3732
	    blk_queue_flag_test_and_set(QUEUE_FLAG_POLL_STATS, q))
3733 3734 3735 3736 3737 3738 3739 3740 3741 3742 3743 3744 3745 3746 3747 3748 3749 3750 3751 3752 3753
		return true;
	blk_stat_add_callback(q, q->poll_cb);
	return false;
}

static void blk_mq_poll_stats_start(struct request_queue *q)
{
	/*
	 * We don't arm the callback if polling stats are not enabled or the
	 * callback is already active.
	 */
	if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
	    blk_stat_is_active(q->poll_cb))
		return;

	blk_stat_activate_msecs(q->poll_cb, 100);
}

static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb)
{
	struct request_queue *q = cb->data;
3754
	int bucket;
3755

3756 3757 3758 3759
	for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
		if (cb->stat[bucket].nr_samples)
			q->poll_stat[bucket] = cb->stat[bucket];
	}
3760 3761
}

3762 3763 3764 3765
static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
				       struct request *rq)
{
	unsigned long ret = 0;
3766
	int bucket;
3767 3768 3769 3770 3771

	/*
	 * If stats collection isn't on, don't sleep but turn it on for
	 * future users
	 */
3772
	if (!blk_poll_stats_enable(q))
3773 3774 3775 3776 3777 3778 3779 3780
		return 0;

	/*
	 * As an optimistic guess, use half of the mean service time
	 * for this type of request. We can (and should) make this smarter.
	 * For instance, if the completion latencies are tight, we can
	 * get closer than just half the mean. This is especially
	 * important on devices where the completion latencies are longer
3781 3782
	 * than ~10 usec. We do use the stats for the relevant IO size
	 * if available which does lead to better estimates.
3783
	 */
3784 3785 3786 3787 3788 3789
	bucket = blk_mq_poll_stats_bkt(rq);
	if (bucket < 0)
		return ret;

	if (q->poll_stat[bucket].nr_samples)
		ret = (q->poll_stat[bucket].mean + 1) / 2;
3790 3791 3792 3793

	return ret;
}

3794 3795 3796 3797 3798
static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
				     struct request *rq)
{
	struct hrtimer_sleeper hs;
	enum hrtimer_mode mode;
3799
	unsigned int nsecs;
3800 3801
	ktime_t kt;

J
Jens Axboe 已提交
3802
	if (rq->rq_flags & RQF_MQ_POLL_SLEPT)
3803 3804 3805
		return false;

	/*
3806
	 * If we get here, hybrid polling is enabled. Hence poll_nsec can be:
3807 3808 3809 3810
	 *
	 *  0:	use half of prev avg
	 * >0:	use this specific value
	 */
3811
	if (q->poll_nsec > 0)
3812 3813
		nsecs = q->poll_nsec;
	else
3814
		nsecs = blk_mq_poll_nsecs(q, rq);
3815 3816

	if (!nsecs)
3817 3818
		return false;

J
Jens Axboe 已提交
3819
	rq->rq_flags |= RQF_MQ_POLL_SLEPT;
3820 3821 3822 3823 3824

	/*
	 * This will be replaced with the stats tracking code, using
	 * 'avg_completion_time / 2' as the pre-sleep target.
	 */
T
Thomas Gleixner 已提交
3825
	kt = nsecs;
3826 3827

	mode = HRTIMER_MODE_REL;
3828
	hrtimer_init_sleeper_on_stack(&hs, CLOCK_MONOTONIC, mode);
3829 3830 3831
	hrtimer_set_expires(&hs.timer, kt);

	do {
T
Tejun Heo 已提交
3832
		if (blk_mq_rq_state(rq) == MQ_RQ_COMPLETE)
3833 3834
			break;
		set_current_state(TASK_UNINTERRUPTIBLE);
3835
		hrtimer_sleeper_start_expires(&hs, mode);
3836 3837 3838 3839 3840 3841 3842 3843 3844 3845 3846
		if (hs.task)
			io_schedule();
		hrtimer_cancel(&hs.timer);
		mode = HRTIMER_MODE_ABS;
	} while (hs.task && !signal_pending(current));

	__set_current_state(TASK_RUNNING);
	destroy_hrtimer_on_stack(&hs.timer);
	return true;
}

3847 3848
static bool blk_mq_poll_hybrid(struct request_queue *q,
			       struct blk_mq_hw_ctx *hctx, blk_qc_t cookie)
J
Jens Axboe 已提交
3849
{
3850 3851
	struct request *rq;

3852
	if (q->poll_nsec == BLK_MQ_POLL_CLASSIC)
3853 3854 3855 3856 3857 3858 3859 3860 3861 3862 3863 3864 3865 3866 3867 3868
		return false;

	if (!blk_qc_t_is_internal(cookie))
		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
	else {
		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
		/*
		 * With scheduling, if the request has completed, we'll
		 * get a NULL return here, as we clear the sched tag when
		 * that happens. The request still remains valid, like always,
		 * so we should be safe with just the NULL check.
		 */
		if (!rq)
			return false;
	}

3869
	return blk_mq_poll_hybrid_sleep(q, rq);
3870 3871
}

C
Christoph Hellwig 已提交
3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884
/**
 * blk_poll - poll for IO completions
 * @q:  the queue
 * @cookie: cookie passed back at IO submission time
 * @spin: whether to spin for completions
 *
 * Description:
 *    Poll for completions on the passed in queue. Returns number of
 *    completed entries found. If @spin is true, then blk_poll will continue
 *    looping until at least one completion is found, unless the task is
 *    otherwise marked running (or we need to reschedule).
 */
int blk_poll(struct request_queue *q, blk_qc_t cookie, bool spin)
3885 3886
{
	struct blk_mq_hw_ctx *hctx;
J
Jens Axboe 已提交
3887 3888
	long state;

C
Christoph Hellwig 已提交
3889 3890
	if (!blk_qc_t_valid(cookie) ||
	    !test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
3891 3892
		return 0;

C
Christoph Hellwig 已提交
3893 3894 3895
	if (current->plug)
		blk_flush_plug_list(current->plug, false);

3896 3897
	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];

3898 3899 3900 3901 3902
	/*
	 * If we sleep, have the caller restart the poll loop to reset
	 * the state. Like for the other success return cases, the
	 * caller is responsible for checking if the IO completed. If
	 * the IO isn't complete, we'll get called again and will go
3903 3904
	 * straight to the busy poll loop. If specified not to spin,
	 * we also should not sleep.
3905
	 */
3906
	if (spin && blk_mq_poll_hybrid(q, hctx, cookie))
3907
		return 1;
3908

J
Jens Axboe 已提交
3909 3910 3911
	hctx->poll_considered++;

	state = current->state;
3912
	do {
J
Jens Axboe 已提交
3913 3914 3915 3916
		int ret;

		hctx->poll_invoked++;

3917
		ret = q->mq_ops->poll(hctx);
J
Jens Axboe 已提交
3918 3919
		if (ret > 0) {
			hctx->poll_success++;
3920
			__set_current_state(TASK_RUNNING);
3921
			return ret;
J
Jens Axboe 已提交
3922 3923 3924
		}

		if (signal_pending_state(state, current))
3925
			__set_current_state(TASK_RUNNING);
J
Jens Axboe 已提交
3926 3927

		if (current->state == TASK_RUNNING)
3928
			return 1;
3929
		if (ret < 0 || !spin)
J
Jens Axboe 已提交
3930 3931
			break;
		cpu_relax();
3932
	} while (!need_resched());
J
Jens Axboe 已提交
3933

3934
	__set_current_state(TASK_RUNNING);
3935
	return 0;
J
Jens Axboe 已提交
3936
}
C
Christoph Hellwig 已提交
3937
EXPORT_SYMBOL_GPL(blk_poll);
J
Jens Axboe 已提交
3938

J
Jens Axboe 已提交
3939 3940 3941 3942 3943 3944
unsigned int blk_mq_rq_cpu(struct request *rq)
{
	return rq->mq_ctx->cpu;
}
EXPORT_SYMBOL(blk_mq_rq_cpu);

3945 3946
static int __init blk_mq_init(void)
{
3947 3948 3949
	int i;

	for_each_possible_cpu(i)
3950
		init_llist_head(&per_cpu(blk_cpu_done, i));
3951 3952 3953 3954 3955
	open_softirq(BLOCK_SOFTIRQ, blk_done_softirq);

	cpuhp_setup_state_nocalls(CPUHP_BLOCK_SOFTIRQ_DEAD,
				  "block/softirq:dead", NULL,
				  blk_softirq_cpu_dead);
3956 3957
	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
				blk_mq_hctx_notify_dead);
3958 3959 3960
	cpuhp_setup_state_multi(CPUHP_AP_BLK_MQ_ONLINE, "block/mq:online",
				blk_mq_hctx_notify_online,
				blk_mq_hctx_notify_offline);
3961 3962 3963
	return 0;
}
subsys_initcall(blk_mq_init);