blk-mq.c 71.4 KB
Newer Older
1 2 3 4 5 6
/*
 * Block multiqueue core code
 *
 * Copyright (C) 2013-2014 Jens Axboe
 * Copyright (C) 2013-2014 Christoph Hellwig
 */
7 8 9 10 11
#include <linux/kernel.h>
#include <linux/module.h>
#include <linux/backing-dev.h>
#include <linux/bio.h>
#include <linux/blkdev.h>
12
#include <linux/kmemleak.h>
13 14 15 16 17 18 19 20 21 22
#include <linux/mm.h>
#include <linux/init.h>
#include <linux/slab.h>
#include <linux/workqueue.h>
#include <linux/smp.h>
#include <linux/llist.h>
#include <linux/list_sort.h>
#include <linux/cpu.h>
#include <linux/cache.h>
#include <linux/sched/sysctl.h>
23
#include <linux/sched/topology.h>
24
#include <linux/sched/signal.h>
25
#include <linux/delay.h>
26
#include <linux/crash_dump.h>
27
#include <linux/prefetch.h>
28 29 30 31 32 33

#include <trace/events/block.h>

#include <linux/blk-mq.h>
#include "blk.h"
#include "blk-mq.h"
34
#include "blk-mq-debugfs.h"
35
#include "blk-mq-tag.h"
36
#include "blk-stat.h"
J
Jens Axboe 已提交
37
#include "blk-wbt.h"
38
#include "blk-mq-sched.h"
39

40
static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie);
41 42 43
static void blk_mq_poll_stats_start(struct request_queue *q);
static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb);

44 45 46 47
static int blk_mq_poll_stats_bkt(const struct request *rq)
{
	int ddir, bytes, bucket;

J
Jens Axboe 已提交
48
	ddir = rq_data_dir(rq);
49 50 51 52 53 54 55 56 57 58 59 60
	bytes = blk_rq_bytes(rq);

	bucket = ddir + 2*(ilog2(bytes) - 9);

	if (bucket < 0)
		return -1;
	else if (bucket >= BLK_MQ_POLL_STATS_BKTS)
		return ddir + BLK_MQ_POLL_STATS_BKTS - 2;

	return bucket;
}

61 62 63
/*
 * Check if any of the ctx's have pending work in this hardware queue
 */
64
bool blk_mq_hctx_has_pending(struct blk_mq_hw_ctx *hctx)
65
{
66 67 68
	return sbitmap_any_bit_set(&hctx->ctx_map) ||
			!list_empty_careful(&hctx->dispatch) ||
			blk_mq_sched_has_work(hctx);
69 70
}

71 72 73 74 75 76
/*
 * Mark this ctx as having pending work in this hardware queue
 */
static void blk_mq_hctx_mark_pending(struct blk_mq_hw_ctx *hctx,
				     struct blk_mq_ctx *ctx)
{
77 78
	if (!sbitmap_test_bit(&hctx->ctx_map, ctx->index_hw))
		sbitmap_set_bit(&hctx->ctx_map, ctx->index_hw);
79 80 81 82 83
}

static void blk_mq_hctx_clear_pending(struct blk_mq_hw_ctx *hctx,
				      struct blk_mq_ctx *ctx)
{
84
	sbitmap_clear_bit(&hctx->ctx_map, ctx->index_hw);
85 86
}

87 88 89 90 91 92 93 94 95 96 97 98 99 100
struct mq_inflight {
	struct hd_struct *part;
	unsigned int *inflight;
};

static void blk_mq_check_inflight(struct blk_mq_hw_ctx *hctx,
				  struct request *rq, void *priv,
				  bool reserved)
{
	struct mq_inflight *mi = priv;

	if (test_bit(REQ_ATOM_STARTED, &rq->atomic_flags) &&
	    !test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) {
		/*
101 102 103 104
		 * index[0] counts the specific partition that was asked
		 * for. index[1] counts the ones that are active on the
		 * whole device, so increment that if mi->part is indeed
		 * a partition, and not a whole device.
105
		 */
106
		if (rq->part == mi->part)
107
			mi->inflight[0]++;
108 109
		if (mi->part->partno)
			mi->inflight[1]++;
110 111 112 113 114 115 116 117
	}
}

void blk_mq_in_flight(struct request_queue *q, struct hd_struct *part,
		      unsigned int inflight[2])
{
	struct mq_inflight mi = { .part = part, .inflight = inflight, };

118
	inflight[0] = inflight[1] = 0;
119 120 121
	blk_mq_queue_tag_busy_iter(q, blk_mq_check_inflight, &mi);
}

122
void blk_freeze_queue_start(struct request_queue *q)
123
{
124
	int freeze_depth;
125

126 127
	freeze_depth = atomic_inc_return(&q->mq_freeze_depth);
	if (freeze_depth == 1) {
128
		percpu_ref_kill(&q->q_usage_counter);
129 130
		if (q->mq_ops)
			blk_mq_run_hw_queues(q, false);
131
	}
132
}
133
EXPORT_SYMBOL_GPL(blk_freeze_queue_start);
134

135
void blk_mq_freeze_queue_wait(struct request_queue *q)
136
{
137
	wait_event(q->mq_freeze_wq, percpu_ref_is_zero(&q->q_usage_counter));
138
}
139
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait);
140

141 142 143 144 145 146 147 148
int blk_mq_freeze_queue_wait_timeout(struct request_queue *q,
				     unsigned long timeout)
{
	return wait_event_timeout(q->mq_freeze_wq,
					percpu_ref_is_zero(&q->q_usage_counter),
					timeout);
}
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue_wait_timeout);
149

150 151 152 153
/*
 * Guarantee no request is in use, so we can change any data structure of
 * the queue afterward.
 */
154
void blk_freeze_queue(struct request_queue *q)
155
{
156 157 158 159 160 161 162
	/*
	 * In the !blk_mq case we are only calling this to kill the
	 * q_usage_counter, otherwise this increases the freeze depth
	 * and waits for it to return to zero.  For this reason there is
	 * no blk_unfreeze_queue(), and blk_freeze_queue() is not
	 * exported to drivers as the only user for unfreeze is blk_mq.
	 */
163
	blk_freeze_queue_start(q);
164 165
	blk_mq_freeze_queue_wait(q);
}
166 167 168 169 170 171 172 173 174

void blk_mq_freeze_queue(struct request_queue *q)
{
	/*
	 * ...just an alias to keep freeze and unfreeze actions balanced
	 * in the blk_mq_* namespace
	 */
	blk_freeze_queue(q);
}
175
EXPORT_SYMBOL_GPL(blk_mq_freeze_queue);
176

177
void blk_mq_unfreeze_queue(struct request_queue *q)
178
{
179
	int freeze_depth;
180

181 182 183
	freeze_depth = atomic_dec_return(&q->mq_freeze_depth);
	WARN_ON_ONCE(freeze_depth < 0);
	if (!freeze_depth) {
184
		percpu_ref_reinit(&q->q_usage_counter);
185
		wake_up_all(&q->mq_freeze_wq);
186
	}
187
}
188
EXPORT_SYMBOL_GPL(blk_mq_unfreeze_queue);
189

190 191 192 193 194 195 196 197 198 199 200 201 202 203
/*
 * FIXME: replace the scsi_internal_device_*block_nowait() calls in the
 * mpt3sas driver such that this function can be removed.
 */
void blk_mq_quiesce_queue_nowait(struct request_queue *q)
{
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
	queue_flag_set(QUEUE_FLAG_QUIESCED, q);
	spin_unlock_irqrestore(q->queue_lock, flags);
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue_nowait);

204
/**
205
 * blk_mq_quiesce_queue() - wait until all ongoing dispatches have finished
206 207 208
 * @q: request queue.
 *
 * Note: this function does not prevent that the struct request end_io()
209 210 211
 * callback function is invoked. Once this function is returned, we make
 * sure no dispatch can happen until the queue is unquiesced via
 * blk_mq_unquiesce_queue().
212 213 214 215 216 217 218
 */
void blk_mq_quiesce_queue(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;
	bool rcu = false;

219
	blk_mq_quiesce_queue_nowait(q);
220

221 222
	queue_for_each_hw_ctx(q, hctx, i) {
		if (hctx->flags & BLK_MQ_F_BLOCKING)
223
			synchronize_srcu(hctx->queue_rq_srcu);
224 225 226 227 228 229 230 231
		else
			rcu = true;
	}
	if (rcu)
		synchronize_rcu();
}
EXPORT_SYMBOL_GPL(blk_mq_quiesce_queue);

232 233 234 235 236 237 238 239 240
/*
 * blk_mq_unquiesce_queue() - counterpart of blk_mq_quiesce_queue()
 * @q: request queue.
 *
 * This function recovers queue into the state before quiescing
 * which is done by blk_mq_quiesce_queue.
 */
void blk_mq_unquiesce_queue(struct request_queue *q)
{
241 242 243
	unsigned long flags;

	spin_lock_irqsave(q->queue_lock, flags);
244
	queue_flag_clear(QUEUE_FLAG_QUIESCED, q);
245
	spin_unlock_irqrestore(q->queue_lock, flags);
246

247 248
	/* dispatch requests which are inserted during quiescing */
	blk_mq_run_hw_queues(q, true);
249 250 251
}
EXPORT_SYMBOL_GPL(blk_mq_unquiesce_queue);

252 253 254 255 256 257 258 259 260 261
void blk_mq_wake_waiters(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hw_queue_mapped(hctx))
			blk_mq_tag_wakeup_all(hctx->tags, true);
}

262 263 264 265 266 267
bool blk_mq_can_queue(struct blk_mq_hw_ctx *hctx)
{
	return blk_mq_has_free_tags(hctx->tags);
}
EXPORT_SYMBOL(blk_mq_can_queue);

268 269
static struct request *blk_mq_rq_ctx_init(struct blk_mq_alloc_data *data,
		unsigned int tag, unsigned int op)
270
{
271 272 273
	struct blk_mq_tags *tags = blk_mq_tags_from_data(data);
	struct request *rq = tags->static_rqs[tag];

274 275
	rq->rq_flags = 0;

276 277 278 279 280 281 282 283 284 285 286 287 288
	if (data->flags & BLK_MQ_REQ_INTERNAL) {
		rq->tag = -1;
		rq->internal_tag = tag;
	} else {
		if (blk_mq_tag_busy(data->hctx)) {
			rq->rq_flags = RQF_MQ_INFLIGHT;
			atomic_inc(&data->hctx->nr_active);
		}
		rq->tag = tag;
		rq->internal_tag = -1;
		data->hctx->tags->rqs[rq->tag] = rq;
	}

289 290
	INIT_LIST_HEAD(&rq->queuelist);
	/* csd/requeue_work/fifo_time is initialized before use */
291 292
	rq->q = data->q;
	rq->mq_ctx = data->ctx;
293
	rq->cmd_flags = op;
294 295
	if (data->flags & BLK_MQ_REQ_PREEMPT)
		rq->rq_flags |= RQF_PREEMPT;
296
	if (blk_queue_io_stat(data->q))
297
		rq->rq_flags |= RQF_IO_STAT;
298 299 300 301 302 303
	/* do not touch atomic flags, it needs atomic ops against the timer */
	rq->cpu = -1;
	INIT_HLIST_NODE(&rq->hash);
	RB_CLEAR_NODE(&rq->rb_node);
	rq->rq_disk = NULL;
	rq->part = NULL;
304
	rq->start_time = jiffies;
305 306
#ifdef CONFIG_BLK_CGROUP
	rq->rl = NULL;
307
	set_start_time_ns(rq);
308 309 310 311 312 313 314 315 316 317 318
	rq->io_start_time_ns = 0;
#endif
	rq->nr_phys_segments = 0;
#if defined(CONFIG_BLK_DEV_INTEGRITY)
	rq->nr_integrity_segments = 0;
#endif
	rq->special = NULL;
	/* tag was already set */
	rq->extra_len = 0;

	INIT_LIST_HEAD(&rq->timeout_list);
319 320
	rq->timeout = 0;

321 322 323 324
	rq->end_io = NULL;
	rq->end_io_data = NULL;
	rq->next_rq = NULL;

325 326
	data->ctx->rq_dispatched[op_is_sync(op)]++;
	return rq;
327 328
}

329 330 331 332 333 334
static struct request *blk_mq_get_request(struct request_queue *q,
		struct bio *bio, unsigned int op,
		struct blk_mq_alloc_data *data)
{
	struct elevator_queue *e = q->elevator;
	struct request *rq;
335
	unsigned int tag;
336
	bool put_ctx_on_error = false;
337 338 339

	blk_queue_enter_live(q);
	data->q = q;
340 341 342 343
	if (likely(!data->ctx)) {
		data->ctx = blk_mq_get_ctx(q);
		put_ctx_on_error = true;
	}
344 345
	if (likely(!data->hctx))
		data->hctx = blk_mq_map_queue(q, data->ctx->cpu);
346 347
	if (op & REQ_NOWAIT)
		data->flags |= BLK_MQ_REQ_NOWAIT;
348 349 350 351 352 353 354 355

	if (e) {
		data->flags |= BLK_MQ_REQ_INTERNAL;

		/*
		 * Flush requests are special and go directly to the
		 * dispatch list.
		 */
356 357
		if (!op_is_flush(op) && e->type->ops.mq.limit_depth)
			e->type->ops.mq.limit_depth(op, data);
358 359
	}

360 361
	tag = blk_mq_get_tag(data);
	if (tag == BLK_MQ_TAG_FAIL) {
362 363
		if (put_ctx_on_error) {
			blk_mq_put_ctx(data->ctx);
364 365
			data->ctx = NULL;
		}
366 367
		blk_queue_exit(q);
		return NULL;
368 369
	}

370
	rq = blk_mq_rq_ctx_init(data, tag, op);
371 372
	if (!op_is_flush(op)) {
		rq->elv.icq = NULL;
373
		if (e && e->type->ops.mq.prepare_request) {
374 375 376
			if (e->type->icq_cache && rq_ioc(bio))
				blk_mq_sched_assign_ioc(rq, bio);

377 378
			e->type->ops.mq.prepare_request(rq, bio);
			rq->rq_flags |= RQF_ELVPRIV;
379
		}
380 381 382
	}
	data->hctx->queued++;
	return rq;
383 384
}

385
struct request *blk_mq_alloc_request(struct request_queue *q, unsigned int op,
386
		unsigned int flags)
387
{
388
	struct blk_mq_alloc_data alloc_data = { .flags = flags };
389
	struct request *rq;
390
	int ret;
391

392
	ret = blk_queue_enter(q, flags & BLK_MQ_REQ_NOWAIT);
393 394
	if (ret)
		return ERR_PTR(ret);
395

396
	rq = blk_mq_get_request(q, NULL, op, &alloc_data);
397
	blk_queue_exit(q);
398

399
	if (!rq)
400
		return ERR_PTR(-EWOULDBLOCK);
401

402 403
	blk_mq_put_ctx(alloc_data.ctx);

404 405 406
	rq->__data_len = 0;
	rq->__sector = (sector_t) -1;
	rq->bio = rq->biotail = NULL;
407 408
	return rq;
}
409
EXPORT_SYMBOL(blk_mq_alloc_request);
410

411 412
struct request *blk_mq_alloc_request_hctx(struct request_queue *q,
		unsigned int op, unsigned int flags, unsigned int hctx_idx)
M
Ming Lin 已提交
413
{
414
	struct blk_mq_alloc_data alloc_data = { .flags = flags };
M
Ming Lin 已提交
415
	struct request *rq;
416
	unsigned int cpu;
M
Ming Lin 已提交
417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434
	int ret;

	/*
	 * If the tag allocator sleeps we could get an allocation for a
	 * different hardware context.  No need to complicate the low level
	 * allocator for this for the rare use case of a command tied to
	 * a specific queue.
	 */
	if (WARN_ON_ONCE(!(flags & BLK_MQ_REQ_NOWAIT)))
		return ERR_PTR(-EINVAL);

	if (hctx_idx >= q->nr_hw_queues)
		return ERR_PTR(-EIO);

	ret = blk_queue_enter(q, true);
	if (ret)
		return ERR_PTR(ret);

435 436 437 438
	/*
	 * Check if the hardware context is actually mapped to anything.
	 * If not tell the caller that it should skip this queue.
	 */
439 440 441 442
	alloc_data.hctx = q->queue_hw_ctx[hctx_idx];
	if (!blk_mq_hw_queue_mapped(alloc_data.hctx)) {
		blk_queue_exit(q);
		return ERR_PTR(-EXDEV);
443
	}
444 445
	cpu = cpumask_first(alloc_data.hctx->cpumask);
	alloc_data.ctx = __blk_mq_get_ctx(q, cpu);
M
Ming Lin 已提交
446

447
	rq = blk_mq_get_request(q, NULL, op, &alloc_data);
448
	blk_queue_exit(q);
449

450 451 452 453
	if (!rq)
		return ERR_PTR(-EWOULDBLOCK);

	return rq;
M
Ming Lin 已提交
454 455 456
}
EXPORT_SYMBOL_GPL(blk_mq_alloc_request_hctx);

457
void blk_mq_free_request(struct request *rq)
458 459
{
	struct request_queue *q = rq->q;
460 461 462 463 464
	struct elevator_queue *e = q->elevator;
	struct blk_mq_ctx *ctx = rq->mq_ctx;
	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(q, ctx->cpu);
	const int sched_tag = rq->internal_tag;

465
	if (rq->rq_flags & RQF_ELVPRIV) {
466 467 468 469 470 471 472
		if (e && e->type->ops.mq.finish_request)
			e->type->ops.mq.finish_request(rq);
		if (rq->elv.icq) {
			put_io_context(rq->elv.icq->ioc);
			rq->elv.icq = NULL;
		}
	}
473

474
	ctx->rq_completed[rq_is_sync(rq)]++;
475
	if (rq->rq_flags & RQF_MQ_INFLIGHT)
476
		atomic_dec(&hctx->nr_active);
J
Jens Axboe 已提交
477

478 479 480
	if (unlikely(laptop_mode && !blk_rq_is_passthrough(rq)))
		laptop_io_completion(q->backing_dev_info);

J
Jens Axboe 已提交
481
	wbt_done(q->rq_wb, &rq->issue_stat);
482

S
Shaohua Li 已提交
483 484 485
	if (blk_rq_rl(rq))
		blk_put_rl(blk_rq_rl(rq));

486
	clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
487
	clear_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);
488 489 490
	if (rq->tag != -1)
		blk_mq_put_tag(hctx, hctx->tags, ctx, rq->tag);
	if (sched_tag != -1)
491
		blk_mq_put_tag(hctx, hctx->sched_tags, ctx, sched_tag);
492
	blk_mq_sched_restart(hctx);
493
	blk_queue_exit(q);
494
}
J
Jens Axboe 已提交
495
EXPORT_SYMBOL_GPL(blk_mq_free_request);
496

497
inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
498
{
M
Ming Lei 已提交
499 500
	blk_account_io_done(rq);

C
Christoph Hellwig 已提交
501
	if (rq->end_io) {
J
Jens Axboe 已提交
502
		wbt_done(rq->q->rq_wb, &rq->issue_stat);
503
		rq->end_io(rq, error);
C
Christoph Hellwig 已提交
504 505 506
	} else {
		if (unlikely(blk_bidi_rq(rq)))
			blk_mq_free_request(rq->next_rq);
507
		blk_mq_free_request(rq);
C
Christoph Hellwig 已提交
508
	}
509
}
510
EXPORT_SYMBOL(__blk_mq_end_request);
511

512
void blk_mq_end_request(struct request *rq, blk_status_t error)
513 514 515
{
	if (blk_update_request(rq, error, blk_rq_bytes(rq)))
		BUG();
516
	__blk_mq_end_request(rq, error);
517
}
518
EXPORT_SYMBOL(blk_mq_end_request);
519

520
static void __blk_mq_complete_request_remote(void *data)
521
{
522
	struct request *rq = data;
523

524
	rq->q->softirq_done_fn(rq);
525 526
}

527
static void __blk_mq_complete_request(struct request *rq)
528 529
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;
C
Christoph Hellwig 已提交
530
	bool shared = false;
531 532
	int cpu;

533 534 535 536 537 538 539
	if (rq->internal_tag != -1)
		blk_mq_sched_completed_request(rq);
	if (rq->rq_flags & RQF_STATS) {
		blk_mq_poll_stats_start(rq->q);
		blk_stat_add(rq);
	}

C
Christoph Hellwig 已提交
540
	if (!test_bit(QUEUE_FLAG_SAME_COMP, &rq->q->queue_flags)) {
541 542 543
		rq->q->softirq_done_fn(rq);
		return;
	}
544 545

	cpu = get_cpu();
C
Christoph Hellwig 已提交
546 547 548 549
	if (!test_bit(QUEUE_FLAG_SAME_FORCE, &rq->q->queue_flags))
		shared = cpus_share_cache(cpu, ctx->cpu);

	if (cpu != ctx->cpu && !shared && cpu_online(ctx->cpu)) {
550
		rq->csd.func = __blk_mq_complete_request_remote;
551 552
		rq->csd.info = rq;
		rq->csd.flags = 0;
553
		smp_call_function_single_async(ctx->cpu, &rq->csd);
554
	} else {
555
		rq->q->softirq_done_fn(rq);
556
	}
557 558
	put_cpu();
}
559 560 561 562 563 564 565 566 567

/**
 * blk_mq_complete_request - end I/O on a request
 * @rq:		the request being processed
 *
 * Description:
 *	Ends all I/O on a request. It does not handle partial completions.
 *	The actual completion happens out-of-order, through a IPI handler.
 **/
568
void blk_mq_complete_request(struct request *rq)
569
{
570 571 572
	struct request_queue *q = rq->q;

	if (unlikely(blk_should_fake_timeout(q)))
573
		return;
574
	if (!blk_mark_rq_complete(rq))
575
		__blk_mq_complete_request(rq);
576 577
}
EXPORT_SYMBOL(blk_mq_complete_request);
578

579 580 581 582 583 584
int blk_mq_request_started(struct request *rq)
{
	return test_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
}
EXPORT_SYMBOL_GPL(blk_mq_request_started);

585
void blk_mq_start_request(struct request *rq)
586 587 588
{
	struct request_queue *q = rq->q;

589 590
	blk_mq_sched_started_request(rq);

591 592
	trace_block_rq_issue(q, rq);

593
	if (test_bit(QUEUE_FLAG_STATS, &q->queue_flags)) {
594
		blk_stat_set_issue(&rq->issue_stat, blk_rq_sectors(rq));
595
		rq->rq_flags |= RQF_STATS;
J
Jens Axboe 已提交
596
		wbt_issue(q->rq_wb, &rq->issue_stat);
597 598
	}

599
	blk_add_timer(rq);
600

601
	WARN_ON_ONCE(test_bit(REQ_ATOM_STARTED, &rq->atomic_flags));
602

603 604 605 606 607
	/*
	 * Mark us as started and clear complete. Complete might have been
	 * set if requeue raced with timeout, which then marked it as
	 * complete. So be sure to clear complete again when we start
	 * the request, otherwise we'll ignore the completion event.
608 609 610 611
	 *
	 * Ensure that ->deadline is visible before we set STARTED, such that
	 * blk_mq_check_expired() is guaranteed to observe our ->deadline when
	 * it observes STARTED.
612
	 */
613 614 615 616 617 618 619 620 621 622 623 624
	smp_wmb();
	set_bit(REQ_ATOM_STARTED, &rq->atomic_flags);
	if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags)) {
		/*
		 * Coherence order guarantees these consecutive stores to a
		 * single variable propagate in the specified order. Thus the
		 * clear_bit() is ordered _after_ the set bit. See
		 * blk_mq_check_expired().
		 *
		 * (the bits must be part of the same byte for this to be
		 * true).
		 */
625
		clear_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags);
626
	}
627 628 629 630 631 632 633 634 635

	if (q->dma_drain_size && blk_rq_bytes(rq)) {
		/*
		 * Make sure space for the drain appears.  We know we can do
		 * this because max_hw_segments has been adjusted to be one
		 * fewer than the device can handle.
		 */
		rq->nr_phys_segments++;
	}
636
}
637
EXPORT_SYMBOL(blk_mq_start_request);
638

639 640
/*
 * When we reach here because queue is busy, REQ_ATOM_COMPLETE
641
 * flag isn't set yet, so there may be race with timeout handler,
642 643 644 645 646 647
 * but given rq->deadline is just set in .queue_rq() under
 * this situation, the race won't be possible in reality because
 * rq->timeout should be set as big enough to cover the window
 * between blk_mq_start_request() called from .queue_rq() and
 * clearing REQ_ATOM_STARTED here.
 */
648
static void __blk_mq_requeue_request(struct request *rq)
649 650 651
{
	struct request_queue *q = rq->q;

652 653
	blk_mq_put_driver_tag(rq);

654
	trace_block_rq_requeue(q, rq);
J
Jens Axboe 已提交
655
	wbt_requeue(q->rq_wb, &rq->issue_stat);
656
	blk_mq_sched_requeue_request(rq);
657

658 659 660 661
	if (test_and_clear_bit(REQ_ATOM_STARTED, &rq->atomic_flags)) {
		if (q->dma_drain_size && blk_rq_bytes(rq))
			rq->nr_phys_segments--;
	}
662 663
}

664
void blk_mq_requeue_request(struct request *rq, bool kick_requeue_list)
665 666 667 668
{
	__blk_mq_requeue_request(rq);

	BUG_ON(blk_queued_rq(rq));
669
	blk_mq_add_to_requeue_list(rq, true, kick_requeue_list);
670 671 672
}
EXPORT_SYMBOL(blk_mq_requeue_request);

673 674 675
static void blk_mq_requeue_work(struct work_struct *work)
{
	struct request_queue *q =
676
		container_of(work, struct request_queue, requeue_work.work);
677 678 679
	LIST_HEAD(rq_list);
	struct request *rq, *next;

680
	spin_lock_irq(&q->requeue_lock);
681
	list_splice_init(&q->requeue_list, &rq_list);
682
	spin_unlock_irq(&q->requeue_lock);
683 684

	list_for_each_entry_safe(rq, next, &rq_list, queuelist) {
685
		if (!(rq->rq_flags & RQF_SOFTBARRIER))
686 687
			continue;

688
		rq->rq_flags &= ~RQF_SOFTBARRIER;
689
		list_del_init(&rq->queuelist);
690
		blk_mq_sched_insert_request(rq, true, false, false, true);
691 692 693 694 695
	}

	while (!list_empty(&rq_list)) {
		rq = list_entry(rq_list.next, struct request, queuelist);
		list_del_init(&rq->queuelist);
696
		blk_mq_sched_insert_request(rq, false, false, false, true);
697 698
	}

699
	blk_mq_run_hw_queues(q, false);
700 701
}

702 703
void blk_mq_add_to_requeue_list(struct request *rq, bool at_head,
				bool kick_requeue_list)
704 705 706 707 708 709 710 711
{
	struct request_queue *q = rq->q;
	unsigned long flags;

	/*
	 * We abuse this flag that is otherwise used by the I/O scheduler to
	 * request head insertation from the workqueue.
	 */
712
	BUG_ON(rq->rq_flags & RQF_SOFTBARRIER);
713 714 715

	spin_lock_irqsave(&q->requeue_lock, flags);
	if (at_head) {
716
		rq->rq_flags |= RQF_SOFTBARRIER;
717 718 719 720 721
		list_add(&rq->queuelist, &q->requeue_list);
	} else {
		list_add_tail(&rq->queuelist, &q->requeue_list);
	}
	spin_unlock_irqrestore(&q->requeue_lock, flags);
722 723 724

	if (kick_requeue_list)
		blk_mq_kick_requeue_list(q);
725 726 727 728 729
}
EXPORT_SYMBOL(blk_mq_add_to_requeue_list);

void blk_mq_kick_requeue_list(struct request_queue *q)
{
730
	kblockd_schedule_delayed_work(&q->requeue_work, 0);
731 732 733
}
EXPORT_SYMBOL(blk_mq_kick_requeue_list);

734 735 736
void blk_mq_delay_kick_requeue_list(struct request_queue *q,
				    unsigned long msecs)
{
737 738
	kblockd_mod_delayed_work_on(WORK_CPU_UNBOUND, &q->requeue_work,
				    msecs_to_jiffies(msecs));
739 740 741
}
EXPORT_SYMBOL(blk_mq_delay_kick_requeue_list);

742 743
struct request *blk_mq_tag_to_rq(struct blk_mq_tags *tags, unsigned int tag)
{
744 745
	if (tag < tags->nr_tags) {
		prefetch(tags->rqs[tag]);
746
		return tags->rqs[tag];
747
	}
748 749

	return NULL;
750 751 752
}
EXPORT_SYMBOL(blk_mq_tag_to_rq);

753
struct blk_mq_timeout_data {
754 755
	unsigned long next;
	unsigned int next_set;
756 757
};

758
void blk_mq_rq_timed_out(struct request *req, bool reserved)
759
{
J
Jens Axboe 已提交
760
	const struct blk_mq_ops *ops = req->q->mq_ops;
761
	enum blk_eh_timer_return ret = BLK_EH_RESET_TIMER;
762 763 764 765 766 767 768

	/*
	 * We know that complete is set at this point. If STARTED isn't set
	 * anymore, then the request isn't active and the "timeout" should
	 * just be ignored. This can happen due to the bitflag ordering.
	 * Timeout first checks if STARTED is set, and if it is, assumes
	 * the request is active. But if we race with completion, then
769
	 * both flags will get cleared. So check here again, and ignore
770 771
	 * a timeout event with a request that isn't active.
	 */
772 773
	if (!test_bit(REQ_ATOM_STARTED, &req->atomic_flags))
		return;
774

775
	if (ops->timeout)
776
		ret = ops->timeout(req, reserved);
777 778 779 780 781 782 783 784 785 786 787 788 789 790 791

	switch (ret) {
	case BLK_EH_HANDLED:
		__blk_mq_complete_request(req);
		break;
	case BLK_EH_RESET_TIMER:
		blk_add_timer(req);
		blk_clear_rq_complete(req);
		break;
	case BLK_EH_NOT_HANDLED:
		break;
	default:
		printk(KERN_ERR "block: bad eh return: %d\n", ret);
		break;
	}
792
}
793

794 795 796 797
static void blk_mq_check_expired(struct blk_mq_hw_ctx *hctx,
		struct request *rq, void *priv, bool reserved)
{
	struct blk_mq_timeout_data *data = priv;
798
	unsigned long deadline;
799

800
	if (!test_bit(REQ_ATOM_STARTED, &rq->atomic_flags))
801
		return;
802

803 804 805 806 807 808 809 810
	/*
	 * Ensures that if we see STARTED we must also see our
	 * up-to-date deadline, see blk_mq_start_request().
	 */
	smp_rmb();

	deadline = READ_ONCE(rq->deadline);

811 812 813 814 815 816 817 818 819 820 821 822 823
	/*
	 * The rq being checked may have been freed and reallocated
	 * out already here, we avoid this race by checking rq->deadline
	 * and REQ_ATOM_COMPLETE flag together:
	 *
	 * - if rq->deadline is observed as new value because of
	 *   reusing, the rq won't be timed out because of timing.
	 * - if rq->deadline is observed as previous value,
	 *   REQ_ATOM_COMPLETE flag won't be cleared in reuse path
	 *   because we put a barrier between setting rq->deadline
	 *   and clearing the flag in blk_mq_start_request(), so
	 *   this rq won't be timed out too.
	 */
824 825 826 827 828 829 830 831 832 833
	if (time_after_eq(jiffies, deadline)) {
		if (!blk_mark_rq_complete(rq)) {
			/*
			 * Again coherence order ensures that consecutive reads
			 * from the same variable must be in that order. This
			 * ensures that if we see COMPLETE clear, we must then
			 * see STARTED set and we'll ignore this timeout.
			 *
			 * (There's also the MB implied by the test_and_clear())
			 */
834
			blk_mq_rq_timed_out(rq, reserved);
835 836 837
		}
	} else if (!data->next_set || time_after(data->next, deadline)) {
		data->next = deadline;
838 839
		data->next_set = 1;
	}
840 841
}

842
static void blk_mq_timeout_work(struct work_struct *work)
843
{
844 845
	struct request_queue *q =
		container_of(work, struct request_queue, timeout_work);
846 847 848 849 850
	struct blk_mq_timeout_data data = {
		.next		= 0,
		.next_set	= 0,
	};
	int i;
851

852 853 854 855 856 857 858 859 860
	/* A deadlock might occur if a request is stuck requiring a
	 * timeout at the same time a queue freeze is waiting
	 * completion, since the timeout code would not be able to
	 * acquire the queue reference here.
	 *
	 * That's why we don't use blk_queue_enter here; instead, we use
	 * percpu_ref_tryget directly, because we need to be able to
	 * obtain a reference even in the short window between the queue
	 * starting to freeze, by dropping the first reference in
861
	 * blk_freeze_queue_start, and the moment the last request is
862 863 864 865
	 * consumed, marked by the instant q_usage_counter reaches
	 * zero.
	 */
	if (!percpu_ref_tryget(&q->q_usage_counter))
866 867
		return;

868
	blk_mq_queue_tag_busy_iter(q, blk_mq_check_expired, &data);
869

870 871 872
	if (data.next_set) {
		data.next = blk_rq_timeout(round_jiffies_up(data.next));
		mod_timer(&q->timeout, data.next);
873
	} else {
874 875
		struct blk_mq_hw_ctx *hctx;

876 877 878 879 880
		queue_for_each_hw_ctx(q, hctx, i) {
			/* the hctx may be unmapped, so check it here */
			if (blk_mq_hw_queue_mapped(hctx))
				blk_mq_tag_idle(hctx);
		}
881
	}
882
	blk_queue_exit(q);
883 884
}

885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902
struct flush_busy_ctx_data {
	struct blk_mq_hw_ctx *hctx;
	struct list_head *list;
};

static bool flush_busy_ctx(struct sbitmap *sb, unsigned int bitnr, void *data)
{
	struct flush_busy_ctx_data *flush_data = data;
	struct blk_mq_hw_ctx *hctx = flush_data->hctx;
	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];

	sbitmap_clear_bit(sb, bitnr);
	spin_lock(&ctx->lock);
	list_splice_tail_init(&ctx->rq_list, flush_data->list);
	spin_unlock(&ctx->lock);
	return true;
}

903 904 905 906
/*
 * Process software queues that have been marked busy, splicing them
 * to the for-dispatch
 */
907
void blk_mq_flush_busy_ctxs(struct blk_mq_hw_ctx *hctx, struct list_head *list)
908
{
909 910 911 912
	struct flush_busy_ctx_data data = {
		.hctx = hctx,
		.list = list,
	};
913

914
	sbitmap_for_each_set(&hctx->ctx_map, flush_busy_ctx, &data);
915
}
916
EXPORT_SYMBOL_GPL(blk_mq_flush_busy_ctxs);
917

918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956
struct dispatch_rq_data {
	struct blk_mq_hw_ctx *hctx;
	struct request *rq;
};

static bool dispatch_rq_from_ctx(struct sbitmap *sb, unsigned int bitnr,
		void *data)
{
	struct dispatch_rq_data *dispatch_data = data;
	struct blk_mq_hw_ctx *hctx = dispatch_data->hctx;
	struct blk_mq_ctx *ctx = hctx->ctxs[bitnr];

	spin_lock(&ctx->lock);
	if (unlikely(!list_empty(&ctx->rq_list))) {
		dispatch_data->rq = list_entry_rq(ctx->rq_list.next);
		list_del_init(&dispatch_data->rq->queuelist);
		if (list_empty(&ctx->rq_list))
			sbitmap_clear_bit(sb, bitnr);
	}
	spin_unlock(&ctx->lock);

	return !dispatch_data->rq;
}

struct request *blk_mq_dequeue_from_ctx(struct blk_mq_hw_ctx *hctx,
					struct blk_mq_ctx *start)
{
	unsigned off = start ? start->index_hw : 0;
	struct dispatch_rq_data data = {
		.hctx = hctx,
		.rq   = NULL,
	};

	__sbitmap_for_each_set(&hctx->ctx_map, off,
			       dispatch_rq_from_ctx, &data);

	return data.rq;
}

957 958 959 960
static inline unsigned int queued_to_index(unsigned int queued)
{
	if (!queued)
		return 0;
961

962
	return min(BLK_MQ_MAX_DISPATCH_ORDER - 1, ilog2(queued) + 1);
963 964
}

965 966
bool blk_mq_get_driver_tag(struct request *rq, struct blk_mq_hw_ctx **hctx,
			   bool wait)
967 968 969 970 971 972 973
{
	struct blk_mq_alloc_data data = {
		.q = rq->q,
		.hctx = blk_mq_map_queue(rq->q, rq->mq_ctx->cpu),
		.flags = wait ? 0 : BLK_MQ_REQ_NOWAIT,
	};

974 975
	might_sleep_if(wait);

976 977
	if (rq->tag != -1)
		goto done;
978

979 980 981
	if (blk_mq_tag_is_reserved(data.hctx->sched_tags, rq->internal_tag))
		data.flags |= BLK_MQ_REQ_RESERVED;

982 983
	rq->tag = blk_mq_get_tag(&data);
	if (rq->tag >= 0) {
984 985 986 987
		if (blk_mq_tag_busy(data.hctx)) {
			rq->rq_flags |= RQF_MQ_INFLIGHT;
			atomic_inc(&data.hctx->nr_active);
		}
988 989 990
		data.hctx->tags->rqs[rq->tag] = rq;
	}

991 992 993 994
done:
	if (hctx)
		*hctx = data.hctx;
	return rq->tag != -1;
995 996
}

997 998
static int blk_mq_dispatch_wake(wait_queue_entry_t *wait, unsigned mode,
				int flags, void *key)
999 1000 1001 1002 1003
{
	struct blk_mq_hw_ctx *hctx;

	hctx = container_of(wait, struct blk_mq_hw_ctx, dispatch_wait);

1004
	list_del_init(&wait->entry);
1005 1006 1007 1008
	blk_mq_run_hw_queue(hctx, true);
	return 1;
}

1009 1010
static bool blk_mq_dispatch_wait_add(struct blk_mq_hw_ctx **hctx,
				     struct request *rq)
1011
{
1012 1013
	struct blk_mq_hw_ctx *this_hctx = *hctx;
	wait_queue_entry_t *wait = &this_hctx->dispatch_wait;
1014 1015
	struct sbq_wait_state *ws;

1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027
	if (!list_empty_careful(&wait->entry))
		return false;

	spin_lock(&this_hctx->lock);
	if (!list_empty(&wait->entry)) {
		spin_unlock(&this_hctx->lock);
		return false;
	}

	ws = bt_wait_ptr(&this_hctx->tags->bitmap_tags, this_hctx);
	add_wait_queue(&ws->wait, wait);

1028
	/*
1029 1030 1031
	 * It's possible that a tag was freed in the window between the
	 * allocation failure and adding the hardware queue to the wait
	 * queue.
1032
	 */
1033 1034
	if (!blk_mq_get_driver_tag(rq, hctx, false)) {
		spin_unlock(&this_hctx->lock);
1035
		return false;
1036
	}
1037 1038

	/*
1039 1040
	 * We got a tag, remove ourselves from the wait queue to ensure
	 * someone else gets the wakeup.
1041
	 */
1042 1043 1044 1045
	spin_lock_irq(&ws->wait.lock);
	list_del_init(&wait->entry);
	spin_unlock_irq(&ws->wait.lock);
	spin_unlock(&this_hctx->lock);
1046 1047 1048
	return true;
}

1049
bool blk_mq_dispatch_rq_list(struct request_queue *q, struct list_head *list,
1050
			     bool got_budget)
1051
{
1052
	struct blk_mq_hw_ctx *hctx;
1053
	struct request *rq, *nxt;
1054
	bool no_tag = false;
1055
	int errors, queued;
1056

1057 1058 1059
	if (list_empty(list))
		return false;

1060 1061
	WARN_ON(!list_is_singular(list) && got_budget);

1062 1063 1064
	/*
	 * Now process all the entries, sending them to the driver.
	 */
1065
	errors = queued = 0;
1066
	do {
1067
		struct blk_mq_queue_data bd;
1068
		blk_status_t ret;
1069

1070
		rq = list_first_entry(list, struct request, queuelist);
1071
		if (!blk_mq_get_driver_tag(rq, &hctx, false)) {
1072
			/*
1073
			 * The initial allocation attempt failed, so we need to
1074 1075 1076 1077
			 * rerun the hardware queue when a tag is freed. The
			 * waitqueue takes care of that. If the queue is run
			 * before we add this entry back on the dispatch list,
			 * we'll re-run it below.
1078
			 */
1079
			if (!blk_mq_dispatch_wait_add(&hctx, rq)) {
1080 1081
				if (got_budget)
					blk_mq_put_dispatch_budget(hctx);
1082
				no_tag = true;
1083 1084 1085 1086
				break;
			}
		}

1087 1088
		if (!got_budget && !blk_mq_get_dispatch_budget(hctx)) {
			blk_mq_put_driver_tag(rq);
1089
			break;
1090
		}
1091

1092 1093
		list_del_init(&rq->queuelist);

1094
		bd.rq = rq;
1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105

		/*
		 * Flag last if we have no more requests, or if we have more
		 * but can't assign a driver tag to it.
		 */
		if (list_empty(list))
			bd.last = true;
		else {
			nxt = list_first_entry(list, struct request, queuelist);
			bd.last = !blk_mq_get_driver_tag(nxt, NULL, false);
		}
1106 1107

		ret = q->mq_ops->queue_rq(hctx, &bd);
1108
		if (ret == BLK_STS_RESOURCE) {
1109 1110 1111 1112 1113 1114 1115 1116
			/*
			 * If an I/O scheduler has been configured and we got a
			 * driver tag for the next request already, free it again.
			 */
			if (!list_empty(list)) {
				nxt = list_first_entry(list, struct request, queuelist);
				blk_mq_put_driver_tag(nxt);
			}
1117
			list_add(&rq->queuelist, list);
1118
			__blk_mq_requeue_request(rq);
1119
			break;
1120 1121 1122
		}

		if (unlikely(ret != BLK_STS_OK)) {
1123
			errors++;
1124
			blk_mq_end_request(rq, BLK_STS_IOERR);
1125
			continue;
1126 1127
		}

1128
		queued++;
1129
	} while (!list_empty(list));
1130

1131
	hctx->dispatched[queued_to_index(queued)]++;
1132 1133 1134 1135 1136

	/*
	 * Any items that need requeuing? Stuff them into hctx->dispatch,
	 * that is where we will continue on next queue run.
	 */
1137
	if (!list_empty(list)) {
1138
		spin_lock(&hctx->lock);
1139
		list_splice_init(list, &hctx->dispatch);
1140
		spin_unlock(&hctx->lock);
1141

1142
		/*
1143 1144 1145
		 * If SCHED_RESTART was set by the caller of this function and
		 * it is no longer set that means that it was cleared by another
		 * thread and hence that a queue rerun is needed.
1146
		 *
1147 1148 1149 1150
		 * If 'no_tag' is set, that means that we failed getting
		 * a driver tag with an I/O scheduler attached. If our dispatch
		 * waitqueue is no longer active, ensure that we run the queue
		 * AFTER adding our entries back to the list.
1151
		 *
1152 1153 1154 1155 1156 1157 1158
		 * If no I/O scheduler has been configured it is possible that
		 * the hardware queue got stopped and restarted before requests
		 * were pushed back onto the dispatch list. Rerun the queue to
		 * avoid starvation. Notes:
		 * - blk_mq_run_hw_queue() checks whether or not a queue has
		 *   been stopped before rerunning a queue.
		 * - Some but not all block drivers stop a queue before
1159
		 *   returning BLK_STS_RESOURCE. Two exceptions are scsi-mq
1160
		 *   and dm-rq.
1161
		 */
1162 1163
		if (!blk_mq_sched_needs_restart(hctx) ||
		    (no_tag && list_empty_careful(&hctx->dispatch_wait.entry)))
1164
			blk_mq_run_hw_queue(hctx, true);
1165
	}
1166

1167
	return (queued + errors) != 0;
1168 1169
}

1170 1171 1172 1173
static void __blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	int srcu_idx;

1174 1175 1176 1177
	/*
	 * We should be running this queue from one of the CPUs that
	 * are mapped to it.
	 */
1178 1179 1180
	WARN_ON(!cpumask_test_cpu(raw_smp_processor_id(), hctx->cpumask) &&
		cpu_online(hctx->next_cpu));

1181 1182 1183 1184 1185 1186
	/*
	 * We can't run the queue inline with ints disabled. Ensure that
	 * we catch bad users of this early.
	 */
	WARN_ON_ONCE(in_interrupt());

1187 1188
	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
		rcu_read_lock();
1189
		blk_mq_sched_dispatch_requests(hctx);
1190 1191
		rcu_read_unlock();
	} else {
1192 1193
		might_sleep();

1194
		srcu_idx = srcu_read_lock(hctx->queue_rq_srcu);
1195
		blk_mq_sched_dispatch_requests(hctx);
1196
		srcu_read_unlock(hctx->queue_rq_srcu, srcu_idx);
1197 1198 1199
	}
}

1200 1201 1202 1203 1204 1205 1206 1207
/*
 * It'd be great if the workqueue API had a way to pass
 * in a mask and had some smarts for more clever placement.
 * For now we just round-robin here, switching for every
 * BLK_MQ_CPU_WORK_BATCH queued items.
 */
static int blk_mq_hctx_next_cpu(struct blk_mq_hw_ctx *hctx)
{
1208 1209
	if (hctx->queue->nr_hw_queues == 1)
		return WORK_CPU_UNBOUND;
1210 1211

	if (--hctx->next_cpu_batch <= 0) {
1212
		int next_cpu;
1213 1214 1215 1216 1217 1218 1219 1220 1221

		next_cpu = cpumask_next(hctx->next_cpu, hctx->cpumask);
		if (next_cpu >= nr_cpu_ids)
			next_cpu = cpumask_first(hctx->cpumask);

		hctx->next_cpu = next_cpu;
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}

1222
	return hctx->next_cpu;
1223 1224
}

1225 1226
static void __blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async,
					unsigned long msecs)
1227
{
1228 1229 1230 1231
	if (WARN_ON_ONCE(!blk_mq_hw_queue_mapped(hctx)))
		return;

	if (unlikely(blk_mq_hctx_stopped(hctx)))
1232 1233
		return;

1234
	if (!async && !(hctx->flags & BLK_MQ_F_BLOCKING)) {
1235 1236
		int cpu = get_cpu();
		if (cpumask_test_cpu(cpu, hctx->cpumask)) {
1237
			__blk_mq_run_hw_queue(hctx);
1238
			put_cpu();
1239 1240
			return;
		}
1241

1242
		put_cpu();
1243
	}
1244

1245 1246 1247
	kblockd_schedule_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
					 &hctx->run_work,
					 msecs_to_jiffies(msecs));
1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258
}

void blk_mq_delay_run_hw_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
	__blk_mq_delay_run_hw_queue(hctx, true, msecs);
}
EXPORT_SYMBOL(blk_mq_delay_run_hw_queue);

void blk_mq_run_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
	__blk_mq_delay_run_hw_queue(hctx, async, 0);
1259
}
O
Omar Sandoval 已提交
1260
EXPORT_SYMBOL(blk_mq_run_hw_queue);
1261

1262
void blk_mq_run_hw_queues(struct request_queue *q, bool async)
1263 1264 1265 1266 1267
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i) {
1268
		if (!blk_mq_hctx_has_pending(hctx) ||
1269
		    blk_mq_hctx_stopped(hctx))
1270 1271
			continue;

1272
		blk_mq_run_hw_queue(hctx, async);
1273 1274
	}
}
1275
EXPORT_SYMBOL(blk_mq_run_hw_queues);
1276

1277 1278 1279 1280 1281 1282 1283 1284 1285 1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296
/**
 * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped
 * @q: request queue.
 *
 * The caller is responsible for serializing this function against
 * blk_mq_{start,stop}_hw_queue().
 */
bool blk_mq_queue_stopped(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		if (blk_mq_hctx_stopped(hctx))
			return true;

	return false;
}
EXPORT_SYMBOL(blk_mq_queue_stopped);

1297 1298 1299
/*
 * This function is often used for pausing .queue_rq() by driver when
 * there isn't enough resource or some conditions aren't satisfied, and
1300
 * BLK_STS_RESOURCE is usually returned.
1301 1302 1303 1304 1305
 *
 * We do not guarantee that dispatch can be drained or blocked
 * after blk_mq_stop_hw_queue() returns. Please use
 * blk_mq_quiesce_queue() for that requirement.
 */
1306 1307
void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx)
{
1308
	cancel_delayed_work(&hctx->run_work);
1309

1310
	set_bit(BLK_MQ_S_STOPPED, &hctx->state);
1311
}
1312
EXPORT_SYMBOL(blk_mq_stop_hw_queue);
1313

1314 1315 1316
/*
 * This function is often used for pausing .queue_rq() by driver when
 * there isn't enough resource or some conditions aren't satisfied, and
1317
 * BLK_STS_RESOURCE is usually returned.
1318 1319 1320 1321 1322
 *
 * We do not guarantee that dispatch can be drained or blocked
 * after blk_mq_stop_hw_queues() returns. Please use
 * blk_mq_quiesce_queue() for that requirement.
 */
1323 1324
void blk_mq_stop_hw_queues(struct request_queue *q)
{
1325 1326 1327 1328 1329
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_stop_hw_queue(hctx);
1330 1331 1332
}
EXPORT_SYMBOL(blk_mq_stop_hw_queues);

1333 1334 1335
void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx)
{
	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
1336

1337
	blk_mq_run_hw_queue(hctx, false);
1338 1339 1340
}
EXPORT_SYMBOL(blk_mq_start_hw_queue);

1341 1342 1343 1344 1345 1346 1347 1348 1349 1350
void blk_mq_start_hw_queues(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	int i;

	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_hw_queue(hctx);
}
EXPORT_SYMBOL(blk_mq_start_hw_queues);

1351 1352 1353 1354 1355 1356 1357 1358 1359 1360
void blk_mq_start_stopped_hw_queue(struct blk_mq_hw_ctx *hctx, bool async)
{
	if (!blk_mq_hctx_stopped(hctx))
		return;

	clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
	blk_mq_run_hw_queue(hctx, async);
}
EXPORT_SYMBOL_GPL(blk_mq_start_stopped_hw_queue);

1361
void blk_mq_start_stopped_hw_queues(struct request_queue *q, bool async)
1362 1363 1364 1365
{
	struct blk_mq_hw_ctx *hctx;
	int i;

1366 1367
	queue_for_each_hw_ctx(q, hctx, i)
		blk_mq_start_stopped_hw_queue(hctx, async);
1368 1369 1370
}
EXPORT_SYMBOL(blk_mq_start_stopped_hw_queues);

1371
static void blk_mq_run_work_fn(struct work_struct *work)
1372 1373 1374
{
	struct blk_mq_hw_ctx *hctx;

1375
	hctx = container_of(work, struct blk_mq_hw_ctx, run_work.work);
1376

1377 1378 1379 1380 1381 1382 1383 1384
	/*
	 * If we are stopped, don't run the queue. The exception is if
	 * BLK_MQ_S_START_ON_RUN is set. For that case, we auto-clear
	 * the STOPPED bit and run it.
	 */
	if (test_bit(BLK_MQ_S_STOPPED, &hctx->state)) {
		if (!test_bit(BLK_MQ_S_START_ON_RUN, &hctx->state))
			return;
1385

1386 1387 1388
		clear_bit(BLK_MQ_S_START_ON_RUN, &hctx->state);
		clear_bit(BLK_MQ_S_STOPPED, &hctx->state);
	}
1389 1390 1391 1392

	__blk_mq_run_hw_queue(hctx);
}

1393 1394 1395

void blk_mq_delay_queue(struct blk_mq_hw_ctx *hctx, unsigned long msecs)
{
1396
	if (WARN_ON_ONCE(!blk_mq_hw_queue_mapped(hctx)))
1397
		return;
1398

1399 1400 1401 1402 1403
	/*
	 * Stop the hw queue, then modify currently delayed work.
	 * This should prevent us from running the queue prematurely.
	 * Mark the queue as auto-clearing STOPPED when it runs.
	 */
1404
	blk_mq_stop_hw_queue(hctx);
1405 1406 1407 1408
	set_bit(BLK_MQ_S_START_ON_RUN, &hctx->state);
	kblockd_mod_delayed_work_on(blk_mq_hctx_next_cpu(hctx),
					&hctx->run_work,
					msecs_to_jiffies(msecs));
1409 1410 1411
}
EXPORT_SYMBOL(blk_mq_delay_queue);

1412 1413 1414
static inline void __blk_mq_insert_req_list(struct blk_mq_hw_ctx *hctx,
					    struct request *rq,
					    bool at_head)
1415
{
J
Jens Axboe 已提交
1416 1417
	struct blk_mq_ctx *ctx = rq->mq_ctx;

1418 1419
	lockdep_assert_held(&ctx->lock);

1420 1421
	trace_block_rq_insert(hctx->queue, rq);

1422 1423 1424 1425
	if (at_head)
		list_add(&rq->queuelist, &ctx->rq_list);
	else
		list_add_tail(&rq->queuelist, &ctx->rq_list);
1426
}
1427

1428 1429
void __blk_mq_insert_request(struct blk_mq_hw_ctx *hctx, struct request *rq,
			     bool at_head)
1430 1431 1432
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;

1433 1434
	lockdep_assert_held(&ctx->lock);

J
Jens Axboe 已提交
1435
	__blk_mq_insert_req_list(hctx, rq, at_head);
1436 1437 1438
	blk_mq_hctx_mark_pending(hctx, ctx);
}

1439 1440 1441 1442
/*
 * Should only be used carefully, when the caller knows we want to
 * bypass a potential IO scheduler on the target device.
 */
1443
void blk_mq_request_bypass_insert(struct request *rq, bool run_queue)
1444 1445 1446 1447 1448 1449 1450 1451
{
	struct blk_mq_ctx *ctx = rq->mq_ctx;
	struct blk_mq_hw_ctx *hctx = blk_mq_map_queue(rq->q, ctx->cpu);

	spin_lock(&hctx->lock);
	list_add_tail(&rq->queuelist, &hctx->dispatch);
	spin_unlock(&hctx->lock);

1452 1453
	if (run_queue)
		blk_mq_run_hw_queue(hctx, false);
1454 1455
}

1456 1457
void blk_mq_insert_requests(struct blk_mq_hw_ctx *hctx, struct blk_mq_ctx *ctx,
			    struct list_head *list)
1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468

{
	/*
	 * preemption doesn't flush plug list, so it's possible ctx->cpu is
	 * offline now
	 */
	spin_lock(&ctx->lock);
	while (!list_empty(list)) {
		struct request *rq;

		rq = list_first_entry(list, struct request, queuelist);
J
Jens Axboe 已提交
1469
		BUG_ON(rq->mq_ctx != ctx);
1470
		list_del_init(&rq->queuelist);
J
Jens Axboe 已提交
1471
		__blk_mq_insert_req_list(hctx, rq, false);
1472
	}
1473
	blk_mq_hctx_mark_pending(hctx, ctx);
1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509
	spin_unlock(&ctx->lock);
}

static int plug_ctx_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct request *rqa = container_of(a, struct request, queuelist);
	struct request *rqb = container_of(b, struct request, queuelist);

	return !(rqa->mq_ctx < rqb->mq_ctx ||
		 (rqa->mq_ctx == rqb->mq_ctx &&
		  blk_rq_pos(rqa) < blk_rq_pos(rqb)));
}

void blk_mq_flush_plug_list(struct blk_plug *plug, bool from_schedule)
{
	struct blk_mq_ctx *this_ctx;
	struct request_queue *this_q;
	struct request *rq;
	LIST_HEAD(list);
	LIST_HEAD(ctx_list);
	unsigned int depth;

	list_splice_init(&plug->mq_list, &list);

	list_sort(NULL, &list, plug_ctx_cmp);

	this_q = NULL;
	this_ctx = NULL;
	depth = 0;

	while (!list_empty(&list)) {
		rq = list_entry_rq(list.next);
		list_del_init(&rq->queuelist);
		BUG_ON(!rq->q);
		if (rq->mq_ctx != this_ctx) {
			if (this_ctx) {
1510 1511 1512 1513
				trace_block_unplug(this_q, depth, from_schedule);
				blk_mq_sched_insert_requests(this_q, this_ctx,
								&ctx_list,
								from_schedule);
1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529
			}

			this_ctx = rq->mq_ctx;
			this_q = rq->q;
			depth = 0;
		}

		depth++;
		list_add_tail(&rq->queuelist, &ctx_list);
	}

	/*
	 * If 'this_ctx' is set, we know we have entries to complete
	 * on 'ctx_list'. Do those.
	 */
	if (this_ctx) {
1530 1531 1532
		trace_block_unplug(this_q, depth, from_schedule);
		blk_mq_sched_insert_requests(this_q, this_ctx, &ctx_list,
						from_schedule);
1533 1534 1535 1536 1537
	}
}

static void blk_mq_bio_to_request(struct request *rq, struct bio *bio)
{
1538
	blk_init_request_from_bio(rq, bio);
1539

S
Shaohua Li 已提交
1540 1541
	blk_rq_set_rl(rq, blk_get_rl(rq->q, bio));

1542
	blk_account_io_start(rq, true);
1543 1544
}

1545 1546 1547 1548 1549 1550 1551
static inline void blk_mq_queue_io(struct blk_mq_hw_ctx *hctx,
				   struct blk_mq_ctx *ctx,
				   struct request *rq)
{
	spin_lock(&ctx->lock);
	__blk_mq_insert_request(hctx, rq, false);
	spin_unlock(&ctx->lock);
1552
}
1553

1554 1555
static blk_qc_t request_to_qc_t(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
1556 1557 1558 1559
	if (rq->tag != -1)
		return blk_tag_to_qc_t(rq->tag, hctx->queue_num, false);

	return blk_tag_to_qc_t(rq->internal_tag, hctx->queue_num, true);
1560 1561
}

M
Ming Lei 已提交
1562 1563 1564
static void __blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
					struct request *rq,
					blk_qc_t *cookie, bool may_sleep)
1565 1566 1567 1568
{
	struct request_queue *q = rq->q;
	struct blk_mq_queue_data bd = {
		.rq = rq,
1569
		.last = true,
1570
	};
1571
	blk_qc_t new_cookie;
1572
	blk_status_t ret;
M
Ming Lei 已提交
1573 1574
	bool run_queue = true;

1575 1576
	/* RCU or SRCU read lock is needed before checking quiesced flag */
	if (blk_mq_hctx_stopped(hctx) || blk_queue_quiesced(q)) {
M
Ming Lei 已提交
1577 1578 1579
		run_queue = false;
		goto insert;
	}
1580

1581
	if (q->elevator)
1582 1583
		goto insert;

M
Ming Lei 已提交
1584
	if (!blk_mq_get_driver_tag(rq, NULL, false))
1585 1586
		goto insert;

1587
	if (!blk_mq_get_dispatch_budget(hctx)) {
1588 1589
		blk_mq_put_driver_tag(rq);
		goto insert;
1590
	}
1591

1592 1593
	new_cookie = request_to_qc_t(hctx, rq);

1594 1595 1596 1597 1598 1599
	/*
	 * For OK queue, we are done. For error, kill it. Any other
	 * error (busy), just add it to our list as we previously
	 * would have done
	 */
	ret = q->mq_ops->queue_rq(hctx, &bd);
1600 1601
	switch (ret) {
	case BLK_STS_OK:
1602
		*cookie = new_cookie;
1603
		return;
1604 1605 1606 1607
	case BLK_STS_RESOURCE:
		__blk_mq_requeue_request(rq);
		goto insert;
	default:
1608
		*cookie = BLK_QC_T_NONE;
1609
		blk_mq_end_request(rq, ret);
1610
		return;
1611
	}
1612

1613
insert:
M
Ming Lei 已提交
1614
	blk_mq_sched_insert_request(rq, false, run_queue, false, may_sleep);
1615 1616
}

1617 1618 1619 1620 1621
static void blk_mq_try_issue_directly(struct blk_mq_hw_ctx *hctx,
		struct request *rq, blk_qc_t *cookie)
{
	if (!(hctx->flags & BLK_MQ_F_BLOCKING)) {
		rcu_read_lock();
M
Ming Lei 已提交
1622
		__blk_mq_try_issue_directly(hctx, rq, cookie, false);
1623 1624
		rcu_read_unlock();
	} else {
1625 1626 1627 1628
		unsigned int srcu_idx;

		might_sleep();

1629
		srcu_idx = srcu_read_lock(hctx->queue_rq_srcu);
M
Ming Lei 已提交
1630
		__blk_mq_try_issue_directly(hctx, rq, cookie, true);
1631
		srcu_read_unlock(hctx->queue_rq_srcu, srcu_idx);
1632 1633 1634
	}
}

1635
static blk_qc_t blk_mq_make_request(struct request_queue *q, struct bio *bio)
1636
{
1637
	const int is_sync = op_is_sync(bio->bi_opf);
1638
	const int is_flush_fua = op_is_flush(bio->bi_opf);
1639
	struct blk_mq_alloc_data data = { .flags = 0 };
1640
	struct request *rq;
1641
	unsigned int request_count = 0;
1642
	struct blk_plug *plug;
1643
	struct request *same_queue_rq = NULL;
1644
	blk_qc_t cookie;
J
Jens Axboe 已提交
1645
	unsigned int wb_acct;
1646 1647 1648

	blk_queue_bounce(q, &bio);

1649
	blk_queue_split(q, &bio);
1650

1651
	if (!bio_integrity_prep(bio))
1652
		return BLK_QC_T_NONE;
1653

1654 1655 1656
	if (!is_flush_fua && !blk_queue_nomerges(q) &&
	    blk_attempt_plug_merge(q, bio, &request_count, &same_queue_rq))
		return BLK_QC_T_NONE;
1657

1658 1659 1660
	if (blk_mq_sched_bio_merge(q, bio))
		return BLK_QC_T_NONE;

J
Jens Axboe 已提交
1661 1662
	wb_acct = wbt_wait(q->rq_wb, bio, NULL);

1663 1664
	trace_block_getrq(q, bio, bio->bi_opf);

1665
	rq = blk_mq_get_request(q, bio, bio->bi_opf, &data);
J
Jens Axboe 已提交
1666 1667
	if (unlikely(!rq)) {
		__wbt_done(q->rq_wb, wb_acct);
1668 1669
		if (bio->bi_opf & REQ_NOWAIT)
			bio_wouldblock_error(bio);
1670
		return BLK_QC_T_NONE;
J
Jens Axboe 已提交
1671 1672 1673
	}

	wbt_track(&rq->issue_stat, wb_acct);
1674

1675
	cookie = request_to_qc_t(data.hctx, rq);
1676

1677
	plug = current->plug;
1678
	if (unlikely(is_flush_fua)) {
1679
		blk_mq_put_ctx(data.ctx);
1680
		blk_mq_bio_to_request(rq, bio);
1681 1682 1683 1684

		/* bypass scheduler for flush rq */
		blk_insert_flush(rq);
		blk_mq_run_hw_queue(data.hctx, true);
1685
	} else if (plug && q->nr_hw_queues == 1) {
1686 1687
		struct request *last = NULL;

1688
		blk_mq_put_ctx(data.ctx);
1689
		blk_mq_bio_to_request(rq, bio);
1690 1691 1692 1693 1694 1695 1696

		/*
		 * @request_count may become stale because of schedule
		 * out, so check the list again.
		 */
		if (list_empty(&plug->mq_list))
			request_count = 0;
1697 1698 1699
		else if (blk_queue_nomerges(q))
			request_count = blk_plug_queued_count(q);

M
Ming Lei 已提交
1700
		if (!request_count)
1701
			trace_block_plug(q);
1702 1703
		else
			last = list_entry_rq(plug->mq_list.prev);
1704

1705 1706
		if (request_count >= BLK_MAX_REQUEST_COUNT || (last &&
		    blk_rq_bytes(last) >= BLK_PLUG_FLUSH_SIZE)) {
1707 1708
			blk_flush_plug_list(plug, false);
			trace_block_plug(q);
1709
		}
1710

1711
		list_add_tail(&rq->queuelist, &plug->mq_list);
1712
	} else if (plug && !blk_queue_nomerges(q)) {
1713
		blk_mq_bio_to_request(rq, bio);
1714 1715

		/*
1716
		 * We do limited plugging. If the bio can be merged, do that.
1717 1718
		 * Otherwise the existing request in the plug list will be
		 * issued. So the plug list will have one request at most
1719 1720
		 * The plug list might get flushed before this. If that happens,
		 * the plug list is empty, and same_queue_rq is invalid.
1721
		 */
1722 1723 1724 1725 1726 1727
		if (list_empty(&plug->mq_list))
			same_queue_rq = NULL;
		if (same_queue_rq)
			list_del_init(&same_queue_rq->queuelist);
		list_add_tail(&rq->queuelist, &plug->mq_list);

1728 1729
		blk_mq_put_ctx(data.ctx);

1730 1731 1732
		if (same_queue_rq) {
			data.hctx = blk_mq_map_queue(q,
					same_queue_rq->mq_ctx->cpu);
1733 1734
			blk_mq_try_issue_directly(data.hctx, same_queue_rq,
					&cookie);
1735
		}
1736
	} else if (q->nr_hw_queues > 1 && is_sync) {
1737
		blk_mq_put_ctx(data.ctx);
1738 1739
		blk_mq_bio_to_request(rq, bio);
		blk_mq_try_issue_directly(data.hctx, rq, &cookie);
1740
	} else if (q->elevator) {
1741
		blk_mq_put_ctx(data.ctx);
1742
		blk_mq_bio_to_request(rq, bio);
1743
		blk_mq_sched_insert_request(rq, false, true, true, true);
1744
	} else {
1745
		blk_mq_put_ctx(data.ctx);
1746 1747
		blk_mq_bio_to_request(rq, bio);
		blk_mq_queue_io(data.hctx, data.ctx, rq);
1748
		blk_mq_run_hw_queue(data.hctx, true);
1749
	}
1750

1751
	return cookie;
1752 1753
}

1754 1755
void blk_mq_free_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx)
1756
{
1757
	struct page *page;
1758

1759
	if (tags->rqs && set->ops->exit_request) {
1760
		int i;
1761

1762
		for (i = 0; i < tags->nr_tags; i++) {
J
Jens Axboe 已提交
1763 1764 1765
			struct request *rq = tags->static_rqs[i];

			if (!rq)
1766
				continue;
1767
			set->ops->exit_request(set, rq, hctx_idx);
J
Jens Axboe 已提交
1768
			tags->static_rqs[i] = NULL;
1769
		}
1770 1771
	}

1772 1773
	while (!list_empty(&tags->page_list)) {
		page = list_first_entry(&tags->page_list, struct page, lru);
1774
		list_del_init(&page->lru);
1775 1776 1777 1778 1779
		/*
		 * Remove kmemleak object previously allocated in
		 * blk_mq_init_rq_map().
		 */
		kmemleak_free(page_address(page));
1780 1781
		__free_pages(page, page->private);
	}
1782
}
1783

1784 1785
void blk_mq_free_rq_map(struct blk_mq_tags *tags)
{
1786
	kfree(tags->rqs);
1787
	tags->rqs = NULL;
J
Jens Axboe 已提交
1788 1789
	kfree(tags->static_rqs);
	tags->static_rqs = NULL;
1790

1791
	blk_mq_free_tags(tags);
1792 1793
}

1794 1795 1796 1797
struct blk_mq_tags *blk_mq_alloc_rq_map(struct blk_mq_tag_set *set,
					unsigned int hctx_idx,
					unsigned int nr_tags,
					unsigned int reserved_tags)
1798
{
1799
	struct blk_mq_tags *tags;
1800
	int node;
1801

1802 1803 1804 1805 1806
	node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
	if (node == NUMA_NO_NODE)
		node = set->numa_node;

	tags = blk_mq_init_tags(nr_tags, reserved_tags, node,
S
Shaohua Li 已提交
1807
				BLK_MQ_FLAG_TO_ALLOC_POLICY(set->flags));
1808 1809
	if (!tags)
		return NULL;
1810

1811
	tags->rqs = kzalloc_node(nr_tags * sizeof(struct request *),
1812
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1813
				 node);
1814 1815 1816 1817
	if (!tags->rqs) {
		blk_mq_free_tags(tags);
		return NULL;
	}
1818

J
Jens Axboe 已提交
1819 1820
	tags->static_rqs = kzalloc_node(nr_tags * sizeof(struct request *),
				 GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY,
1821
				 node);
J
Jens Axboe 已提交
1822 1823 1824 1825 1826 1827
	if (!tags->static_rqs) {
		kfree(tags->rqs);
		blk_mq_free_tags(tags);
		return NULL;
	}

1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840
	return tags;
}

static size_t order_to_size(unsigned int order)
{
	return (size_t)PAGE_SIZE << order;
}

int blk_mq_alloc_rqs(struct blk_mq_tag_set *set, struct blk_mq_tags *tags,
		     unsigned int hctx_idx, unsigned int depth)
{
	unsigned int i, j, entries_per_page, max_order = 4;
	size_t rq_size, left;
1841 1842 1843 1844 1845
	int node;

	node = blk_mq_hw_queue_to_node(set->mq_map, hctx_idx);
	if (node == NUMA_NO_NODE)
		node = set->numa_node;
1846 1847 1848

	INIT_LIST_HEAD(&tags->page_list);

1849 1850 1851 1852
	/*
	 * rq_size is the size of the request plus driver payload, rounded
	 * to the cacheline size
	 */
1853
	rq_size = round_up(sizeof(struct request) + set->cmd_size,
1854
				cache_line_size());
1855
	left = rq_size * depth;
1856

1857
	for (i = 0; i < depth; ) {
1858 1859 1860 1861 1862
		int this_order = max_order;
		struct page *page;
		int to_do;
		void *p;

1863
		while (this_order && left < order_to_size(this_order - 1))
1864 1865 1866
			this_order--;

		do {
1867
			page = alloc_pages_node(node,
1868
				GFP_NOIO | __GFP_NOWARN | __GFP_NORETRY | __GFP_ZERO,
1869
				this_order);
1870 1871 1872 1873 1874 1875 1876 1877 1878
			if (page)
				break;
			if (!this_order--)
				break;
			if (order_to_size(this_order) < rq_size)
				break;
		} while (1);

		if (!page)
1879
			goto fail;
1880 1881

		page->private = this_order;
1882
		list_add_tail(&page->lru, &tags->page_list);
1883 1884

		p = page_address(page);
1885 1886 1887 1888
		/*
		 * Allow kmemleak to scan these pages as they contain pointers
		 * to additional allocations like via ops->init_request().
		 */
1889
		kmemleak_alloc(p, order_to_size(this_order), 1, GFP_NOIO);
1890
		entries_per_page = order_to_size(this_order) / rq_size;
1891
		to_do = min(entries_per_page, depth - i);
1892 1893
		left -= to_do * rq_size;
		for (j = 0; j < to_do; j++) {
J
Jens Axboe 已提交
1894 1895 1896
			struct request *rq = p;

			tags->static_rqs[i] = rq;
1897
			if (set->ops->init_request) {
1898
				if (set->ops->init_request(set, rq, hctx_idx,
1899
						node)) {
J
Jens Axboe 已提交
1900
					tags->static_rqs[i] = NULL;
1901
					goto fail;
1902
				}
1903 1904
			}

1905 1906 1907 1908
			p += rq_size;
			i++;
		}
	}
1909
	return 0;
1910

1911
fail:
1912 1913
	blk_mq_free_rqs(set, tags, hctx_idx);
	return -ENOMEM;
1914 1915
}

J
Jens Axboe 已提交
1916 1917 1918 1919 1920
/*
 * 'cpu' is going away. splice any existing rq_list entries from this
 * software queue to the hw queue dispatch list, and ensure that it
 * gets run.
 */
1921
static int blk_mq_hctx_notify_dead(unsigned int cpu, struct hlist_node *node)
1922
{
1923
	struct blk_mq_hw_ctx *hctx;
1924 1925 1926
	struct blk_mq_ctx *ctx;
	LIST_HEAD(tmp);

1927
	hctx = hlist_entry_safe(node, struct blk_mq_hw_ctx, cpuhp_dead);
J
Jens Axboe 已提交
1928
	ctx = __blk_mq_get_ctx(hctx->queue, cpu);
1929 1930 1931 1932 1933 1934 1935 1936 1937

	spin_lock(&ctx->lock);
	if (!list_empty(&ctx->rq_list)) {
		list_splice_init(&ctx->rq_list, &tmp);
		blk_mq_hctx_clear_pending(hctx, ctx);
	}
	spin_unlock(&ctx->lock);

	if (list_empty(&tmp))
1938
		return 0;
1939

J
Jens Axboe 已提交
1940 1941 1942
	spin_lock(&hctx->lock);
	list_splice_tail_init(&tmp, &hctx->dispatch);
	spin_unlock(&hctx->lock);
1943 1944

	blk_mq_run_hw_queue(hctx, true);
1945
	return 0;
1946 1947
}

1948
static void blk_mq_remove_cpuhp(struct blk_mq_hw_ctx *hctx)
1949
{
1950 1951
	cpuhp_state_remove_instance_nocalls(CPUHP_BLK_MQ_DEAD,
					    &hctx->cpuhp_dead);
1952 1953
}

1954
/* hctx->ctxs will be freed in queue's release handler */
1955 1956 1957 1958
static void blk_mq_exit_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
1959 1960
	blk_mq_debugfs_unregister_hctx(hctx);

1961 1962
	blk_mq_tag_idle(hctx);

1963
	if (set->ops->exit_request)
1964
		set->ops->exit_request(set, hctx->fq->flush_rq, hctx_idx);
1965

1966 1967
	blk_mq_sched_exit_hctx(q, hctx, hctx_idx);

1968 1969 1970
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);

1971
	if (hctx->flags & BLK_MQ_F_BLOCKING)
1972
		cleanup_srcu_struct(hctx->queue_rq_srcu);
1973

1974
	blk_mq_remove_cpuhp(hctx);
1975
	blk_free_flush_queue(hctx->fq);
1976
	sbitmap_free(&hctx->ctx_map);
1977 1978
}

M
Ming Lei 已提交
1979 1980 1981 1982 1983 1984 1985 1986 1987
static void blk_mq_exit_hw_queues(struct request_queue *q,
		struct blk_mq_tag_set *set, int nr_queue)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	queue_for_each_hw_ctx(q, hctx, i) {
		if (i == nr_queue)
			break;
1988
		blk_mq_exit_hctx(q, set, hctx, i);
M
Ming Lei 已提交
1989 1990 1991
	}
}

1992 1993 1994
static int blk_mq_init_hctx(struct request_queue *q,
		struct blk_mq_tag_set *set,
		struct blk_mq_hw_ctx *hctx, unsigned hctx_idx)
1995
{
1996 1997 1998 1999 2000 2001
	int node;

	node = hctx->numa_node;
	if (node == NUMA_NO_NODE)
		node = hctx->numa_node = set->numa_node;

2002
	INIT_DELAYED_WORK(&hctx->run_work, blk_mq_run_work_fn);
2003 2004 2005
	spin_lock_init(&hctx->lock);
	INIT_LIST_HEAD(&hctx->dispatch);
	hctx->queue = q;
2006
	hctx->flags = set->flags & ~BLK_MQ_F_TAG_SHARED;
2007

2008
	cpuhp_state_add_instance_nocalls(CPUHP_BLK_MQ_DEAD, &hctx->cpuhp_dead);
2009 2010

	hctx->tags = set->tags[hctx_idx];
2011 2012

	/*
2013 2014
	 * Allocate space for all possible cpus to avoid allocation at
	 * runtime
2015
	 */
2016 2017 2018 2019
	hctx->ctxs = kmalloc_node(nr_cpu_ids * sizeof(void *),
					GFP_KERNEL, node);
	if (!hctx->ctxs)
		goto unregister_cpu_notifier;
2020

2021 2022
	if (sbitmap_init_node(&hctx->ctx_map, nr_cpu_ids, ilog2(8), GFP_KERNEL,
			      node))
2023
		goto free_ctxs;
2024

2025
	hctx->nr_ctx = 0;
2026

2027 2028 2029
	init_waitqueue_func_entry(&hctx->dispatch_wait, blk_mq_dispatch_wake);
	INIT_LIST_HEAD(&hctx->dispatch_wait.entry);

2030 2031 2032
	if (set->ops->init_hctx &&
	    set->ops->init_hctx(hctx, set->driver_data, hctx_idx))
		goto free_bitmap;
2033

2034 2035 2036
	if (blk_mq_sched_init_hctx(q, hctx, hctx_idx))
		goto exit_hctx;

2037 2038
	hctx->fq = blk_alloc_flush_queue(q, hctx->numa_node, set->cmd_size);
	if (!hctx->fq)
2039
		goto sched_exit_hctx;
2040

2041
	if (set->ops->init_request &&
2042 2043
	    set->ops->init_request(set, hctx->fq->flush_rq, hctx_idx,
				   node))
2044
		goto free_fq;
2045

2046
	if (hctx->flags & BLK_MQ_F_BLOCKING)
2047
		init_srcu_struct(hctx->queue_rq_srcu);
2048

2049 2050
	blk_mq_debugfs_register_hctx(q, hctx);

2051
	return 0;
2052

2053 2054
 free_fq:
	kfree(hctx->fq);
2055 2056
 sched_exit_hctx:
	blk_mq_sched_exit_hctx(q, hctx, hctx_idx);
2057 2058 2059
 exit_hctx:
	if (set->ops->exit_hctx)
		set->ops->exit_hctx(hctx, hctx_idx);
2060
 free_bitmap:
2061
	sbitmap_free(&hctx->ctx_map);
2062 2063 2064
 free_ctxs:
	kfree(hctx->ctxs);
 unregister_cpu_notifier:
2065
	blk_mq_remove_cpuhp(hctx);
2066 2067
	return -1;
}
2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082

static void blk_mq_init_cpu_queues(struct request_queue *q,
				   unsigned int nr_hw_queues)
{
	unsigned int i;

	for_each_possible_cpu(i) {
		struct blk_mq_ctx *__ctx = per_cpu_ptr(q->queue_ctx, i);
		struct blk_mq_hw_ctx *hctx;

		__ctx->cpu = i;
		spin_lock_init(&__ctx->lock);
		INIT_LIST_HEAD(&__ctx->rq_list);
		__ctx->queue = q;

2083 2084
		/* If the cpu isn't present, the cpu is mapped to first hctx */
		if (!cpu_present(i))
2085 2086
			continue;

C
Christoph Hellwig 已提交
2087
		hctx = blk_mq_map_queue(q, i);
2088

2089 2090 2091 2092 2093
		/*
		 * Set local node, IFF we have more than one hw queue. If
		 * not, we remain on the home node of the device
		 */
		if (nr_hw_queues > 1 && hctx->numa_node == NUMA_NO_NODE)
2094
			hctx->numa_node = local_memory_node(cpu_to_node(i));
2095 2096 2097
	}
}

2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119
static bool __blk_mq_alloc_rq_map(struct blk_mq_tag_set *set, int hctx_idx)
{
	int ret = 0;

	set->tags[hctx_idx] = blk_mq_alloc_rq_map(set, hctx_idx,
					set->queue_depth, set->reserved_tags);
	if (!set->tags[hctx_idx])
		return false;

	ret = blk_mq_alloc_rqs(set, set->tags[hctx_idx], hctx_idx,
				set->queue_depth);
	if (!ret)
		return true;

	blk_mq_free_rq_map(set->tags[hctx_idx]);
	set->tags[hctx_idx] = NULL;
	return false;
}

static void blk_mq_free_map_and_requests(struct blk_mq_tag_set *set,
					 unsigned int hctx_idx)
{
2120 2121 2122 2123 2124
	if (set->tags[hctx_idx]) {
		blk_mq_free_rqs(set, set->tags[hctx_idx], hctx_idx);
		blk_mq_free_rq_map(set->tags[hctx_idx]);
		set->tags[hctx_idx] = NULL;
	}
2125 2126
}

2127
static void blk_mq_map_swqueue(struct request_queue *q)
2128
{
2129
	unsigned int i, hctx_idx;
2130 2131
	struct blk_mq_hw_ctx *hctx;
	struct blk_mq_ctx *ctx;
M
Ming Lei 已提交
2132
	struct blk_mq_tag_set *set = q->tag_set;
2133

2134 2135 2136 2137 2138
	/*
	 * Avoid others reading imcomplete hctx->cpumask through sysfs
	 */
	mutex_lock(&q->sysfs_lock);

2139
	queue_for_each_hw_ctx(q, hctx, i) {
2140
		cpumask_clear(hctx->cpumask);
2141 2142 2143 2144
		hctx->nr_ctx = 0;
	}

	/*
2145 2146 2147
	 * Map software to hardware queues.
	 *
	 * If the cpu isn't present, the cpu is mapped to first hctx.
2148
	 */
2149
	for_each_present_cpu(i) {
2150 2151
		hctx_idx = q->mq_map[i];
		/* unmapped hw queue can be remapped after CPU topo changed */
2152 2153
		if (!set->tags[hctx_idx] &&
		    !__blk_mq_alloc_rq_map(set, hctx_idx)) {
2154 2155 2156 2157 2158 2159
			/*
			 * If tags initialization fail for some hctx,
			 * that hctx won't be brought online.  In this
			 * case, remap the current ctx to hctx[0] which
			 * is guaranteed to always have tags allocated
			 */
2160
			q->mq_map[i] = 0;
2161 2162
		}

2163
		ctx = per_cpu_ptr(q->queue_ctx, i);
C
Christoph Hellwig 已提交
2164
		hctx = blk_mq_map_queue(q, i);
K
Keith Busch 已提交
2165

2166
		cpumask_set_cpu(i, hctx->cpumask);
2167 2168 2169
		ctx->index_hw = hctx->nr_ctx;
		hctx->ctxs[hctx->nr_ctx++] = ctx;
	}
2170

2171 2172
	mutex_unlock(&q->sysfs_lock);

2173
	queue_for_each_hw_ctx(q, hctx, i) {
2174
		/*
2175 2176
		 * If no software queues are mapped to this hardware queue,
		 * disable it and free the request entries.
2177 2178
		 */
		if (!hctx->nr_ctx) {
2179 2180 2181 2182
			/* Never unmap queue 0.  We need it as a
			 * fallback in case of a new remap fails
			 * allocation
			 */
2183 2184 2185
			if (i && set->tags[i])
				blk_mq_free_map_and_requests(set, i);

M
Ming Lei 已提交
2186
			hctx->tags = NULL;
2187 2188 2189
			continue;
		}

M
Ming Lei 已提交
2190 2191 2192
		hctx->tags = set->tags[i];
		WARN_ON(!hctx->tags);

2193 2194 2195 2196 2197
		/*
		 * Set the map size to the number of mapped software queues.
		 * This is more accurate and more efficient than looping
		 * over all possibly mapped software queues.
		 */
2198
		sbitmap_resize(&hctx->ctx_map, hctx->nr_ctx);
2199

2200 2201 2202
		/*
		 * Initialize batch roundrobin counts
		 */
2203 2204 2205
		hctx->next_cpu = cpumask_first(hctx->cpumask);
		hctx->next_cpu_batch = BLK_MQ_CPU_WORK_BATCH;
	}
2206 2207
}

2208 2209 2210 2211
/*
 * Caller needs to ensure that we're either frozen/quiesced, or that
 * the queue isn't live yet.
 */
2212
static void queue_set_hctx_shared(struct request_queue *q, bool shared)
2213 2214 2215 2216
{
	struct blk_mq_hw_ctx *hctx;
	int i;

2217
	queue_for_each_hw_ctx(q, hctx, i) {
2218 2219 2220
		if (shared) {
			if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
				atomic_inc(&q->shared_hctx_restart);
2221
			hctx->flags |= BLK_MQ_F_TAG_SHARED;
2222 2223 2224
		} else {
			if (test_bit(BLK_MQ_S_SCHED_RESTART, &hctx->state))
				atomic_dec(&q->shared_hctx_restart);
2225
			hctx->flags &= ~BLK_MQ_F_TAG_SHARED;
2226
		}
2227 2228 2229
	}
}

2230 2231
static void blk_mq_update_tag_set_depth(struct blk_mq_tag_set *set,
					bool shared)
2232 2233
{
	struct request_queue *q;
2234

2235 2236
	lockdep_assert_held(&set->tag_list_lock);

2237 2238
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_freeze_queue(q);
2239
		queue_set_hctx_shared(q, shared);
2240 2241 2242 2243 2244 2245 2246 2247 2248
		blk_mq_unfreeze_queue(q);
	}
}

static void blk_mq_del_queue_tag_set(struct request_queue *q)
{
	struct blk_mq_tag_set *set = q->tag_set;

	mutex_lock(&set->tag_list_lock);
2249 2250
	list_del_rcu(&q->tag_set_list);
	INIT_LIST_HEAD(&q->tag_set_list);
2251 2252 2253 2254 2255 2256
	if (list_is_singular(&set->tag_list)) {
		/* just transitioned to unshared */
		set->flags &= ~BLK_MQ_F_TAG_SHARED;
		/* update existing queue */
		blk_mq_update_tag_set_depth(set, false);
	}
2257
	mutex_unlock(&set->tag_list_lock);
2258 2259

	synchronize_rcu();
2260 2261 2262 2263 2264 2265 2266 2267
}

static void blk_mq_add_queue_tag_set(struct blk_mq_tag_set *set,
				     struct request_queue *q)
{
	q->tag_set = set;

	mutex_lock(&set->tag_list_lock);
2268 2269 2270 2271 2272 2273 2274 2275 2276

	/* Check to see if we're transitioning to shared (from 1 to 2 queues). */
	if (!list_empty(&set->tag_list) && !(set->flags & BLK_MQ_F_TAG_SHARED)) {
		set->flags |= BLK_MQ_F_TAG_SHARED;
		/* update existing queue */
		blk_mq_update_tag_set_depth(set, true);
	}
	if (set->flags & BLK_MQ_F_TAG_SHARED)
		queue_set_hctx_shared(q, true);
2277
	list_add_tail_rcu(&q->tag_set_list, &set->tag_list);
2278

2279 2280 2281
	mutex_unlock(&set->tag_list_lock);
}

2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293
/*
 * It is the actual release handler for mq, but we do it from
 * request queue's release handler for avoiding use-after-free
 * and headache because q->mq_kobj shouldn't have been introduced,
 * but we can't group ctx/kctx kobj without it.
 */
void blk_mq_release(struct request_queue *q)
{
	struct blk_mq_hw_ctx *hctx;
	unsigned int i;

	/* hctx kobj stays in hctx */
2294 2295 2296
	queue_for_each_hw_ctx(q, hctx, i) {
		if (!hctx)
			continue;
2297
		kobject_put(&hctx->kobj);
2298
	}
2299

2300 2301
	q->mq_map = NULL;

2302 2303
	kfree(q->queue_hw_ctx);

2304 2305 2306 2307 2308 2309
	/*
	 * release .mq_kobj and sw queue's kobject now because
	 * both share lifetime with request queue.
	 */
	blk_mq_sysfs_deinit(q);

2310 2311 2312
	free_percpu(q->queue_ctx);
}

2313
struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328
{
	struct request_queue *uninit_q, *q;

	uninit_q = blk_alloc_queue_node(GFP_KERNEL, set->numa_node);
	if (!uninit_q)
		return ERR_PTR(-ENOMEM);

	q = blk_mq_init_allocated_queue(set, uninit_q);
	if (IS_ERR(q))
		blk_cleanup_queue(uninit_q);

	return q;
}
EXPORT_SYMBOL(blk_mq_init_queue);

2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342
static int blk_mq_hw_ctx_size(struct blk_mq_tag_set *tag_set)
{
	int hw_ctx_size = sizeof(struct blk_mq_hw_ctx);

	BUILD_BUG_ON(ALIGN(offsetof(struct blk_mq_hw_ctx, queue_rq_srcu),
			   __alignof__(struct blk_mq_hw_ctx)) !=
		     sizeof(struct blk_mq_hw_ctx));

	if (tag_set->flags & BLK_MQ_F_BLOCKING)
		hw_ctx_size += sizeof(struct srcu_struct);

	return hw_ctx_size;
}

K
Keith Busch 已提交
2343 2344
static void blk_mq_realloc_hw_ctxs(struct blk_mq_tag_set *set,
						struct request_queue *q)
2345
{
K
Keith Busch 已提交
2346 2347
	int i, j;
	struct blk_mq_hw_ctx **hctxs = q->queue_hw_ctx;
2348

K
Keith Busch 已提交
2349
	blk_mq_sysfs_unregister(q);
2350
	for (i = 0; i < set->nr_hw_queues; i++) {
K
Keith Busch 已提交
2351
		int node;
2352

K
Keith Busch 已提交
2353 2354 2355 2356
		if (hctxs[i])
			continue;

		node = blk_mq_hw_queue_to_node(q->mq_map, i);
2357
		hctxs[i] = kzalloc_node(blk_mq_hw_ctx_size(set),
2358
					GFP_KERNEL, node);
2359
		if (!hctxs[i])
K
Keith Busch 已提交
2360
			break;
2361

2362
		if (!zalloc_cpumask_var_node(&hctxs[i]->cpumask, GFP_KERNEL,
K
Keith Busch 已提交
2363 2364 2365 2366 2367
						node)) {
			kfree(hctxs[i]);
			hctxs[i] = NULL;
			break;
		}
2368

2369
		atomic_set(&hctxs[i]->nr_active, 0);
2370
		hctxs[i]->numa_node = node;
2371
		hctxs[i]->queue_num = i;
K
Keith Busch 已提交
2372 2373 2374 2375 2376 2377 2378 2379

		if (blk_mq_init_hctx(q, set, hctxs[i], i)) {
			free_cpumask_var(hctxs[i]->cpumask);
			kfree(hctxs[i]);
			hctxs[i] = NULL;
			break;
		}
		blk_mq_hctx_kobj_init(hctxs[i]);
2380
	}
K
Keith Busch 已提交
2381 2382 2383 2384
	for (j = i; j < q->nr_hw_queues; j++) {
		struct blk_mq_hw_ctx *hctx = hctxs[j];

		if (hctx) {
2385 2386
			if (hctx->tags)
				blk_mq_free_map_and_requests(set, j);
K
Keith Busch 已提交
2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399
			blk_mq_exit_hctx(q, set, hctx, j);
			kobject_put(&hctx->kobj);
			hctxs[j] = NULL;

		}
	}
	q->nr_hw_queues = i;
	blk_mq_sysfs_register(q);
}

struct request_queue *blk_mq_init_allocated_queue(struct blk_mq_tag_set *set,
						  struct request_queue *q)
{
M
Ming Lei 已提交
2400 2401 2402
	/* mark the queue as mq asap */
	q->mq_ops = set->ops;

2403
	q->poll_cb = blk_stat_alloc_callback(blk_mq_poll_stats_fn,
2404 2405
					     blk_mq_poll_stats_bkt,
					     BLK_MQ_POLL_STATS_BKTS, q);
2406 2407 2408
	if (!q->poll_cb)
		goto err_exit;

K
Keith Busch 已提交
2409 2410
	q->queue_ctx = alloc_percpu(struct blk_mq_ctx);
	if (!q->queue_ctx)
M
Ming Lin 已提交
2411
		goto err_exit;
K
Keith Busch 已提交
2412

2413 2414 2415
	/* init q->mq_kobj and sw queues' kobjects */
	blk_mq_sysfs_init(q);

K
Keith Busch 已提交
2416 2417 2418 2419 2420
	q->queue_hw_ctx = kzalloc_node(nr_cpu_ids * sizeof(*(q->queue_hw_ctx)),
						GFP_KERNEL, set->numa_node);
	if (!q->queue_hw_ctx)
		goto err_percpu;

2421
	q->mq_map = set->mq_map;
K
Keith Busch 已提交
2422 2423 2424 2425

	blk_mq_realloc_hw_ctxs(set, q);
	if (!q->nr_hw_queues)
		goto err_hctxs;
2426

2427
	INIT_WORK(&q->timeout_work, blk_mq_timeout_work);
2428
	blk_queue_rq_timeout(q, set->timeout ? set->timeout : 30 * HZ);
2429 2430 2431

	q->nr_queues = nr_cpu_ids;

2432
	q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
2433

2434 2435 2436
	if (!(set->flags & BLK_MQ_F_SG_MERGE))
		q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;

2437 2438
	q->sg_reserved_size = INT_MAX;

2439
	INIT_DELAYED_WORK(&q->requeue_work, blk_mq_requeue_work);
2440 2441 2442
	INIT_LIST_HEAD(&q->requeue_list);
	spin_lock_init(&q->requeue_lock);

2443
	blk_queue_make_request(q, blk_mq_make_request);
2444 2445
	if (q->mq_ops->poll)
		q->poll_fn = blk_mq_poll;
2446

2447 2448 2449 2450 2451
	/*
	 * Do this after blk_queue_make_request() overrides it...
	 */
	q->nr_requests = set->queue_depth;

2452 2453 2454 2455 2456
	/*
	 * Default to classic polling
	 */
	q->poll_nsec = -1;

2457 2458
	if (set->ops->complete)
		blk_queue_softirq_done(q, set->ops->complete);
2459

2460
	blk_mq_init_cpu_queues(q, set->nr_hw_queues);
2461
	blk_mq_add_queue_tag_set(set, q);
2462
	blk_mq_map_swqueue(q);
2463

2464 2465 2466 2467 2468 2469 2470 2471
	if (!(set->flags & BLK_MQ_F_NO_SCHED)) {
		int ret;

		ret = blk_mq_sched_init(q);
		if (ret)
			return ERR_PTR(ret);
	}

2472
	return q;
2473

2474
err_hctxs:
K
Keith Busch 已提交
2475
	kfree(q->queue_hw_ctx);
2476
err_percpu:
K
Keith Busch 已提交
2477
	free_percpu(q->queue_ctx);
M
Ming Lin 已提交
2478 2479
err_exit:
	q->mq_ops = NULL;
2480 2481
	return ERR_PTR(-ENOMEM);
}
2482
EXPORT_SYMBOL(blk_mq_init_allocated_queue);
2483 2484 2485

void blk_mq_free_queue(struct request_queue *q)
{
M
Ming Lei 已提交
2486
	struct blk_mq_tag_set	*set = q->tag_set;
2487

2488
	blk_mq_del_queue_tag_set(q);
M
Ming Lei 已提交
2489
	blk_mq_exit_hw_queues(q, set, set->nr_hw_queues);
2490 2491 2492
}

/* Basically redo blk_mq_init_queue with queue frozen */
2493
static void blk_mq_queue_reinit(struct request_queue *q)
2494
{
2495
	WARN_ON_ONCE(!atomic_read(&q->mq_freeze_depth));
2496

2497
	blk_mq_debugfs_unregister_hctxs(q);
2498 2499
	blk_mq_sysfs_unregister(q);

2500 2501 2502 2503 2504 2505
	/*
	 * redo blk_mq_init_cpu_queues and blk_mq_init_hw_queues. FIXME: maybe
	 * we should change hctx numa_node according to new topology (this
	 * involves free and re-allocate memory, worthy doing?)
	 */

2506
	blk_mq_map_swqueue(q);
2507

2508
	blk_mq_sysfs_register(q);
2509
	blk_mq_debugfs_register_hctxs(q);
2510 2511
}

2512 2513 2514 2515
static int __blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	int i;

2516 2517
	for (i = 0; i < set->nr_hw_queues; i++)
		if (!__blk_mq_alloc_rq_map(set, i))
2518 2519 2520 2521 2522 2523
			goto out_unwind;

	return 0;

out_unwind:
	while (--i >= 0)
2524
		blk_mq_free_rq_map(set->tags[i]);
2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560 2561 2562 2563

	return -ENOMEM;
}

/*
 * Allocate the request maps associated with this tag_set. Note that this
 * may reduce the depth asked for, if memory is tight. set->queue_depth
 * will be updated to reflect the allocated depth.
 */
static int blk_mq_alloc_rq_maps(struct blk_mq_tag_set *set)
{
	unsigned int depth;
	int err;

	depth = set->queue_depth;
	do {
		err = __blk_mq_alloc_rq_maps(set);
		if (!err)
			break;

		set->queue_depth >>= 1;
		if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN) {
			err = -ENOMEM;
			break;
		}
	} while (set->queue_depth);

	if (!set->queue_depth || err) {
		pr_err("blk-mq: failed to allocate request map\n");
		return -ENOMEM;
	}

	if (depth != set->queue_depth)
		pr_info("blk-mq: reduced tag depth (%u -> %u)\n",
						depth, set->queue_depth);

	return 0;
}

2564 2565 2566 2567 2568 2569 2570 2571
static int blk_mq_update_queue_map(struct blk_mq_tag_set *set)
{
	if (set->ops->map_queues)
		return set->ops->map_queues(set);
	else
		return blk_mq_map_queues(set);
}

2572 2573 2574 2575 2576 2577
/*
 * Alloc a tag set to be associated with one or more request queues.
 * May fail with EINVAL for various error conditions. May adjust the
 * requested depth down, if if it too large. In that case, the set
 * value will be stored in set->queue_depth.
 */
2578 2579
int blk_mq_alloc_tag_set(struct blk_mq_tag_set *set)
{
2580 2581
	int ret;

B
Bart Van Assche 已提交
2582 2583
	BUILD_BUG_ON(BLK_MQ_MAX_DEPTH > 1 << BLK_MQ_UNIQUE_TAG_BITS);

2584 2585
	if (!set->nr_hw_queues)
		return -EINVAL;
2586
	if (!set->queue_depth)
2587 2588 2589 2590
		return -EINVAL;
	if (set->queue_depth < set->reserved_tags + BLK_MQ_TAG_MIN)
		return -EINVAL;

C
Christoph Hellwig 已提交
2591
	if (!set->ops->queue_rq)
2592 2593
		return -EINVAL;

2594 2595 2596
	if (!set->ops->get_budget ^ !set->ops->put_budget)
		return -EINVAL;

2597 2598 2599 2600 2601
	if (set->queue_depth > BLK_MQ_MAX_DEPTH) {
		pr_info("blk-mq: reduced tag depth to %u\n",
			BLK_MQ_MAX_DEPTH);
		set->queue_depth = BLK_MQ_MAX_DEPTH;
	}
2602

2603 2604 2605 2606 2607 2608 2609 2610 2611
	/*
	 * If a crashdump is active, then we are potentially in a very
	 * memory constrained environment. Limit us to 1 queue and
	 * 64 tags to prevent using too much memory.
	 */
	if (is_kdump_kernel()) {
		set->nr_hw_queues = 1;
		set->queue_depth = min(64U, set->queue_depth);
	}
K
Keith Busch 已提交
2612 2613 2614 2615 2616
	/*
	 * There is no use for more h/w queues than cpus.
	 */
	if (set->nr_hw_queues > nr_cpu_ids)
		set->nr_hw_queues = nr_cpu_ids;
2617

K
Keith Busch 已提交
2618
	set->tags = kzalloc_node(nr_cpu_ids * sizeof(struct blk_mq_tags *),
2619 2620
				 GFP_KERNEL, set->numa_node);
	if (!set->tags)
2621
		return -ENOMEM;
2622

2623 2624 2625
	ret = -ENOMEM;
	set->mq_map = kzalloc_node(sizeof(*set->mq_map) * nr_cpu_ids,
			GFP_KERNEL, set->numa_node);
2626 2627 2628
	if (!set->mq_map)
		goto out_free_tags;

2629
	ret = blk_mq_update_queue_map(set);
2630 2631 2632 2633 2634
	if (ret)
		goto out_free_mq_map;

	ret = blk_mq_alloc_rq_maps(set);
	if (ret)
2635
		goto out_free_mq_map;
2636

2637 2638 2639
	mutex_init(&set->tag_list_lock);
	INIT_LIST_HEAD(&set->tag_list);

2640
	return 0;
2641 2642 2643 2644 2645

out_free_mq_map:
	kfree(set->mq_map);
	set->mq_map = NULL;
out_free_tags:
2646 2647
	kfree(set->tags);
	set->tags = NULL;
2648
	return ret;
2649 2650 2651 2652 2653 2654 2655
}
EXPORT_SYMBOL(blk_mq_alloc_tag_set);

void blk_mq_free_tag_set(struct blk_mq_tag_set *set)
{
	int i;

2656 2657
	for (i = 0; i < nr_cpu_ids; i++)
		blk_mq_free_map_and_requests(set, i);
2658

2659 2660 2661
	kfree(set->mq_map);
	set->mq_map = NULL;

M
Ming Lei 已提交
2662
	kfree(set->tags);
2663
	set->tags = NULL;
2664 2665 2666
}
EXPORT_SYMBOL(blk_mq_free_tag_set);

2667 2668 2669 2670 2671 2672
int blk_mq_update_nr_requests(struct request_queue *q, unsigned int nr)
{
	struct blk_mq_tag_set *set = q->tag_set;
	struct blk_mq_hw_ctx *hctx;
	int i, ret;

2673
	if (!set)
2674 2675
		return -EINVAL;

2676 2677
	blk_mq_freeze_queue(q);

2678 2679
	ret = 0;
	queue_for_each_hw_ctx(q, hctx, i) {
2680 2681
		if (!hctx->tags)
			continue;
2682 2683 2684 2685
		/*
		 * If we're using an MQ scheduler, just update the scheduler
		 * queue depth. This is similar to what the old code would do.
		 */
2686
		if (!hctx->sched_tags) {
2687
			ret = blk_mq_tag_update_depth(hctx, &hctx->tags, nr,
2688 2689 2690 2691 2692
							false);
		} else {
			ret = blk_mq_tag_update_depth(hctx, &hctx->sched_tags,
							nr, true);
		}
2693 2694 2695 2696 2697 2698 2699
		if (ret)
			break;
	}

	if (!ret)
		q->nr_requests = nr;

2700 2701
	blk_mq_unfreeze_queue(q);

2702 2703 2704
	return ret;
}

2705 2706
static void __blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set,
							int nr_hw_queues)
K
Keith Busch 已提交
2707 2708 2709
{
	struct request_queue *q;

2710 2711
	lockdep_assert_held(&set->tag_list_lock);

K
Keith Busch 已提交
2712 2713 2714 2715 2716 2717 2718 2719 2720
	if (nr_hw_queues > nr_cpu_ids)
		nr_hw_queues = nr_cpu_ids;
	if (nr_hw_queues < 1 || nr_hw_queues == set->nr_hw_queues)
		return;

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_freeze_queue(q);

	set->nr_hw_queues = nr_hw_queues;
2721
	blk_mq_update_queue_map(set);
K
Keith Busch 已提交
2722 2723
	list_for_each_entry(q, &set->tag_list, tag_set_list) {
		blk_mq_realloc_hw_ctxs(set, q);
2724
		blk_mq_queue_reinit(q);
K
Keith Busch 已提交
2725 2726 2727 2728 2729
	}

	list_for_each_entry(q, &set->tag_list, tag_set_list)
		blk_mq_unfreeze_queue(q);
}
2730 2731 2732 2733 2734 2735 2736

void blk_mq_update_nr_hw_queues(struct blk_mq_tag_set *set, int nr_hw_queues)
{
	mutex_lock(&set->tag_list_lock);
	__blk_mq_update_nr_hw_queues(set, nr_hw_queues);
	mutex_unlock(&set->tag_list_lock);
}
K
Keith Busch 已提交
2737 2738
EXPORT_SYMBOL_GPL(blk_mq_update_nr_hw_queues);

2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764
/* Enable polling stats and return whether they were already enabled. */
static bool blk_poll_stats_enable(struct request_queue *q)
{
	if (test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
	    test_and_set_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags))
		return true;
	blk_stat_add_callback(q, q->poll_cb);
	return false;
}

static void blk_mq_poll_stats_start(struct request_queue *q)
{
	/*
	 * We don't arm the callback if polling stats are not enabled or the
	 * callback is already active.
	 */
	if (!test_bit(QUEUE_FLAG_POLL_STATS, &q->queue_flags) ||
	    blk_stat_is_active(q->poll_cb))
		return;

	blk_stat_activate_msecs(q->poll_cb, 100);
}

static void blk_mq_poll_stats_fn(struct blk_stat_callback *cb)
{
	struct request_queue *q = cb->data;
2765
	int bucket;
2766

2767 2768 2769 2770
	for (bucket = 0; bucket < BLK_MQ_POLL_STATS_BKTS; bucket++) {
		if (cb->stat[bucket].nr_samples)
			q->poll_stat[bucket] = cb->stat[bucket];
	}
2771 2772
}

2773 2774 2775 2776 2777
static unsigned long blk_mq_poll_nsecs(struct request_queue *q,
				       struct blk_mq_hw_ctx *hctx,
				       struct request *rq)
{
	unsigned long ret = 0;
2778
	int bucket;
2779 2780 2781 2782 2783

	/*
	 * If stats collection isn't on, don't sleep but turn it on for
	 * future users
	 */
2784
	if (!blk_poll_stats_enable(q))
2785 2786 2787 2788 2789 2790 2791 2792
		return 0;

	/*
	 * As an optimistic guess, use half of the mean service time
	 * for this type of request. We can (and should) make this smarter.
	 * For instance, if the completion latencies are tight, we can
	 * get closer than just half the mean. This is especially
	 * important on devices where the completion latencies are longer
2793 2794
	 * than ~10 usec. We do use the stats for the relevant IO size
	 * if available which does lead to better estimates.
2795
	 */
2796 2797 2798 2799 2800 2801
	bucket = blk_mq_poll_stats_bkt(rq);
	if (bucket < 0)
		return ret;

	if (q->poll_stat[bucket].nr_samples)
		ret = (q->poll_stat[bucket].mean + 1) / 2;
2802 2803 2804 2805

	return ret;
}

2806
static bool blk_mq_poll_hybrid_sleep(struct request_queue *q,
2807
				     struct blk_mq_hw_ctx *hctx,
2808 2809 2810 2811
				     struct request *rq)
{
	struct hrtimer_sleeper hs;
	enum hrtimer_mode mode;
2812
	unsigned int nsecs;
2813 2814
	ktime_t kt;

2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825 2826 2827 2828 2829 2830 2831 2832
	if (test_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags))
		return false;

	/*
	 * poll_nsec can be:
	 *
	 * -1:	don't ever hybrid sleep
	 *  0:	use half of prev avg
	 * >0:	use this specific value
	 */
	if (q->poll_nsec == -1)
		return false;
	else if (q->poll_nsec > 0)
		nsecs = q->poll_nsec;
	else
		nsecs = blk_mq_poll_nsecs(q, hctx, rq);

	if (!nsecs)
2833 2834 2835 2836 2837 2838 2839 2840
		return false;

	set_bit(REQ_ATOM_POLL_SLEPT, &rq->atomic_flags);

	/*
	 * This will be replaced with the stats tracking code, using
	 * 'avg_completion_time / 2' as the pre-sleep target.
	 */
T
Thomas Gleixner 已提交
2841
	kt = nsecs;
2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863

	mode = HRTIMER_MODE_REL;
	hrtimer_init_on_stack(&hs.timer, CLOCK_MONOTONIC, mode);
	hrtimer_set_expires(&hs.timer, kt);

	hrtimer_init_sleeper(&hs, current);
	do {
		if (test_bit(REQ_ATOM_COMPLETE, &rq->atomic_flags))
			break;
		set_current_state(TASK_UNINTERRUPTIBLE);
		hrtimer_start_expires(&hs.timer, mode);
		if (hs.task)
			io_schedule();
		hrtimer_cancel(&hs.timer);
		mode = HRTIMER_MODE_ABS;
	} while (hs.task && !signal_pending(current));

	__set_current_state(TASK_RUNNING);
	destroy_hrtimer_on_stack(&hs.timer);
	return true;
}

J
Jens Axboe 已提交
2864 2865 2866 2867 2868
static bool __blk_mq_poll(struct blk_mq_hw_ctx *hctx, struct request *rq)
{
	struct request_queue *q = hctx->queue;
	long state;

2869 2870 2871 2872 2873 2874 2875
	/*
	 * If we sleep, have the caller restart the poll loop to reset
	 * the state. Like for the other success return cases, the
	 * caller is responsible for checking if the IO completed. If
	 * the IO isn't complete, we'll get called again and will go
	 * straight to the busy poll loop.
	 */
2876
	if (blk_mq_poll_hybrid_sleep(q, hctx, rq))
2877 2878
		return true;

J
Jens Axboe 已提交
2879 2880 2881 2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906
	hctx->poll_considered++;

	state = current->state;
	while (!need_resched()) {
		int ret;

		hctx->poll_invoked++;

		ret = q->mq_ops->poll(hctx, rq->tag);
		if (ret > 0) {
			hctx->poll_success++;
			set_current_state(TASK_RUNNING);
			return true;
		}

		if (signal_pending_state(state, current))
			set_current_state(TASK_RUNNING);

		if (current->state == TASK_RUNNING)
			return true;
		if (ret < 0)
			break;
		cpu_relax();
	}

	return false;
}

2907
static bool blk_mq_poll(struct request_queue *q, blk_qc_t cookie)
J
Jens Axboe 已提交
2908 2909 2910 2911
{
	struct blk_mq_hw_ctx *hctx;
	struct request *rq;

2912
	if (!test_bit(QUEUE_FLAG_POLL, &q->queue_flags))
J
Jens Axboe 已提交
2913 2914 2915
		return false;

	hctx = q->queue_hw_ctx[blk_qc_t_to_queue_num(cookie)];
2916 2917
	if (!blk_qc_t_is_internal(cookie))
		rq = blk_mq_tag_to_rq(hctx->tags, blk_qc_t_to_tag(cookie));
2918
	else {
2919
		rq = blk_mq_tag_to_rq(hctx->sched_tags, blk_qc_t_to_tag(cookie));
2920 2921 2922 2923 2924 2925 2926 2927 2928
		/*
		 * With scheduling, if the request has completed, we'll
		 * get a NULL return here, as we clear the sched tag when
		 * that happens. The request still remains valid, like always,
		 * so we should be safe with just the NULL check.
		 */
		if (!rq)
			return false;
	}
J
Jens Axboe 已提交
2929 2930 2931 2932

	return __blk_mq_poll(hctx, rq);
}

2933 2934
static int __init blk_mq_init(void)
{
2935 2936 2937 2938 2939 2940
	/*
	 * See comment in block/blk.h rq_atomic_flags enum
	 */
	BUILD_BUG_ON((REQ_ATOM_STARTED / BITS_PER_BYTE) !=
			(REQ_ATOM_COMPLETE / BITS_PER_BYTE));

2941 2942
	cpuhp_setup_state_multi(CPUHP_BLK_MQ_DEAD, "block/mq:dead", NULL,
				blk_mq_hctx_notify_dead);
2943 2944 2945
	return 0;
}
subsys_initcall(blk_mq_init);