pci.c 60.5 KB
Newer Older
M
Matthew Wilcox 已提交
1 2
/*
 * NVM Express device driver
3
 * Copyright (c) 2011-2014, Intel Corporation.
M
Matthew Wilcox 已提交
4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

15
#include <linux/bitops.h>
M
Matthew Wilcox 已提交
16
#include <linux/blkdev.h>
M
Matias Bjørling 已提交
17
#include <linux/blk-mq.h>
K
Keith Busch 已提交
18
#include <linux/cpu.h>
19
#include <linux/delay.h>
M
Matthew Wilcox 已提交
20 21 22
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/genhd.h>
K
Keith Busch 已提交
23
#include <linux/hdreg.h>
24
#include <linux/idr.h>
M
Matthew Wilcox 已提交
25 26 27 28
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kdev_t.h>
29
#include <linux/kthread.h>
M
Matthew Wilcox 已提交
30 31 32 33
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
34
#include <linux/mutex.h>
M
Matthew Wilcox 已提交
35
#include <linux/pci.h>
36
#include <linux/poison.h>
37
#include <linux/ptrace.h>
M
Matthew Wilcox 已提交
38 39
#include <linux/sched.h>
#include <linux/slab.h>
K
Keith Busch 已提交
40
#include <linux/t10-pi.h>
M
Matthew Wilcox 已提交
41
#include <linux/types.h>
42
#include <linux/io-64-nonatomic-lo-hi.h>
K
Keith Busch 已提交
43
#include <asm/unaligned.h>
44

45 46
#include "nvme.h"

47
#define NVME_Q_DEPTH		1024
J
Jens Axboe 已提交
48
#define NVME_AQ_DEPTH		256
M
Matthew Wilcox 已提交
49 50
#define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))
#define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))
51

52
unsigned char admin_timeout = 60;
53 54
module_param(admin_timeout, byte, 0644);
MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
M
Matthew Wilcox 已提交
55

56 57
unsigned char nvme_io_timeout = 30;
module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
58
MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
M
Matthew Wilcox 已提交
59

60
unsigned char shutdown_timeout = 5;
61 62 63
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");

64 65 66
static int use_threaded_interrupts;
module_param(use_threaded_interrupts, int, 0);

67 68 69 70
static bool use_cmb_sqes = true;
module_param(use_cmb_sqes, bool, 0644);
MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");

71 72
static LIST_HEAD(dev_list);
static struct task_struct *nvme_thread;
K
Keith Busch 已提交
73
static struct workqueue_struct *nvme_workq;
74
static wait_queue_head_t nvme_kthread_wait;
75

76 77
struct nvme_dev;
struct nvme_queue;
78
struct nvme_iod;
79

80
static int nvme_reset(struct nvme_dev *dev);
J
Jens Axboe 已提交
81
static void nvme_process_cq(struct nvme_queue *nvmeq);
82
static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_iod *iod);
83
static void nvme_remove_dead_ctrl(struct nvme_dev *dev);
84
static void nvme_dev_shutdown(struct nvme_dev *dev);
85

K
Keith Busch 已提交
86 87 88 89 90 91
struct async_cmd_info {
	struct kthread_work work;
	struct kthread_worker *worker;
	int status;
	void *ctx;
};
92

93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113
/*
 * Represents an NVM Express device.  Each nvme_dev is a PCI function.
 */
struct nvme_dev {
	struct list_head node;
	struct nvme_queue **queues;
	struct blk_mq_tag_set tagset;
	struct blk_mq_tag_set admin_tagset;
	u32 __iomem *dbs;
	struct device *dev;
	struct dma_pool *prp_page_pool;
	struct dma_pool *prp_small_pool;
	unsigned queue_count;
	unsigned online_queues;
	unsigned max_qid;
	int q_depth;
	u32 db_stride;
	struct msix_entry *entry;
	void __iomem *bar;
	struct work_struct reset_work;
	struct work_struct scan_work;
114
	struct work_struct remove_work;
115
	struct mutex shutdown_lock;
116 117 118 119 120
	bool subsystem;
	void __iomem *cmb;
	dma_addr_t cmb_dma_addr;
	u64 cmb_size;
	u32 cmbsz;
121 122
	unsigned long flags;
#define NVME_CTRL_RESETTING    0
123 124 125 126 127 128 129 130 131

	struct nvme_ctrl ctrl;
};

static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
{
	return container_of(ctrl, struct nvme_dev, ctrl);
}

M
Matthew Wilcox 已提交
132 133 134 135 136 137
/*
 * An NVM Express queue.  Each device has at least two (one for admin
 * commands and one for I/O commands).
 */
struct nvme_queue {
	struct device *q_dmadev;
M
Matthew Wilcox 已提交
138
	struct nvme_dev *dev;
139
	char irqname[24];	/* nvme4294967295-65535\0 */
M
Matthew Wilcox 已提交
140 141
	spinlock_t q_lock;
	struct nvme_command *sq_cmds;
142
	struct nvme_command __iomem *sq_cmds_io;
M
Matthew Wilcox 已提交
143
	volatile struct nvme_completion *cqes;
144
	struct blk_mq_tags **tags;
M
Matthew Wilcox 已提交
145 146 147 148
	dma_addr_t sq_dma_addr;
	dma_addr_t cq_dma_addr;
	u32 __iomem *q_db;
	u16 q_depth;
J
Jens Axboe 已提交
149
	s16 cq_vector;
M
Matthew Wilcox 已提交
150 151 152
	u16 sq_head;
	u16 sq_tail;
	u16 cq_head;
K
Keith Busch 已提交
153
	u16 qid;
154 155
	u8 cq_phase;
	u8 cqe_seen;
K
Keith Busch 已提交
156
	struct async_cmd_info cmdinfo;
M
Matthew Wilcox 已提交
157 158
};

159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
/*
 * The nvme_iod describes the data in an I/O, including the list of PRP
 * entries.  You can't see it in this data structure because C doesn't let
 * me express that.  Use nvme_alloc_iod to ensure there's enough space
 * allocated to store the PRP list.
 */
struct nvme_iod {
	unsigned long private;	/* For the use of the submitter of the I/O */
	int npages;		/* In the PRP list. 0 means small pool in use */
	int offset;		/* Of PRP list */
	int nents;		/* Used in scatterlist */
	int length;		/* Of data, in bytes */
	dma_addr_t first_dma;
	struct scatterlist meta_sg[1]; /* metadata requires single contiguous buffer */
	struct scatterlist sg[0];
};

M
Matthew Wilcox 已提交
176 177 178 179 180 181 182 183 184 185
/*
 * Check we didin't inadvertently grow the command struct
 */
static inline void _nvme_check_size(void)
{
	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
186
	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
K
Keith Busch 已提交
187
	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
M
Matthew Wilcox 已提交
188 189 190 191
	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
K
Keith Busch 已提交
192
	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
M
Matthew Wilcox 已提交
193 194
}

195
typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
196 197
						struct nvme_completion *);

198
struct nvme_cmd_info {
199 200
	nvme_completion_fn fn;
	void *ctx;
K
Keith Busch 已提交
201
	int aborted;
M
Matias Bjørling 已提交
202
	struct nvme_queue *nvmeq;
203
	struct nvme_iod iod[0];
204 205
};

206 207 208 209
/*
 * Max size of iod being embedded in the request payload
 */
#define NVME_INT_PAGES		2
210
#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->ctrl.page_size)
C
Chong Yuan 已提交
211
#define NVME_INT_MASK		0x01
212 213 214 215 216 217 218 219

/*
 * Will slightly overestimate the number of pages needed.  This is OK
 * as it only leads to a small amount of wasted memory for the lifetime of
 * the I/O.
 */
static int nvme_npages(unsigned size, struct nvme_dev *dev)
{
220 221
	unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
				      dev->ctrl.page_size);
222 223 224 225 226 227 228 229 230 231 232 233 234 235
	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}

static unsigned int nvme_cmd_size(struct nvme_dev *dev)
{
	unsigned int ret = sizeof(struct nvme_cmd_info);

	ret += sizeof(struct nvme_iod);
	ret += sizeof(__le64 *) * nvme_npages(NVME_INT_BYTES(dev), dev);
	ret += sizeof(struct scatterlist) * NVME_INT_PAGES;

	return ret;
}

M
Matias Bjørling 已提交
236 237
static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
				unsigned int hctx_idx)
238
{
M
Matias Bjørling 已提交
239 240 241
	struct nvme_dev *dev = data;
	struct nvme_queue *nvmeq = dev->queues[0];

242 243 244 245
	WARN_ON(hctx_idx != 0);
	WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
	WARN_ON(nvmeq->tags);

M
Matias Bjørling 已提交
246
	hctx->driver_data = nvmeq;
247
	nvmeq->tags = &dev->admin_tagset.tags[0];
M
Matias Bjørling 已提交
248
	return 0;
249 250
}

251 252 253 254 255 256 257
static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	nvmeq->tags = NULL;
}

M
Matias Bjørling 已提交
258 259 260
static int nvme_admin_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
261
{
M
Matias Bjørling 已提交
262 263 264 265 266 267 268
	struct nvme_dev *dev = data;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = dev->queues[0];

	BUG_ON(!nvmeq);
	cmd->nvmeq = nvmeq;
	return 0;
269 270
}

M
Matias Bjørling 已提交
271 272
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			  unsigned int hctx_idx)
M
Matthew Wilcox 已提交
273
{
M
Matias Bjørling 已提交
274
	struct nvme_dev *dev = data;
275
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
M
Matias Bjørling 已提交
276

277 278
	if (!nvmeq->tags)
		nvmeq->tags = &dev->tagset.tags[hctx_idx];
M
Matthew Wilcox 已提交
279

280
	WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
M
Matias Bjørling 已提交
281 282
	hctx->driver_data = nvmeq;
	return 0;
M
Matthew Wilcox 已提交
283 284
}

M
Matias Bjørling 已提交
285 286 287
static int nvme_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
M
Matthew Wilcox 已提交
288
{
M
Matias Bjørling 已提交
289 290 291 292 293 294 295 296 297 298 299 300 301 302 303
	struct nvme_dev *dev = data;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];

	BUG_ON(!nvmeq);
	cmd->nvmeq = nvmeq;
	return 0;
}

static void nvme_set_info(struct nvme_cmd_info *cmd, void *ctx,
				nvme_completion_fn handler)
{
	cmd->fn = handler;
	cmd->ctx = ctx;
	cmd->aborted = 0;
K
Keith Busch 已提交
304
	blk_mq_start_request(blk_mq_rq_from_pdu(cmd));
M
Matthew Wilcox 已提交
305 306
}

307 308 309 310 311 312 313 314 315 316
static void *iod_get_private(struct nvme_iod *iod)
{
	return (void *) (iod->private & ~0x1UL);
}

/*
 * If bit 0 is set, the iod is embedded in the request payload.
 */
static bool iod_should_kfree(struct nvme_iod *iod)
{
C
Chong Yuan 已提交
317
	return (iod->private & NVME_INT_MASK) == 0;
318 319
}

320 321
/* Special values must be less than 0x1000 */
#define CMD_CTX_BASE		((void *)POISON_POINTER_DELTA)
322 323 324
#define CMD_CTX_CANCELLED	(0x30C + CMD_CTX_BASE)
#define CMD_CTX_COMPLETED	(0x310 + CMD_CTX_BASE)
#define CMD_CTX_INVALID		(0x314 + CMD_CTX_BASE)
325

326
static void special_completion(struct nvme_queue *nvmeq, void *ctx,
327 328 329 330 331
						struct nvme_completion *cqe)
{
	if (ctx == CMD_CTX_CANCELLED)
		return;
	if (ctx == CMD_CTX_COMPLETED) {
332
		dev_warn(nvmeq->q_dmadev,
333 334 335 336 337
				"completed id %d twice on queue %d\n",
				cqe->command_id, le16_to_cpup(&cqe->sq_id));
		return;
	}
	if (ctx == CMD_CTX_INVALID) {
338
		dev_warn(nvmeq->q_dmadev,
339 340 341 342
				"invalid id %d completed on queue %d\n",
				cqe->command_id, le16_to_cpup(&cqe->sq_id));
		return;
	}
343
	dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
344 345
}

M
Matias Bjørling 已提交
346
static void *cancel_cmd_info(struct nvme_cmd_info *cmd, nvme_completion_fn *fn)
M
Matthew Wilcox 已提交
347
{
348
	void *ctx;
M
Matthew Wilcox 已提交
349

350
	if (fn)
M
Matias Bjørling 已提交
351 352 353 354
		*fn = cmd->fn;
	ctx = cmd->ctx;
	cmd->fn = special_completion;
	cmd->ctx = CMD_CTX_CANCELLED;
355
	return ctx;
M
Matthew Wilcox 已提交
356 357
}

M
Matias Bjørling 已提交
358 359
static void async_req_completion(struct nvme_queue *nvmeq, void *ctx,
						struct nvme_completion *cqe)
360
{
M
Matias Bjørling 已提交
361 362 363 364
	u32 result = le32_to_cpup(&cqe->result);
	u16 status = le16_to_cpup(&cqe->status) >> 1;

	if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
365
		++nvmeq->dev->ctrl.event_limit;
K
Keith Busch 已提交
366 367 368 369 370 371
	if (status != NVME_SC_SUCCESS)
		return;

	switch (result & 0xff07) {
	case NVME_AER_NOTICE_NS_CHANGED:
		dev_info(nvmeq->q_dmadev, "rescanning\n");
372
		queue_work(nvme_workq, &nvmeq->dev->scan_work);
K
Keith Busch 已提交
373 374 375
	default:
		dev_warn(nvmeq->q_dmadev, "async event result %08x\n", result);
	}
M
Matthew Wilcox 已提交
376 377
}

M
Matias Bjørling 已提交
378 379
static inline struct nvme_cmd_info *get_cmd_from_tag(struct nvme_queue *nvmeq,
				  unsigned int tag)
M
Matthew Wilcox 已提交
380
{
381
	struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, tag);
382

M
Matias Bjørling 已提交
383
	return blk_mq_rq_to_pdu(req);
384 385
}

M
Matias Bjørling 已提交
386 387 388 389 390
/*
 * Called with local interrupts disabled and the q_lock held.  May not sleep.
 */
static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag,
						nvme_completion_fn *fn)
391
{
M
Matias Bjørling 已提交
392 393 394 395 396 397 398 399 400 401 402 403
	struct nvme_cmd_info *cmd = get_cmd_from_tag(nvmeq, tag);
	void *ctx;
	if (tag >= nvmeq->q_depth) {
		*fn = special_completion;
		return CMD_CTX_INVALID;
	}
	if (fn)
		*fn = cmd->fn;
	ctx = cmd->ctx;
	cmd->fn = special_completion;
	cmd->ctx = CMD_CTX_COMPLETED;
	return ctx;
M
Matthew Wilcox 已提交
404 405 406
}

/**
407
 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
M
Matthew Wilcox 已提交
408 409 410 411 412
 * @nvmeq: The queue to use
 * @cmd: The command to send
 *
 * Safe to use from interrupt context
 */
413 414
static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
						struct nvme_command *cmd)
M
Matthew Wilcox 已提交
415
{
M
Matias Bjørling 已提交
416 417
	u16 tail = nvmeq->sq_tail;

418 419 420 421 422
	if (nvmeq->sq_cmds_io)
		memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd));
	else
		memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));

M
Matthew Wilcox 已提交
423 424
	if (++tail == nvmeq->q_depth)
		tail = 0;
425
	writel(tail, nvmeq->q_db);
M
Matthew Wilcox 已提交
426 427 428
	nvmeq->sq_tail = tail;
}

429
static __le64 **iod_list(struct nvme_iod *iod)
430
{
431
	return ((void *)iod) + iod->offset;
432 433
}

434 435
static inline void iod_init(struct nvme_iod *iod, unsigned nbytes,
			    unsigned nseg, unsigned long private)
436
{
437 438 439 440 441
	iod->private = private;
	iod->offset = offsetof(struct nvme_iod, sg[nseg]);
	iod->npages = -1;
	iod->length = nbytes;
	iod->nents = 0;
442
}
M
Matthew Wilcox 已提交
443

444
static struct nvme_iod *
445 446
__nvme_alloc_iod(unsigned nseg, unsigned bytes, struct nvme_dev *dev,
		 unsigned long priv, gfp_t gfp)
M
Matthew Wilcox 已提交
447
{
448
	struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
449
				sizeof(__le64 *) * nvme_npages(bytes, dev) +
450 451
				sizeof(struct scatterlist) * nseg, gfp);

452 453
	if (iod)
		iod_init(iod, bytes, nseg, priv);
454 455

	return iod;
M
Matthew Wilcox 已提交
456 457
}

458 459 460 461 462 463 464 465 466 467 468 469 470
static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
			               gfp_t gfp)
{
	unsigned size = !(rq->cmd_flags & REQ_DISCARD) ? blk_rq_bytes(rq) :
                                                sizeof(struct nvme_dsm_range);
	struct nvme_iod *iod;

	if (rq->nr_phys_segments <= NVME_INT_PAGES &&
	    size <= NVME_INT_BYTES(dev)) {
		struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);

		iod = cmd->iod;
		iod_init(iod, size, rq->nr_phys_segments,
C
Chong Yuan 已提交
471
				(unsigned long) rq | NVME_INT_MASK);
472 473 474 475 476 477 478
		return iod;
	}

	return __nvme_alloc_iod(rq->nr_phys_segments, size, dev,
				(unsigned long) rq, gfp);
}

479
static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
M
Matthew Wilcox 已提交
480
{
481
	const int last_prp = dev->ctrl.page_size / 8 - 1;
482 483 484 485 486 487 488 489 490 491 492 493
	int i;
	__le64 **list = iod_list(iod);
	dma_addr_t prp_dma = iod->first_dma;

	if (iod->npages == 0)
		dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
	for (i = 0; i < iod->npages; i++) {
		__le64 *prp_list = list[i];
		dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
		dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
		prp_dma = next_prp_dma;
	}
494 495 496

	if (iod_should_kfree(iod))
		kfree(iod);
M
Matthew Wilcox 已提交
497 498
}

499
#ifdef CONFIG_BLK_DEV_INTEGRITY
K
Keith Busch 已提交
500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == v)
		pi->ref_tag = cpu_to_be32(p);
}

static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == p)
		pi->ref_tag = cpu_to_be32(v);
}

/**
 * nvme_dif_remap - remaps ref tags to bip seed and physical lba
 *
 * The virtual start sector is the one that was originally submitted by the
 * block layer.	Due to partitioning, MD/DM cloning, etc. the actual physical
 * start sector may be different. Remap protection information to match the
 * physical LBA on writes, and back to the original seed on reads.
 *
 * Type 0 and 3 do not have a ref tag, so no remapping required.
 */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
	struct nvme_ns *ns = req->rq_disk->private_data;
	struct bio_integrity_payload *bip;
	struct t10_pi_tuple *pi;
	void *p, *pmap;
	u32 i, nlb, ts, phys, virt;

	if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
		return;

	bip = bio_integrity(req->bio);
	if (!bip)
		return;

	pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;

	p = pmap;
	virt = bip_get_seed(bip);
	phys = nvme_block_nr(ns, blk_rq_pos(req));
	nlb = (blk_rq_bytes(req) >> ns->lba_shift);
544
	ts = ns->disk->queue->integrity.tuple_size;
K
Keith Busch 已提交
545 546 547 548 549 550 551 552

	for (i = 0; i < nlb; i++, virt++, phys++) {
		pi = (struct t10_pi_tuple *)p;
		dif_swap(phys, virt, pi);
		p += ts;
	}
	kunmap_atomic(pmap);
}
553 554 555 556 557 558 559 560 561 562 563 564 565
#else /* CONFIG_BLK_DEV_INTEGRITY */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
}
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
#endif

M
Matias Bjørling 已提交
566
static void req_completion(struct nvme_queue *nvmeq, void *ctx,
M
Matthew Wilcox 已提交
567 568
						struct nvme_completion *cqe)
{
569
	struct nvme_iod *iod = ctx;
570
	struct request *req = iod_get_private(iod);
M
Matias Bjørling 已提交
571
	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
M
Matthew Wilcox 已提交
572
	u16 status = le16_to_cpup(&cqe->status) >> 1;
573
	int error = 0;
M
Matthew Wilcox 已提交
574

575
	if (unlikely(status)) {
576
		if (nvme_req_needs_retry(req, status)) {
577
			nvme_unmap_data(nvmeq->dev, iod);
578
			nvme_requeue_req(req);
579
			return;
580
		}
581

582
		if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
583
			if (cmd_rq->ctx == CMD_CTX_CANCELLED)
C
Christoph Hellwig 已提交
584
				error = NVME_SC_CANCELLED;
585 586
			else
				error = status;
587
		} else {
588
			error = nvme_error_status(status);
589
		}
590 591
	}

K
Keith Busch 已提交
592 593 594 595
	if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
		u32 result = le32_to_cpup(&cqe->result);
		req->special = (void *)(uintptr_t)result;
	}
M
Matias Bjørling 已提交
596 597

	if (cmd_rq->aborted)
598
		dev_warn(nvmeq->dev->dev,
M
Matias Bjørling 已提交
599
			"completing aborted command with status:%04x\n",
600
			error);
M
Matias Bjørling 已提交
601

602 603
	nvme_unmap_data(nvmeq->dev, iod);
	blk_mq_complete_request(req, error);
M
Matthew Wilcox 已提交
604 605
}

606 607
static bool nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
		int total_len)
M
Matthew Wilcox 已提交
608
{
609
	struct dma_pool *pool;
610 611
	int length = total_len;
	struct scatterlist *sg = iod->sg;
M
Matthew Wilcox 已提交
612 613
	int dma_len = sg_dma_len(sg);
	u64 dma_addr = sg_dma_address(sg);
614
	u32 page_size = dev->ctrl.page_size;
615
	int offset = dma_addr & (page_size - 1);
616
	__le64 *prp_list;
617
	__le64 **list = iod_list(iod);
618
	dma_addr_t prp_dma;
619
	int nprps, i;
M
Matthew Wilcox 已提交
620

621
	length -= (page_size - offset);
M
Matthew Wilcox 已提交
622
	if (length <= 0)
623
		return true;
M
Matthew Wilcox 已提交
624

625
	dma_len -= (page_size - offset);
M
Matthew Wilcox 已提交
626
	if (dma_len) {
627
		dma_addr += (page_size - offset);
M
Matthew Wilcox 已提交
628 629 630 631 632 633
	} else {
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
	}

634
	if (length <= page_size) {
635
		iod->first_dma = dma_addr;
636
		return true;
637 638
	}

639
	nprps = DIV_ROUND_UP(length, page_size);
640 641
	if (nprps <= (256 / 8)) {
		pool = dev->prp_small_pool;
642
		iod->npages = 0;
643 644
	} else {
		pool = dev->prp_page_pool;
645
		iod->npages = 1;
646 647
	}

648
	prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
649
	if (!prp_list) {
650
		iod->first_dma = dma_addr;
651
		iod->npages = -1;
652
		return false;
653
	}
654 655
	list[0] = prp_list;
	iod->first_dma = prp_dma;
656 657
	i = 0;
	for (;;) {
658
		if (i == page_size >> 3) {
659
			__le64 *old_prp_list = prp_list;
660
			prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
661
			if (!prp_list)
662
				return false;
663
			list[iod->npages++] = prp_list;
664 665 666
			prp_list[0] = old_prp_list[i - 1];
			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
			i = 1;
667 668
		}
		prp_list[i++] = cpu_to_le64(dma_addr);
669 670 671
		dma_len -= page_size;
		dma_addr += page_size;
		length -= page_size;
672 673 674 675 676 677 678 679
		if (length <= 0)
			break;
		if (dma_len > 0)
			continue;
		BUG_ON(dma_len < 0);
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
M
Matthew Wilcox 已提交
680 681
	}

682
	return true;
M
Matthew Wilcox 已提交
683 684
}

C
Christoph Hellwig 已提交
685 686
static int nvme_map_data(struct nvme_dev *dev, struct nvme_iod *iod,
		struct nvme_command *cmnd)
687
{
C
Christoph Hellwig 已提交
688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713
	struct request *req = iod_get_private(iod);
	struct request_queue *q = req->q;
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;
	int ret = BLK_MQ_RQ_QUEUE_ERROR;

	sg_init_table(iod->sg, req->nr_phys_segments);
	iod->nents = blk_rq_map_sg(q, req, iod->sg);
	if (!iod->nents)
		goto out;

	ret = BLK_MQ_RQ_QUEUE_BUSY;
	if (!dma_map_sg(dev->dev, iod->sg, iod->nents, dma_dir))
		goto out;

	if (!nvme_setup_prps(dev, iod, blk_rq_bytes(req)))
		goto out_unmap;

	ret = BLK_MQ_RQ_QUEUE_ERROR;
	if (blk_integrity_rq(req)) {
		if (blk_rq_count_integrity_sg(q, req->bio) != 1)
			goto out_unmap;

		sg_init_table(iod->meta_sg, 1);
		if (blk_rq_map_integrity_sg(q, req->bio, iod->meta_sg) != 1)
			goto out_unmap;
714

C
Christoph Hellwig 已提交
715 716 717 718 719
		if (rq_data_dir(req))
			nvme_dif_remap(req, nvme_dif_prep);

		if (!dma_map_sg(dev->dev, iod->meta_sg, 1, dma_dir))
			goto out_unmap;
720 721
	}

C
Christoph Hellwig 已提交
722 723 724 725 726 727 728 729 730 731
	cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
	cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);
	if (blk_integrity_rq(req))
		cmnd->rw.metadata = cpu_to_le64(sg_dma_address(iod->meta_sg));
	return BLK_MQ_RQ_QUEUE_OK;

out_unmap:
	dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
out:
	return ret;
732 733
}

734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751
static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_iod *iod)
{
	struct request *req = iod_get_private(iod);
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;

	if (iod->nents) {
		dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
		if (blk_integrity_rq(req)) {
			if (!rq_data_dir(req))
				nvme_dif_remap(req, nvme_dif_complete);
			dma_unmap_sg(dev->dev, iod->meta_sg, 1, dma_dir);
		}
	}

	nvme_free_iod(dev, iod);
}

M
Matias Bjørling 已提交
752 753 754 755 756
/*
 * We reuse the small pool to allocate the 16-byte range here as it is not
 * worth having a special pool for these or additional cases to handle freeing
 * the iod.
 */
C
Christoph Hellwig 已提交
757 758
static int nvme_setup_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
		struct nvme_iod *iod, struct nvme_command *cmnd)
759
{
C
Christoph Hellwig 已提交
760 761 762 763 764 765 766 767 768
	struct request *req = iod_get_private(iod);
	struct nvme_dsm_range *range;

	range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
						&iod->first_dma);
	if (!range)
		return BLK_MQ_RQ_QUEUE_BUSY;
	iod_list(iod)[0] = (__le64 *)range;
	iod->npages = 0;
769 770

	range->cattr = cpu_to_le32(0);
M
Matias Bjørling 已提交
771 772
	range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
	range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
773

C
Christoph Hellwig 已提交
774 775 776 777 778 779 780
	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->dsm.opcode = nvme_cmd_dsm;
	cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
	cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
	cmnd->dsm.nr = 0;
	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
	return BLK_MQ_RQ_QUEUE_OK;
781 782
}

783 784 785
/*
 * NOTE: ns is NULL when called on the admin queue.
 */
M
Matias Bjørling 已提交
786 787
static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
			 const struct blk_mq_queue_data *bd)
788
{
M
Matias Bjørling 已提交
789 790
	struct nvme_ns *ns = hctx->queue->queuedata;
	struct nvme_queue *nvmeq = hctx->driver_data;
791
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
792 793
	struct request *req = bd->rq;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
794
	struct nvme_iod *iod;
C
Christoph Hellwig 已提交
795 796
	struct nvme_command cmnd;
	int ret = BLK_MQ_RQ_QUEUE_OK;
797

K
Keith Busch 已提交
798 799 800 801 802
	/*
	 * If formated with metadata, require the block layer provide a buffer
	 * unless this namespace is formated such that the metadata can be
	 * stripped/generated by the controller with PRACT=1.
	 */
803
	if (ns && ns->ms && !blk_integrity_rq(req)) {
804 805
		if (!(ns->pi_type && ns->ms == 8) &&
					req->cmd_type != REQ_TYPE_DRV_PRIV) {
806
			blk_mq_complete_request(req, -EFAULT);
K
Keith Busch 已提交
807 808 809 810
			return BLK_MQ_RQ_QUEUE_OK;
		}
	}

811
	iod = nvme_alloc_iod(req, dev, GFP_ATOMIC);
812
	if (!iod)
813
		return BLK_MQ_RQ_QUEUE_BUSY;
M
Matias Bjørling 已提交
814 815

	if (req->cmd_flags & REQ_DISCARD) {
C
Christoph Hellwig 已提交
816 817 818 819 820 821 822 823
		ret = nvme_setup_discard(nvmeq, ns, iod, &cmnd);
	} else {
		if (req->cmd_type == REQ_TYPE_DRV_PRIV)
			memcpy(&cmnd, req->cmd, sizeof(cmnd));
		else if (req->cmd_flags & REQ_FLUSH)
			nvme_setup_flush(ns, &cmnd);
		else
			nvme_setup_rw(ns, req, &cmnd);
M
Matias Bjørling 已提交
824

C
Christoph Hellwig 已提交
825 826
		if (req->nr_phys_segments)
			ret = nvme_map_data(dev, iod, &cmnd);
827
	}
828

C
Christoph Hellwig 已提交
829 830 831 832
	if (ret)
		goto out;

	cmnd.common.command_id = req->tag;
K
Keith Busch 已提交
833
	nvme_set_info(cmd, iod, req_completion);
M
Matias Bjørling 已提交
834

C
Christoph Hellwig 已提交
835 836
	spin_lock_irq(&nvmeq->q_lock);
	__nvme_submit_cmd(nvmeq, &cmnd);
M
Matias Bjørling 已提交
837 838 839
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
	return BLK_MQ_RQ_QUEUE_OK;
C
Christoph Hellwig 已提交
840
out:
841
	nvme_free_iod(dev, iod);
C
Christoph Hellwig 已提交
842
	return ret;
M
Matthew Wilcox 已提交
843 844
}

J
Jens Axboe 已提交
845
static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
M
Matthew Wilcox 已提交
846
{
M
Matthew Wilcox 已提交
847
	u16 head, phase;
M
Matthew Wilcox 已提交
848 849

	head = nvmeq->cq_head;
M
Matthew Wilcox 已提交
850
	phase = nvmeq->cq_phase;
M
Matthew Wilcox 已提交
851 852

	for (;;) {
853 854
		void *ctx;
		nvme_completion_fn fn;
M
Matthew Wilcox 已提交
855
		struct nvme_completion cqe = nvmeq->cqes[head];
M
Matthew Wilcox 已提交
856
		if ((le16_to_cpu(cqe.status) & 1) != phase)
M
Matthew Wilcox 已提交
857 858 859 860
			break;
		nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
		if (++head == nvmeq->q_depth) {
			head = 0;
M
Matthew Wilcox 已提交
861
			phase = !phase;
M
Matthew Wilcox 已提交
862
		}
J
Jens Axboe 已提交
863 864
		if (tag && *tag == cqe.command_id)
			*tag = -1;
M
Matias Bjørling 已提交
865
		ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
866
		fn(nvmeq, ctx, &cqe);
M
Matthew Wilcox 已提交
867 868 869 870 871 872 873 874
	}

	/* If the controller ignores the cq head doorbell and continuously
	 * writes to the queue, it is theoretically possible to wrap around
	 * the queue twice and mistakenly return IRQ_NONE.  Linux only
	 * requires that 0.1% of your interrupts are handled, so this isn't
	 * a big problem.
	 */
M
Matthew Wilcox 已提交
875
	if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
J
Jens Axboe 已提交
876
		return;
M
Matthew Wilcox 已提交
877

878 879
	if (likely(nvmeq->cq_vector >= 0))
		writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
M
Matthew Wilcox 已提交
880
	nvmeq->cq_head = head;
M
Matthew Wilcox 已提交
881
	nvmeq->cq_phase = phase;
M
Matthew Wilcox 已提交
882

883
	nvmeq->cqe_seen = 1;
J
Jens Axboe 已提交
884 885 886 887 888
}

static void nvme_process_cq(struct nvme_queue *nvmeq)
{
	__nvme_process_cq(nvmeq, NULL);
M
Matthew Wilcox 已提交
889 890 891
}

static irqreturn_t nvme_irq(int irq, void *data)
892 893 894 895
{
	irqreturn_t result;
	struct nvme_queue *nvmeq = data;
	spin_lock(&nvmeq->q_lock);
896 897 898
	nvme_process_cq(nvmeq);
	result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
	nvmeq->cqe_seen = 0;
899 900 901 902 903 904 905 906 907 908 909 910 911
	spin_unlock(&nvmeq->q_lock);
	return result;
}

static irqreturn_t nvme_irq_check(int irq, void *data)
{
	struct nvme_queue *nvmeq = data;
	struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
	if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
		return IRQ_NONE;
	return IRQ_WAKE_THREAD;
}

J
Jens Axboe 已提交
912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928
static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	if ((le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
	    nvmeq->cq_phase) {
		spin_lock_irq(&nvmeq->q_lock);
		__nvme_process_cq(nvmeq, &tag);
		spin_unlock_irq(&nvmeq->q_lock);

		if (tag == -1)
			return 1;
	}

	return 0;
}

M
Matias Bjørling 已提交
929 930 931 932 933 934 935
static int nvme_submit_async_admin_req(struct nvme_dev *dev)
{
	struct nvme_queue *nvmeq = dev->queues[0];
	struct nvme_command c;
	struct nvme_cmd_info *cmd_info;
	struct request *req;

936
	req = blk_mq_alloc_request(dev->ctrl.admin_q, WRITE,
937
			BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED);
938 939
	if (IS_ERR(req))
		return PTR_ERR(req);
M
Matias Bjørling 已提交
940

K
Keith Busch 已提交
941
	req->cmd_flags |= REQ_NO_TIMEOUT;
M
Matias Bjørling 已提交
942
	cmd_info = blk_mq_rq_to_pdu(req);
943
	nvme_set_info(cmd_info, NULL, async_req_completion);
M
Matias Bjørling 已提交
944 945 946 947 948

	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_async_event;
	c.common.command_id = req->tag;

949
	blk_mq_free_request(req);
950 951
	__nvme_submit_cmd(nvmeq, &c);
	return 0;
M
Matias Bjørling 已提交
952 953
}

954
static void async_cmd_info_endio(struct request *req, int error)
K
Keith Busch 已提交
955
{
956
	struct async_cmd_info *cmdinfo = req->end_io_data;
M
Matias Bjørling 已提交
957

958 959 960
	cmdinfo->status = req->errors;
	queue_kthread_work(cmdinfo->worker, &cmdinfo->work);
	blk_mq_free_request(req);
K
Keith Busch 已提交
961 962
}

M
Matthew Wilcox 已提交
963 964 965 966 967 968 969 970
static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(id);

971
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
972 973 974 975 976 977 978 979
}

static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;

980 981 982 983
	/*
	 * Note: we (ab)use the fact the the prp fields survive if no data
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
984 985 986 987 988 989 990 991
	memset(&c, 0, sizeof(c));
	c.create_cq.opcode = nvme_admin_create_cq;
	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
	c.create_cq.cqid = cpu_to_le16(qid);
	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_cq.cq_flags = cpu_to_le16(flags);
	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);

992
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
993 994 995 996 997 998 999 1000
}

static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;

1001 1002 1003 1004
	/*
	 * Note: we (ab)use the fact the the prp fields survive if no data
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
1005 1006 1007 1008 1009 1010 1011 1012
	memset(&c, 0, sizeof(c));
	c.create_sq.opcode = nvme_admin_create_sq;
	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
	c.create_sq.sqid = cpu_to_le16(qid);
	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_sq.sq_flags = cpu_to_le16(flags);
	c.create_sq.cqid = cpu_to_le16(qid);

1013
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025
}

static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
}

static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
}

1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038
static void abort_endio(struct request *req, int error)
{
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd->nvmeq;
	u32 result = (u32)(uintptr_t)req->special;
	u16 status = req->errors;

	dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
	atomic_inc(&nvmeq->dev->ctrl.abort_limit);

	blk_mq_free_request(req);
}

1039
static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
K
Keith Busch 已提交
1040
{
M
Matias Bjørling 已提交
1041 1042
	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd_rq->nvmeq;
K
Keith Busch 已提交
1043
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
1044 1045
	struct request *abort_req;
	struct nvme_command cmd;
K
Keith Busch 已提交
1046

1047
	/*
1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
	 * Shutdown immediately if controller times out while starting. The
	 * reset work will see the pci device disabled when it gets the forced
	 * cancellation error. All outstanding requests are completed on
	 * shutdown, so we return BLK_EH_HANDLED.
	 */
	if (test_bit(NVME_CTRL_RESETTING, &dev->flags)) {
		dev_warn(dev->dev,
			 "I/O %d QID %d timeout, disable controller\n",
			 req->tag, nvmeq->qid);
		nvme_dev_shutdown(dev);
		req->errors = NVME_SC_CANCELLED;
		return BLK_EH_HANDLED;
	}

	/*
 	 * Shutdown the controller immediately and schedule a reset if the
 	 * command was already aborted once before and still hasn't been
 	 * returned to the driver, or if this is the admin queue.
1066
	 */
M
Matias Bjørling 已提交
1067
	if (!nvmeq->qid || cmd_rq->aborted) {
1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
		dev_warn(dev->dev,
			 "I/O %d QID %d timeout, reset controller\n",
			 req->tag, nvmeq->qid);
		nvme_dev_shutdown(dev);
		queue_work(nvme_workq, &dev->reset_work);

		/*
		 * Mark the request as handled, since the inline shutdown
		 * forces all outstanding requests to complete.
		 */
		req->errors = NVME_SC_CANCELLED;
		return BLK_EH_HANDLED;
K
Keith Busch 已提交
1080 1081
	}

1082
	cmd_rq->aborted = 1;
K
Keith Busch 已提交
1083

1084
	if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1085
		atomic_inc(&dev->ctrl.abort_limit);
1086
		return BLK_EH_RESET_TIMER;
1087
	}
K
Keith Busch 已提交
1088 1089 1090

	memset(&cmd, 0, sizeof(cmd));
	cmd.abort.opcode = nvme_admin_abort_cmd;
M
Matias Bjørling 已提交
1091
	cmd.abort.cid = req->tag;
K
Keith Busch 已提交
1092 1093
	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);

1094 1095
	dev_warn(nvmeq->q_dmadev, "I/O %d QID %d timeout, aborting\n",
				 req->tag, nvmeq->qid);
1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106

	abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
			BLK_MQ_REQ_NOWAIT);
	if (IS_ERR(abort_req)) {
		atomic_inc(&dev->ctrl.abort_limit);
		return BLK_EH_RESET_TIMER;
	}

	abort_req->timeout = ADMIN_TIMEOUT;
	abort_req->end_io_data = NULL;
	blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
1107 1108 1109 1110 1111 1112 1113

	/*
	 * The aborted req will be completed on receiving the abort req.
	 * We enable the timer again. If hit twice, it'll cause a device reset,
	 * as the device then is in a faulty state.
	 */
	return BLK_EH_RESET_TIMER;
K
Keith Busch 已提交
1114 1115
}

1116
static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved)
1117
{
M
Matias Bjørling 已提交
1118 1119 1120 1121
	struct nvme_queue *nvmeq = data;
	void *ctx;
	nvme_completion_fn fn;
	struct nvme_cmd_info *cmd;
K
Keith Busch 已提交
1122 1123 1124 1125
	struct nvme_completion cqe;

	if (!blk_mq_request_started(req))
		return;
1126

M
Matias Bjørling 已提交
1127
	cmd = blk_mq_rq_to_pdu(req);
1128

M
Matias Bjørling 已提交
1129 1130 1131
	if (cmd->ctx == CMD_CTX_CANCELLED)
		return;

K
Keith Busch 已提交
1132 1133 1134 1135 1136 1137
	if (blk_queue_dying(req->q))
		cqe.status = cpu_to_le16((NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1);
	else
		cqe.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);


M
Matias Bjørling 已提交
1138 1139 1140 1141
	dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n",
						req->tag, nvmeq->qid);
	ctx = cancel_cmd_info(cmd, &fn);
	fn(nvmeq, ctx, &cqe);
1142 1143
}

M
Matias Bjørling 已提交
1144 1145
static void nvme_free_queue(struct nvme_queue *nvmeq)
{
1146 1147
	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1148 1149
	if (nvmeq->sq_cmds)
		dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1150 1151 1152 1153
					nvmeq->sq_cmds, nvmeq->sq_dma_addr);
	kfree(nvmeq);
}

1154
static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1155 1156 1157
{
	int i;

1158
	for (i = dev->queue_count - 1; i >= lowest; i--) {
M
Matias Bjørling 已提交
1159
		struct nvme_queue *nvmeq = dev->queues[i];
1160
		dev->queue_count--;
M
Matias Bjørling 已提交
1161
		dev->queues[i] = NULL;
K
Keith Busch 已提交
1162
		nvme_free_queue(nvmeq);
1163
	}
1164 1165
}

K
Keith Busch 已提交
1166 1167 1168 1169 1170
/**
 * nvme_suspend_queue - put queue into suspended state
 * @nvmeq - queue to suspend
 */
static int nvme_suspend_queue(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
1171
{
K
Keith Busch 已提交
1172
	int vector;
M
Matthew Wilcox 已提交
1173

1174
	spin_lock_irq(&nvmeq->q_lock);
K
Keith Busch 已提交
1175 1176 1177 1178 1179
	if (nvmeq->cq_vector == -1) {
		spin_unlock_irq(&nvmeq->q_lock);
		return 1;
	}
	vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
K
Keith Busch 已提交
1180
	nvmeq->dev->online_queues--;
K
Keith Busch 已提交
1181
	nvmeq->cq_vector = -1;
1182 1183
	spin_unlock_irq(&nvmeq->q_lock);

1184 1185
	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
		blk_mq_freeze_queue_start(nvmeq->dev->ctrl.admin_q);
1186

M
Matthew Wilcox 已提交
1187 1188
	irq_set_affinity_hint(vector, NULL);
	free_irq(vector, nvmeq);
M
Matthew Wilcox 已提交
1189

K
Keith Busch 已提交
1190 1191
	return 0;
}
M
Matthew Wilcox 已提交
1192

K
Keith Busch 已提交
1193 1194
static void nvme_clear_queue(struct nvme_queue *nvmeq)
{
1195
	spin_lock_irq(&nvmeq->q_lock);
1196 1197
	if (nvmeq->tags && *nvmeq->tags)
		blk_mq_all_tag_busy_iter(*nvmeq->tags, nvme_cancel_queue_ios, nvmeq);
1198
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1199 1200
}

K
Keith Busch 已提交
1201 1202
static void nvme_disable_queue(struct nvme_dev *dev, int qid)
{
M
Matias Bjørling 已提交
1203
	struct nvme_queue *nvmeq = dev->queues[qid];
K
Keith Busch 已提交
1204 1205 1206 1207 1208 1209

	if (!nvmeq)
		return;
	if (nvme_suspend_queue(nvmeq))
		return;

K
Keith Busch 已提交
1210 1211
	/* Don't tell the adapter to delete the admin queue.
	 * Don't tell a removed adapter to delete IO queues. */
1212
	if (qid && readl(dev->bar + NVME_REG_CSTS) != -1) {
M
Matthew Wilcox 已提交
1213 1214 1215
		adapter_delete_sq(dev, qid);
		adapter_delete_cq(dev, qid);
	}
1216 1217 1218 1219

	spin_lock_irq(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1220 1221
}

1222 1223 1224 1225
static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
				int entry_size)
{
	int q_depth = dev->q_depth;
1226 1227
	unsigned q_size_aligned = roundup(q_depth * entry_size,
					  dev->ctrl.page_size);
1228 1229

	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1230
		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1231
		mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
1232
		q_depth = div_u64(mem_per_q, entry_size);
1233 1234 1235 1236 1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249

		/*
		 * Ensure the reduced q_depth is above some threshold where it
		 * would be better to map queues in system memory with the
		 * original depth
		 */
		if (q_depth < 64)
			return -ENOMEM;
	}

	return q_depth;
}

static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
				int qid, int depth)
{
	if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
1250 1251
		unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth),
						      dev->ctrl.page_size);
1252 1253 1254 1255 1256 1257 1258 1259 1260 1261 1262 1263
		nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
		nvmeq->sq_cmds_io = dev->cmb + offset;
	} else {
		nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
					&nvmeq->sq_dma_addr, GFP_KERNEL);
		if (!nvmeq->sq_cmds)
			return -ENOMEM;
	}

	return 0;
}

M
Matthew Wilcox 已提交
1264
static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
K
Keith Busch 已提交
1265
							int depth)
M
Matthew Wilcox 已提交
1266
{
M
Matias Bjørling 已提交
1267
	struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
M
Matthew Wilcox 已提交
1268 1269 1270
	if (!nvmeq)
		return NULL;

1271
	nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
J
Joe Perches 已提交
1272
					  &nvmeq->cq_dma_addr, GFP_KERNEL);
M
Matthew Wilcox 已提交
1273 1274 1275
	if (!nvmeq->cqes)
		goto free_nvmeq;

1276
	if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
M
Matthew Wilcox 已提交
1277 1278
		goto free_cqdma;

1279
	nvmeq->q_dmadev = dev->dev;
M
Matthew Wilcox 已提交
1280
	nvmeq->dev = dev;
1281
	snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
1282
			dev->ctrl.instance, qid);
M
Matthew Wilcox 已提交
1283 1284
	spin_lock_init(&nvmeq->q_lock);
	nvmeq->cq_head = 0;
M
Matthew Wilcox 已提交
1285
	nvmeq->cq_phase = 1;
1286
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
M
Matthew Wilcox 已提交
1287
	nvmeq->q_depth = depth;
K
Keith Busch 已提交
1288
	nvmeq->qid = qid;
1289
	nvmeq->cq_vector = -1;
M
Matias Bjørling 已提交
1290
	dev->queues[qid] = nvmeq;
M
Matthew Wilcox 已提交
1291

1292 1293 1294 1295
	/* make sure queue descriptor is set before queue count, for kthread */
	mb();
	dev->queue_count++;

M
Matthew Wilcox 已提交
1296 1297 1298
	return nvmeq;

 free_cqdma:
1299
	dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
M
Matthew Wilcox 已提交
1300 1301 1302 1303 1304 1305
							nvmeq->cq_dma_addr);
 free_nvmeq:
	kfree(nvmeq);
	return NULL;
}

1306 1307 1308
static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
							const char *name)
{
1309 1310
	if (use_threaded_interrupts)
		return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1311
					nvme_irq_check, nvme_irq, IRQF_SHARED,
1312
					name, nvmeq);
1313
	return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1314
				IRQF_SHARED, name, nvmeq);
1315 1316
}

1317
static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
M
Matthew Wilcox 已提交
1318
{
1319
	struct nvme_dev *dev = nvmeq->dev;
M
Matthew Wilcox 已提交
1320

1321
	spin_lock_irq(&nvmeq->q_lock);
1322 1323 1324
	nvmeq->sq_tail = 0;
	nvmeq->cq_head = 0;
	nvmeq->cq_phase = 1;
1325
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1326
	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
K
Keith Busch 已提交
1327
	dev->online_queues++;
1328
	spin_unlock_irq(&nvmeq->q_lock);
1329 1330 1331 1332 1333 1334
}

static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
{
	struct nvme_dev *dev = nvmeq->dev;
	int result;
1335

K
Keith Busch 已提交
1336
	nvmeq->cq_vector = qid - 1;
M
Matthew Wilcox 已提交
1337 1338
	result = adapter_alloc_cq(dev, qid, nvmeq);
	if (result < 0)
1339
		return result;
M
Matthew Wilcox 已提交
1340 1341 1342 1343 1344

	result = adapter_alloc_sq(dev, qid, nvmeq);
	if (result < 0)
		goto release_cq;

1345
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
M
Matthew Wilcox 已提交
1346 1347 1348
	if (result < 0)
		goto release_sq;

1349 1350
	nvme_init_queue(nvmeq, qid);
	return result;
M
Matthew Wilcox 已提交
1351 1352 1353 1354 1355

 release_sq:
	adapter_delete_sq(dev, qid);
 release_cq:
	adapter_delete_cq(dev, qid);
1356
	return result;
M
Matthew Wilcox 已提交
1357 1358
}

M
Matias Bjørling 已提交
1359
static struct blk_mq_ops nvme_mq_admin_ops = {
1360
	.queue_rq	= nvme_queue_rq,
M
Matias Bjørling 已提交
1361 1362
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_admin_init_hctx,
1363
	.exit_hctx      = nvme_admin_exit_hctx,
M
Matias Bjørling 已提交
1364 1365 1366 1367 1368 1369 1370 1371 1372 1373
	.init_request	= nvme_admin_init_request,
	.timeout	= nvme_timeout,
};

static struct blk_mq_ops nvme_mq_ops = {
	.queue_rq	= nvme_queue_rq,
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_init_hctx,
	.init_request	= nvme_init_request,
	.timeout	= nvme_timeout,
J
Jens Axboe 已提交
1374
	.poll		= nvme_poll,
M
Matias Bjørling 已提交
1375 1376
};

1377 1378
static void nvme_dev_remove_admin(struct nvme_dev *dev)
{
1379 1380
	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
		blk_cleanup_queue(dev->ctrl.admin_q);
1381 1382 1383 1384
		blk_mq_free_tag_set(&dev->admin_tagset);
	}
}

M
Matias Bjørling 已提交
1385 1386
static int nvme_alloc_admin_tags(struct nvme_dev *dev)
{
1387
	if (!dev->ctrl.admin_q) {
M
Matias Bjørling 已提交
1388 1389
		dev->admin_tagset.ops = &nvme_mq_admin_ops;
		dev->admin_tagset.nr_hw_queues = 1;
C
Christoph Hellwig 已提交
1390
		dev->admin_tagset.queue_depth = NVME_AQ_DEPTH;
1391
		dev->admin_tagset.reserved_tags = 1;
M
Matias Bjørling 已提交
1392
		dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1393
		dev->admin_tagset.numa_node = dev_to_node(dev->dev);
1394
		dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
M
Matias Bjørling 已提交
1395 1396 1397 1398 1399
		dev->admin_tagset.driver_data = dev;

		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
			return -ENOMEM;

1400 1401
		dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
		if (IS_ERR(dev->ctrl.admin_q)) {
M
Matias Bjørling 已提交
1402 1403 1404
			blk_mq_free_tag_set(&dev->admin_tagset);
			return -ENOMEM;
		}
1405
		if (!blk_get_queue(dev->ctrl.admin_q)) {
1406
			nvme_dev_remove_admin(dev);
1407
			dev->ctrl.admin_q = NULL;
1408 1409
			return -ENODEV;
		}
K
Keith Busch 已提交
1410
	} else
1411
		blk_mq_unfreeze_queue(dev->ctrl.admin_q);
M
Matias Bjørling 已提交
1412 1413 1414 1415

	return 0;
}

1416
static int nvme_configure_admin_queue(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1417
{
1418
	int result;
M
Matthew Wilcox 已提交
1419
	u32 aqa;
1420
	u64 cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
M
Matthew Wilcox 已提交
1421 1422
	struct nvme_queue *nvmeq;

1423
	dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1) ?
1424 1425
						NVME_CAP_NSSRC(cap) : 0;

1426 1427 1428
	if (dev->subsystem &&
	    (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
		writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1429

1430
	result = nvme_disable_ctrl(&dev->ctrl, cap);
1431 1432
	if (result < 0)
		return result;
M
Matthew Wilcox 已提交
1433

M
Matias Bjørling 已提交
1434
	nvmeq = dev->queues[0];
1435
	if (!nvmeq) {
K
Keith Busch 已提交
1436
		nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1437 1438 1439
		if (!nvmeq)
			return -ENOMEM;
	}
M
Matthew Wilcox 已提交
1440 1441 1442 1443

	aqa = nvmeq->q_depth - 1;
	aqa |= aqa << 16;

1444 1445 1446
	writel(aqa, dev->bar + NVME_REG_AQA);
	lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
	lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
M
Matthew Wilcox 已提交
1447

1448
	result = nvme_enable_ctrl(&dev->ctrl, cap);
1449
	if (result)
M
Matias Bjørling 已提交
1450 1451
		goto free_nvmeq;

K
Keith Busch 已提交
1452
	nvmeq->cq_vector = 0;
1453
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
1454 1455
	if (result) {
		nvmeq->cq_vector = -1;
K
Keith Busch 已提交
1456
		goto free_nvmeq;
1457
	}
1458

M
Matthew Wilcox 已提交
1459
	return result;
M
Matias Bjørling 已提交
1460 1461 1462 1463

 free_nvmeq:
	nvme_free_queues(dev, 0);
	return result;
M
Matthew Wilcox 已提交
1464 1465
}

1466 1467
static int nvme_kthread(void *data)
{
1468
	struct nvme_dev *dev, *next;
1469 1470

	while (!kthread_should_stop()) {
1471
		set_current_state(TASK_INTERRUPTIBLE);
1472
		spin_lock(&dev_list_lock);
1473
		list_for_each_entry_safe(dev, next, &dev_list, node) {
1474
			int i;
1475
			u32 csts = readl(dev->bar + NVME_REG_CSTS);
1476

C
Christoph Hellwig 已提交
1477 1478 1479 1480 1481 1482
			/*
			 * Skip controllers currently under reset.
			 */
			if (work_pending(&dev->reset_work) || work_busy(&dev->reset_work))
				continue;

1483 1484
			if ((dev->subsystem && (csts & NVME_CSTS_NSSRO)) ||
							csts & NVME_CSTS_CFS) {
C
Christoph Hellwig 已提交
1485
				if (queue_work(nvme_workq, &dev->reset_work)) {
1486 1487
					dev_warn(dev->dev,
						"Failed status: %x, reset controller\n",
1488
						readl(dev->bar + NVME_REG_CSTS));
1489
				}
1490 1491
				continue;
			}
1492
			for (i = 0; i < dev->queue_count; i++) {
M
Matias Bjørling 已提交
1493
				struct nvme_queue *nvmeq = dev->queues[i];
1494 1495
				if (!nvmeq)
					continue;
1496
				spin_lock_irq(&nvmeq->q_lock);
1497
				nvme_process_cq(nvmeq);
K
Keith Busch 已提交
1498

1499
				while (i == 0 && dev->ctrl.event_limit > 0) {
M
Matias Bjørling 已提交
1500
					if (nvme_submit_async_admin_req(dev))
K
Keith Busch 已提交
1501
						break;
1502
					dev->ctrl.event_limit--;
K
Keith Busch 已提交
1503
				}
1504 1505 1506 1507
				spin_unlock_irq(&nvmeq->q_lock);
			}
		}
		spin_unlock(&dev_list_lock);
1508
		schedule_timeout(round_jiffies_relative(HZ));
1509 1510 1511 1512
	}
	return 0;
}

1513
static int nvme_create_io_queues(struct nvme_dev *dev)
K
Keith Busch 已提交
1514
{
M
Matias Bjørling 已提交
1515
	unsigned i;
1516
	int ret = 0;
K
Keith Busch 已提交
1517

1518 1519 1520
	for (i = dev->queue_count; i <= dev->max_qid; i++) {
		if (!nvme_alloc_queue(dev, i, dev->q_depth)) {
			ret = -ENOMEM;
K
Keith Busch 已提交
1521
			break;
1522 1523
		}
	}
K
Keith Busch 已提交
1524

1525 1526 1527
	for (i = dev->online_queues; i <= dev->queue_count - 1; i++) {
		ret = nvme_create_queue(dev->queues[i], i);
		if (ret) {
1528
			nvme_free_queues(dev, i);
K
Keith Busch 已提交
1529
			break;
1530
		}
1531 1532 1533 1534 1535 1536 1537 1538 1539
	}

	/*
	 * Ignore failing Create SQ/CQ commands, we can continue with less
	 * than the desired aount of queues, and even a controller without
	 * I/O queues an still be used to issue admin commands.  This might
	 * be useful to upgrade a buggy firmware for example.
	 */
	return ret >= 0 ? 0 : ret;
K
Keith Busch 已提交
1540 1541
}

1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553
static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
{
	u64 szu, size, offset;
	u32 cmbloc;
	resource_size_t bar_size;
	struct pci_dev *pdev = to_pci_dev(dev->dev);
	void __iomem *cmb;
	dma_addr_t dma_addr;

	if (!use_cmb_sqes)
		return NULL;

1554
	dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1555 1556 1557
	if (!(NVME_CMB_SZ(dev->cmbsz)))
		return NULL;

1558
	cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593

	szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
	size = szu * NVME_CMB_SZ(dev->cmbsz);
	offset = szu * NVME_CMB_OFST(cmbloc);
	bar_size = pci_resource_len(pdev, NVME_CMB_BIR(cmbloc));

	if (offset > bar_size)
		return NULL;

	/*
	 * Controllers may support a CMB size larger than their BAR,
	 * for example, due to being behind a bridge. Reduce the CMB to
	 * the reported size of the BAR
	 */
	if (size > bar_size - offset)
		size = bar_size - offset;

	dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(cmbloc)) + offset;
	cmb = ioremap_wc(dma_addr, size);
	if (!cmb)
		return NULL;

	dev->cmb_dma_addr = dma_addr;
	dev->cmb_size = size;
	return cmb;
}

static inline void nvme_release_cmb(struct nvme_dev *dev)
{
	if (dev->cmb) {
		iounmap(dev->cmb);
		dev->cmb = NULL;
	}
}

K
Keith Busch 已提交
1594 1595
static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
{
1596
	return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
K
Keith Busch 已提交
1597 1598
}

1599
static int nvme_setup_io_queues(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1600
{
M
Matias Bjørling 已提交
1601
	struct nvme_queue *adminq = dev->queues[0];
1602
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
1603
	int result, i, vecs, nr_io_queues, size;
M
Matthew Wilcox 已提交
1604

K
Keith Busch 已提交
1605
	nr_io_queues = num_possible_cpus();
C
Christoph Hellwig 已提交
1606 1607
	result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
	if (result < 0)
M
Matthew Wilcox 已提交
1608
		return result;
C
Christoph Hellwig 已提交
1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619

	/*
	 * Degraded controllers might return an error when setting the queue
	 * count.  We still want to be able to bring them online and offer
	 * access to the admin queue, as that might be only way to fix them up.
	 */
	if (result > 0) {
		dev_err(dev->dev, "Could not set queue count (%d)\n", result);
		nr_io_queues = 0;
		result = 0;
	}
M
Matthew Wilcox 已提交
1620

1621 1622 1623 1624 1625 1626 1627 1628 1629
	if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
		result = nvme_cmb_qdepth(dev, nr_io_queues,
				sizeof(struct nvme_command));
		if (result > 0)
			dev->q_depth = result;
		else
			nvme_release_cmb(dev);
	}

K
Keith Busch 已提交
1630 1631
	size = db_bar_size(dev, nr_io_queues);
	if (size > 8192) {
1632
		iounmap(dev->bar);
K
Keith Busch 已提交
1633 1634 1635 1636 1637 1638 1639 1640
		do {
			dev->bar = ioremap(pci_resource_start(pdev, 0), size);
			if (dev->bar)
				break;
			if (!--nr_io_queues)
				return -ENOMEM;
			size = db_bar_size(dev, nr_io_queues);
		} while (1);
1641
		dev->dbs = dev->bar + 4096;
1642
		adminq->q_db = dev->dbs;
1643 1644
	}

K
Keith Busch 已提交
1645
	/* Deregister the admin queue's interrupt */
1646
	free_irq(dev->entry[0].vector, adminq);
K
Keith Busch 已提交
1647

1648 1649 1650 1651 1652 1653 1654
	/*
	 * If we enable msix early due to not intx, disable it again before
	 * setting up the full range we need.
	 */
	if (!pdev->irq)
		pci_disable_msix(pdev);

1655
	for (i = 0; i < nr_io_queues; i++)
M
Matthew Wilcox 已提交
1656
		dev->entry[i].entry = i;
1657 1658 1659 1660 1661 1662 1663 1664
	vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
	if (vecs < 0) {
		vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
		if (vecs < 0) {
			vecs = 1;
		} else {
			for (i = 0; i < vecs; i++)
				dev->entry[i].vector = i + pdev->irq;
R
Ramachandra Rao Gajula 已提交
1665 1666 1667
		}
	}

1668 1669 1670 1671 1672 1673 1674
	/*
	 * Should investigate if there's a performance win from allocating
	 * more queues than interrupt vectors; it might allow the submission
	 * path to scale better, even if the receive path is limited by the
	 * number of interrupts.
	 */
	nr_io_queues = vecs;
K
Keith Busch 已提交
1675
	dev->max_qid = nr_io_queues;
1676

1677
	result = queue_request_irq(dev, adminq, adminq->irqname);
1678 1679
	if (result) {
		adminq->cq_vector = -1;
1680
		goto free_queues;
1681
	}
M
Matthew Wilcox 已提交
1682

1683
	/* Free previously allocated queues that are no longer usable */
K
Keith Busch 已提交
1684
	nvme_free_queues(dev, nr_io_queues + 1);
1685
	return nvme_create_io_queues(dev);
M
Matthew Wilcox 已提交
1686

1687
 free_queues:
1688
	nvme_free_queues(dev, 1);
1689
	return result;
M
Matthew Wilcox 已提交
1690 1691
}

1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707
static void nvme_set_irq_hints(struct nvme_dev *dev)
{
	struct nvme_queue *nvmeq;
	int i;

	for (i = 0; i < dev->online_queues; i++) {
		nvmeq = dev->queues[i];

		if (!nvmeq->tags || !(*nvmeq->tags))
			continue;

		irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
					blk_mq_tags_cpumask(*nvmeq->tags));
	}
}

K
Keith Busch 已提交
1708 1709 1710 1711 1712 1713
static void nvme_dev_scan(struct work_struct *work)
{
	struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);

	if (!dev->tagset.tags)
		return;
1714
	nvme_scan_namespaces(&dev->ctrl);
1715
	nvme_set_irq_hints(dev);
K
Keith Busch 已提交
1716 1717
}

1718 1719 1720 1721 1722 1723
/*
 * Return: error value if an error occurred setting up the queues or calling
 * Identify Device.  0 if these succeeded, even if adding some of the
 * namespaces failed.  At the moment, these failures are silent.  TBD which
 * failures should be reported.
 */
1724
static int nvme_dev_add(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1725
{
1726
	if (!dev->ctrl.tagset) {
1727 1728 1729 1730 1731
		dev->tagset.ops = &nvme_mq_ops;
		dev->tagset.nr_hw_queues = dev->online_queues - 1;
		dev->tagset.timeout = NVME_IO_TIMEOUT;
		dev->tagset.numa_node = dev_to_node(dev->dev);
		dev->tagset.queue_depth =
M
Matias Bjørling 已提交
1732
				min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
1733 1734 1735
		dev->tagset.cmd_size = nvme_cmd_size(dev);
		dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
		dev->tagset.driver_data = dev;
M
Matthew Wilcox 已提交
1736

1737 1738
		if (blk_mq_alloc_tag_set(&dev->tagset))
			return 0;
1739
		dev->ctrl.tagset = &dev->tagset;
1740
	}
1741
	queue_work(nvme_workq, &dev->scan_work);
K
Keith Busch 已提交
1742
	return 0;
M
Matthew Wilcox 已提交
1743 1744
}

1745 1746
static int nvme_dev_map(struct nvme_dev *dev)
{
K
Keith Busch 已提交
1747
	u64 cap;
1748
	int bars, result = -ENOMEM;
1749
	struct pci_dev *pdev = to_pci_dev(dev->dev);
1750 1751 1752 1753 1754 1755 1756

	if (pci_enable_device_mem(pdev))
		return result;

	dev->entry[0].vector = pdev->irq;
	pci_set_master(pdev);
	bars = pci_select_bars(pdev, IORESOURCE_MEM);
1757 1758 1759
	if (!bars)
		goto disable_pci;

1760 1761 1762
	if (pci_request_selected_regions(pdev, bars, "nvme"))
		goto disable_pci;

1763 1764
	if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
	    dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
1765
		goto disable;
1766 1767 1768 1769

	dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
	if (!dev->bar)
		goto disable;
1770

1771
	if (readl(dev->bar + NVME_REG_CSTS) == -1) {
K
Keith Busch 已提交
1772 1773 1774
		result = -ENODEV;
		goto unmap;
	}
1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785

	/*
	 * Some devices don't advertse INTx interrupts, pre-enable a single
	 * MSIX vec for setup. We'll adjust this later.
	 */
	if (!pdev->irq) {
		result = pci_enable_msix(pdev, dev->entry, 1);
		if (result < 0)
			goto unmap;
	}

1786 1787
	cap = lo_hi_readq(dev->bar + NVME_REG_CAP);

K
Keith Busch 已提交
1788 1789
	dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
	dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
1790 1791
	dev->dbs = dev->bar + 4096;
	if (readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 2))
1792
		dev->cmb = nvme_map_cmb(dev);
1793 1794 1795

	return 0;

K
Keith Busch 已提交
1796 1797 1798
 unmap:
	iounmap(dev->bar);
	dev->bar = NULL;
1799 1800 1801 1802 1803 1804 1805 1806 1807
 disable:
	pci_release_regions(pdev);
 disable_pci:
	pci_disable_device(pdev);
	return result;
}

static void nvme_dev_unmap(struct nvme_dev *dev)
{
1808 1809 1810 1811 1812 1813
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	if (pdev->msi_enabled)
		pci_disable_msi(pdev);
	else if (pdev->msix_enabled)
		pci_disable_msix(pdev);
1814 1815 1816 1817

	if (dev->bar) {
		iounmap(dev->bar);
		dev->bar = NULL;
1818
		pci_release_regions(pdev);
1819 1820
	}

1821 1822
	if (pci_is_enabled(pdev))
		pci_disable_device(pdev);
1823 1824
}

K
Keith Busch 已提交
1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841
struct nvme_delq_ctx {
	struct task_struct *waiter;
	struct kthread_worker *worker;
	atomic_t refcount;
};

static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
{
	dq->waiter = current;
	mb();

	for (;;) {
		set_current_state(TASK_KILLABLE);
		if (!atomic_read(&dq->refcount))
			break;
		if (!schedule_timeout(ADMIN_TIMEOUT) ||
					fatal_signal_pending(current)) {
K
Keith Busch 已提交
1842 1843 1844 1845 1846 1847 1848
			/*
			 * Disable the controller first since we can't trust it
			 * at this point, but leave the admin queue enabled
			 * until all queue deletion requests are flushed.
			 * FIXME: This may take a while if there are more h/w
			 * queues than admin tags.
			 */
K
Keith Busch 已提交
1849
			set_current_state(TASK_RUNNING);
1850
			nvme_disable_ctrl(&dev->ctrl,
1851
				lo_hi_readq(dev->bar + NVME_REG_CAP));
K
Keith Busch 已提交
1852
			nvme_clear_queue(dev->queues[0]);
K
Keith Busch 已提交
1853
			flush_kthread_worker(dq->worker);
K
Keith Busch 已提交
1854
			nvme_disable_queue(dev, 0);
K
Keith Busch 已提交
1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873 1874 1875 1876 1877
			return;
		}
	}
	set_current_state(TASK_RUNNING);
}

static void nvme_put_dq(struct nvme_delq_ctx *dq)
{
	atomic_dec(&dq->refcount);
	if (dq->waiter)
		wake_up_process(dq->waiter);
}

static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq)
{
	atomic_inc(&dq->refcount);
	return dq;
}

static void nvme_del_queue_end(struct nvme_queue *nvmeq)
{
	struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx;
	nvme_put_dq(dq);
1878 1879 1880 1881

	spin_lock_irq(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
K
Keith Busch 已提交
1882 1883 1884 1885 1886
}

static int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode,
						kthread_work_func_t fn)
{
1887
	struct request *req;
K
Keith Busch 已提交
1888 1889 1890 1891 1892 1893 1894
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(nvmeq->qid);

	init_kthread_work(&nvmeq->cmdinfo.work, fn);
1895 1896 1897 1898 1899 1900 1901 1902 1903

	req = nvme_alloc_request(nvmeq->dev->ctrl.admin_q, &c, 0);
	if (IS_ERR(req))
		return PTR_ERR(req);

	req->timeout = ADMIN_TIMEOUT;
	req->end_io_data = &nvmeq->cmdinfo;
	blk_execute_rq_nowait(req->q, NULL, req, 0, async_cmd_info_endio);
	return 0;
K
Keith Busch 已提交
1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918 1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950
}

static void nvme_del_cq_work_handler(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	nvme_del_queue_end(nvmeq);
}

static int nvme_delete_cq(struct nvme_queue *nvmeq)
{
	return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq,
						nvme_del_cq_work_handler);
}

static void nvme_del_sq_work_handler(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	int status = nvmeq->cmdinfo.status;

	if (!status)
		status = nvme_delete_cq(nvmeq);
	if (status)
		nvme_del_queue_end(nvmeq);
}

static int nvme_delete_sq(struct nvme_queue *nvmeq)
{
	return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq,
						nvme_del_sq_work_handler);
}

static void nvme_del_queue_start(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	if (nvme_delete_sq(nvmeq))
		nvme_del_queue_end(nvmeq);
}

static void nvme_disable_io_queues(struct nvme_dev *dev)
{
	int i;
	DEFINE_KTHREAD_WORKER_ONSTACK(worker);
	struct nvme_delq_ctx dq;
	struct task_struct *kworker_task = kthread_run(kthread_worker_fn,
1951
					&worker, "nvme%d", dev->ctrl.instance);
K
Keith Busch 已提交
1952 1953

	if (IS_ERR(kworker_task)) {
1954
		dev_err(dev->dev,
K
Keith Busch 已提交
1955 1956 1957 1958 1959 1960 1961 1962 1963 1964
			"Failed to create queue del task\n");
		for (i = dev->queue_count - 1; i > 0; i--)
			nvme_disable_queue(dev, i);
		return;
	}

	dq.waiter = NULL;
	atomic_set(&dq.refcount, 0);
	dq.worker = &worker;
	for (i = dev->queue_count - 1; i > 0; i--) {
M
Matias Bjørling 已提交
1965
		struct nvme_queue *nvmeq = dev->queues[i];
K
Keith Busch 已提交
1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977

		if (nvme_suspend_queue(nvmeq))
			continue;
		nvmeq->cmdinfo.ctx = nvme_get_dq(&dq);
		nvmeq->cmdinfo.worker = dq.worker;
		init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start);
		queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work);
	}
	nvme_wait_dq(&dq, dev);
	kthread_stop(kworker_task);
}

1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001
static int nvme_dev_list_add(struct nvme_dev *dev)
{
	bool start_thread = false;

	spin_lock(&dev_list_lock);
	if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
		start_thread = true;
		nvme_thread = NULL;
	}
	list_add(&dev->node, &dev_list);
	spin_unlock(&dev_list_lock);

	if (start_thread) {
		nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
		wake_up_all(&nvme_kthread_wait);
	} else
		wait_event_killable(nvme_kthread_wait, nvme_thread);

	if (IS_ERR_OR_NULL(nvme_thread))
		return nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;

	return 0;
}

2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016 2017 2018 2019 2020 2021
/*
* Remove the node from the device list and check
* for whether or not we need to stop the nvme_thread.
*/
static void nvme_dev_list_remove(struct nvme_dev *dev)
{
	struct task_struct *tmp = NULL;

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) {
		tmp = nvme_thread;
		nvme_thread = NULL;
	}
	spin_unlock(&dev_list_lock);

	if (tmp)
		kthread_stop(tmp);
}

2022 2023 2024 2025
static void nvme_freeze_queues(struct nvme_dev *dev)
{
	struct nvme_ns *ns;

2026
	list_for_each_entry(ns, &dev->ctrl.namespaces, list) {
2027 2028
		blk_mq_freeze_queue_start(ns->queue);

2029
		spin_lock_irq(ns->queue->queue_lock);
2030
		queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
2031
		spin_unlock_irq(ns->queue->queue_lock);
2032 2033 2034 2035 2036 2037 2038 2039 2040 2041

		blk_mq_cancel_requeue_work(ns->queue);
		blk_mq_stop_hw_queues(ns->queue);
	}
}

static void nvme_unfreeze_queues(struct nvme_dev *dev)
{
	struct nvme_ns *ns;

2042
	list_for_each_entry(ns, &dev->ctrl.namespaces, list) {
2043 2044 2045 2046 2047 2048 2049
		queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
		blk_mq_unfreeze_queue(ns->queue);
		blk_mq_start_stopped_hw_queues(ns->queue, true);
		blk_mq_kick_requeue_list(ns->queue);
	}
}

2050
static void nvme_dev_shutdown(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2051
{
2052
	int i;
2053
	u32 csts = -1;
2054

2055
	nvme_dev_list_remove(dev);
2056

2057
	mutex_lock(&dev->shutdown_lock);
2058 2059
	if (dev->bar) {
		nvme_freeze_queues(dev);
2060
		csts = readl(dev->bar + NVME_REG_CSTS);
2061
	}
2062
	if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
K
Keith Busch 已提交
2063
		for (i = dev->queue_count - 1; i >= 0; i--) {
M
Matias Bjørling 已提交
2064
			struct nvme_queue *nvmeq = dev->queues[i];
K
Keith Busch 已提交
2065 2066 2067 2068
			nvme_suspend_queue(nvmeq);
		}
	} else {
		nvme_disable_io_queues(dev);
2069
		nvme_shutdown_ctrl(&dev->ctrl);
K
Keith Busch 已提交
2070 2071
		nvme_disable_queue(dev, 0);
	}
2072
	nvme_dev_unmap(dev);
2073 2074 2075

	for (i = dev->queue_count - 1; i >= 0; i--)
		nvme_clear_queue(dev->queues[i]);
2076
	mutex_unlock(&dev->shutdown_lock);
2077 2078
}

M
Matthew Wilcox 已提交
2079 2080
static int nvme_setup_prp_pools(struct nvme_dev *dev)
{
2081
	dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
M
Matthew Wilcox 已提交
2082 2083 2084 2085
						PAGE_SIZE, PAGE_SIZE, 0);
	if (!dev->prp_page_pool)
		return -ENOMEM;

2086
	/* Optimisation for I/Os between 4k and 128k */
2087
	dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2088 2089 2090 2091 2092
						256, 256, 0);
	if (!dev->prp_small_pool) {
		dma_pool_destroy(dev->prp_page_pool);
		return -ENOMEM;
	}
M
Matthew Wilcox 已提交
2093 2094 2095 2096 2097 2098
	return 0;
}

static void nvme_release_prp_pools(struct nvme_dev *dev)
{
	dma_pool_destroy(dev->prp_page_pool);
2099
	dma_pool_destroy(dev->prp_small_pool);
M
Matthew Wilcox 已提交
2100 2101
}

2102
static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
2103
{
2104
	struct nvme_dev *dev = to_nvme_dev(ctrl);
2105

2106
	put_device(dev->dev);
2107 2108
	if (dev->tagset.tags)
		blk_mq_free_tag_set(&dev->tagset);
2109 2110
	if (dev->ctrl.admin_q)
		blk_put_queue(dev->ctrl.admin_q);
2111 2112 2113 2114 2115
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
}

2116
static void nvme_reset_work(struct work_struct *work)
2117
{
2118
	struct nvme_dev *dev = container_of(work, struct nvme_dev, reset_work);
2119
	int result;
2120

2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132
	if (WARN_ON(test_bit(NVME_CTRL_RESETTING, &dev->flags)))
		goto out;

	/*
	 * If we're called to reset a live controller first shut it down before
	 * moving on.
	 */
	if (dev->bar)
		nvme_dev_shutdown(dev);

	set_bit(NVME_CTRL_RESETTING, &dev->flags);

2133 2134
	result = nvme_dev_map(dev);
	if (result)
2135
		goto out;
2136 2137 2138 2139 2140

	result = nvme_configure_admin_queue(dev);
	if (result)
		goto unmap;

M
Matias Bjørling 已提交
2141
	nvme_init_queue(dev->queues[0], 0);
K
Keith Busch 已提交
2142 2143 2144
	result = nvme_alloc_admin_tags(dev);
	if (result)
		goto disable;
2145

2146 2147 2148 2149
	result = nvme_init_identify(&dev->ctrl);
	if (result)
		goto free_tags;

2150
	result = nvme_setup_io_queues(dev);
2151
	if (result)
K
Keith Busch 已提交
2152
		goto free_tags;
2153

2154
	dev->ctrl.event_limit = 1;
2155

2156 2157 2158 2159
	result = nvme_dev_list_add(dev);
	if (result)
		goto remove;

2160 2161 2162 2163
	/*
	 * Keep the controller around but remove all namespaces if we don't have
	 * any working I/O queue.
	 */
2164 2165
	if (dev->online_queues < 2) {
		dev_warn(dev->dev, "IO queues not created\n");
2166
		nvme_remove_namespaces(&dev->ctrl);
2167 2168 2169 2170 2171
	} else {
		nvme_unfreeze_queues(dev);
		nvme_dev_add(dev);
	}

2172
	clear_bit(NVME_CTRL_RESETTING, &dev->flags);
2173
	return;
2174

2175 2176
 remove:
	nvme_dev_list_remove(dev);
K
Keith Busch 已提交
2177 2178
 free_tags:
	nvme_dev_remove_admin(dev);
2179 2180
	blk_put_queue(dev->ctrl.admin_q);
	dev->ctrl.admin_q = NULL;
2181
	dev->queues[0]->tags = NULL;
2182
 disable:
2183
	nvme_disable_queue(dev, 0);
2184 2185
 unmap:
	nvme_dev_unmap(dev);
2186
 out:
2187
	nvme_remove_dead_ctrl(dev);
2188 2189
}

2190
static void nvme_remove_dead_ctrl_work(struct work_struct *work)
K
Keith Busch 已提交
2191
{
2192
	struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
2193
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
2194 2195

	if (pci_get_drvdata(pdev))
2196
		pci_stop_and_remove_bus_device_locked(pdev);
2197
	nvme_put_ctrl(&dev->ctrl);
K
Keith Busch 已提交
2198 2199
}

2200
static void nvme_remove_dead_ctrl(struct nvme_dev *dev)
2201
{
2202
	dev_warn(dev->dev, "Removing after probe failure\n");
2203
	kref_get(&dev->ctrl.kref);
2204
	if (!schedule_work(&dev->remove_work))
2205
		nvme_put_ctrl(&dev->ctrl);
2206 2207
}

2208 2209
static int nvme_reset(struct nvme_dev *dev)
{
2210
	if (!dev->ctrl.admin_q || blk_queue_dying(dev->ctrl.admin_q))
2211 2212
		return -ENODEV;

C
Christoph Hellwig 已提交
2213 2214
	if (!queue_work(nvme_workq, &dev->reset_work))
		return -EBUSY;
2215

C
Christoph Hellwig 已提交
2216 2217
	flush_work(&dev->reset_work);
	return 0;
2218 2219
}

2220 2221 2222 2223 2224 2225
static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
{
	*val = readl(to_nvme_dev(ctrl)->bar + off);
	return 0;
}

2226 2227 2228 2229 2230 2231
static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
{
	writel(val, to_nvme_dev(ctrl)->bar + off);
	return 0;
}

2232 2233 2234 2235 2236 2237
static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
{
	*val = readq(to_nvme_dev(ctrl)->bar + off);
	return 0;
}

2238 2239 2240 2241 2242 2243 2244
static bool nvme_pci_io_incapable(struct nvme_ctrl *ctrl)
{
	struct nvme_dev *dev = to_nvme_dev(ctrl);

	return !dev->bar || dev->online_queues < 2;
}

2245 2246 2247 2248 2249
static int nvme_pci_reset_ctrl(struct nvme_ctrl *ctrl)
{
	return nvme_reset(to_nvme_dev(ctrl));
}

2250 2251
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
	.reg_read32		= nvme_pci_reg_read32,
2252
	.reg_write32		= nvme_pci_reg_write32,
2253
	.reg_read64		= nvme_pci_reg_read64,
2254
	.io_incapable		= nvme_pci_io_incapable,
2255
	.reset_ctrl		= nvme_pci_reset_ctrl,
2256
	.free_ctrl		= nvme_pci_free_ctrl,
2257 2258
};

2259
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
M
Matthew Wilcox 已提交
2260
{
M
Matias Bjørling 已提交
2261
	int node, result = -ENOMEM;
M
Matthew Wilcox 已提交
2262 2263
	struct nvme_dev *dev;

M
Matias Bjørling 已提交
2264 2265 2266 2267 2268
	node = dev_to_node(&pdev->dev);
	if (node == NUMA_NO_NODE)
		set_dev_node(&pdev->dev, 0);

	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2269 2270
	if (!dev)
		return -ENOMEM;
M
Matias Bjørling 已提交
2271 2272
	dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2273 2274
	if (!dev->entry)
		goto free;
M
Matias Bjørling 已提交
2275 2276
	dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2277 2278 2279
	if (!dev->queues)
		goto free;

2280
	dev->dev = get_device(&pdev->dev);
K
Keith Busch 已提交
2281
	pci_set_drvdata(pdev, dev);
2282

2283 2284 2285
	INIT_LIST_HEAD(&dev->node);
	INIT_WORK(&dev->scan_work, nvme_dev_scan);
	INIT_WORK(&dev->reset_work, nvme_reset_work);
2286
	INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
2287
	mutex_init(&dev->shutdown_lock);
2288

2289
	result = nvme_setup_prp_pools(dev);
2290
	if (result)
K
Keith Busch 已提交
2291
		goto put_pci;
M
Matthew Wilcox 已提交
2292

2293 2294
	result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
			id->driver_data);
M
Matthew Wilcox 已提交
2295
	if (result)
2296
		goto release_pools;
2297

2298
	queue_work(nvme_workq, &dev->reset_work);
M
Matthew Wilcox 已提交
2299 2300
	return 0;

2301
 release_pools:
M
Matthew Wilcox 已提交
2302
	nvme_release_prp_pools(dev);
K
Keith Busch 已提交
2303
 put_pci:
2304
	put_device(dev->dev);
M
Matthew Wilcox 已提交
2305 2306 2307 2308 2309 2310 2311
 free:
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
	return result;
}

2312 2313
static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
{
K
Keith Busch 已提交
2314
	struct nvme_dev *dev = pci_get_drvdata(pdev);
2315

K
Keith Busch 已提交
2316 2317 2318
	if (prepare)
		nvme_dev_shutdown(dev);
	else
2319
		queue_work(nvme_workq, &dev->reset_work);
2320 2321
}

2322 2323 2324 2325 2326 2327
static void nvme_shutdown(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
	nvme_dev_shutdown(dev);
}

2328
static void nvme_remove(struct pci_dev *pdev)
M
Matthew Wilcox 已提交
2329 2330
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
K
Keith Busch 已提交
2331 2332 2333 2334 2335 2336 2337

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	spin_unlock(&dev_list_lock);

	pci_set_drvdata(pdev, NULL);
	flush_work(&dev->reset_work);
K
Keith Busch 已提交
2338
	flush_work(&dev->scan_work);
2339
	nvme_remove_namespaces(&dev->ctrl);
2340
	nvme_uninit_ctrl(&dev->ctrl);
2341
	nvme_dev_shutdown(dev);
M
Matias Bjørling 已提交
2342
	nvme_dev_remove_admin(dev);
2343
	nvme_free_queues(dev, 0);
2344
	nvme_release_cmb(dev);
K
Keith Busch 已提交
2345
	nvme_release_prp_pools(dev);
2346
	nvme_put_ctrl(&dev->ctrl);
M
Matthew Wilcox 已提交
2347 2348 2349 2350 2351 2352 2353 2354
}

/* These functions are yet to be implemented */
#define nvme_error_detected NULL
#define nvme_dump_registers NULL
#define nvme_link_reset NULL
#define nvme_slot_reset NULL
#define nvme_error_resume NULL
2355

2356
#ifdef CONFIG_PM_SLEEP
2357 2358 2359 2360 2361 2362 2363 2364 2365 2366 2367 2368 2369 2370
static int nvme_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

	nvme_dev_shutdown(ndev);
	return 0;
}

static int nvme_resume(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

2371
	queue_work(nvme_workq, &ndev->reset_work);
K
Keith Busch 已提交
2372
	return 0;
2373
}
2374
#endif
2375 2376

static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
M
Matthew Wilcox 已提交
2377

2378
static const struct pci_error_handlers nvme_err_handler = {
M
Matthew Wilcox 已提交
2379 2380 2381 2382 2383
	.error_detected	= nvme_error_detected,
	.mmio_enabled	= nvme_dump_registers,
	.link_reset	= nvme_link_reset,
	.slot_reset	= nvme_slot_reset,
	.resume		= nvme_error_resume,
2384
	.reset_notify	= nvme_reset_notify,
M
Matthew Wilcox 已提交
2385 2386 2387 2388 2389
};

/* Move to pci_ids.h later */
#define PCI_CLASS_STORAGE_EXPRESS	0x010802

2390
static const struct pci_device_id nvme_id_table[] = {
2391 2392
	{ PCI_VDEVICE(INTEL, 0x0953),
		.driver_data = NVME_QUIRK_STRIPE_SIZE, },
2393 2394
	{ PCI_VDEVICE(INTEL, 0x5845),	/* Qemu emulated controller */
		.driver_data = NVME_QUIRK_IDENTIFY_CNS, },
M
Matthew Wilcox 已提交
2395
	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2396
	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
M
Matthew Wilcox 已提交
2397 2398 2399 2400 2401 2402 2403 2404
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, nvme_id_table);

static struct pci_driver nvme_driver = {
	.name		= "nvme",
	.id_table	= nvme_id_table,
	.probe		= nvme_probe,
2405
	.remove		= nvme_remove,
2406
	.shutdown	= nvme_shutdown,
2407 2408 2409
	.driver		= {
		.pm	= &nvme_dev_pm_ops,
	},
M
Matthew Wilcox 已提交
2410 2411 2412 2413 2414
	.err_handler	= &nvme_err_handler,
};

static int __init nvme_init(void)
{
2415
	int result;
2416

2417
	init_waitqueue_head(&nvme_kthread_wait);
M
Matthew Wilcox 已提交
2418

2419
	nvme_workq = alloc_workqueue("nvme", WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
K
Keith Busch 已提交
2420
	if (!nvme_workq)
2421
		return -ENOMEM;
K
Keith Busch 已提交
2422

2423
	result = nvme_core_init();
2424
	if (result < 0)
K
Keith Busch 已提交
2425
		goto kill_workq;
M
Matthew Wilcox 已提交
2426

2427 2428
	result = pci_register_driver(&nvme_driver);
	if (result)
2429
		goto core_exit;
2430
	return 0;
M
Matthew Wilcox 已提交
2431

2432
 core_exit:
2433
	nvme_core_exit();
K
Keith Busch 已提交
2434 2435
 kill_workq:
	destroy_workqueue(nvme_workq);
M
Matthew Wilcox 已提交
2436 2437 2438 2439 2440 2441
	return result;
}

static void __exit nvme_exit(void)
{
	pci_unregister_driver(&nvme_driver);
2442
	nvme_core_exit();
K
Keith Busch 已提交
2443
	destroy_workqueue(nvme_workq);
2444
	BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
2445
	_nvme_check_size();
M
Matthew Wilcox 已提交
2446 2447 2448 2449
}

MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
MODULE_LICENSE("GPL");
2450
MODULE_VERSION("1.0");
M
Matthew Wilcox 已提交
2451 2452
module_init(nvme_init);
module_exit(nvme_exit);