pci.c 69.1 KB
Newer Older
M
Matthew Wilcox 已提交
1 2
/*
 * NVM Express device driver
3
 * Copyright (c) 2011-2014, Intel Corporation.
M
Matthew Wilcox 已提交
4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

K
Keith Busch 已提交
15
#include <linux/aer.h>
16
#include <linux/async.h>
M
Matthew Wilcox 已提交
17
#include <linux/blkdev.h>
M
Matias Bjørling 已提交
18
#include <linux/blk-mq.h>
19
#include <linux/blk-mq-pci.h>
20
#include <linux/dmi.h>
M
Matthew Wilcox 已提交
21 22 23 24 25
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/mm.h>
#include <linux/module.h>
26
#include <linux/mutex.h>
27
#include <linux/once.h>
M
Matthew Wilcox 已提交
28
#include <linux/pci.h>
K
Keith Busch 已提交
29
#include <linux/t10-pi.h>
M
Matthew Wilcox 已提交
30
#include <linux/types.h>
31
#include <linux/io-64-nonatomic-lo-hi.h>
32
#include <linux/sed-opal.h>
33

34 35
#include "nvme.h"

M
Matthew Wilcox 已提交
36 37
#define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))
#define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))
38

C
Chaitanya Kulkarni 已提交
39
#define SGES_PER_PAGE	(PAGE_SIZE / sizeof(struct nvme_sgl_desc))
40

41 42 43 44 45 46 47
/*
 * These can be higher, but we need to ensure that any command doesn't
 * require an sg allocation that needs more than a page of data.
 */
#define NVME_MAX_KB_SZ	4096
#define NVME_MAX_SEGS	127

48 49 50
static int use_threaded_interrupts;
module_param(use_threaded_interrupts, int, 0);

51
static bool use_cmb_sqes = true;
52
module_param(use_cmb_sqes, bool, 0444);
53 54
MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");

55 56 57 58
static unsigned int max_host_mem_size_mb = 128;
module_param(max_host_mem_size_mb, uint, 0444);
MODULE_PARM_DESC(max_host_mem_size_mb,
	"Maximum Host Memory Buffer (HMB) size per controller (in MiB)");
59

C
Chaitanya Kulkarni 已提交
60 61 62 63 64 65
static unsigned int sgl_threshold = SZ_32K;
module_param(sgl_threshold, uint, 0644);
MODULE_PARM_DESC(sgl_threshold,
		"Use SGLs when average request segment size is larger or equal to "
		"this size. Use 0 to disable SGLs.");

66 67 68 69 70 71 72 73 74 75
static int io_queue_depth_set(const char *val, const struct kernel_param *kp);
static const struct kernel_param_ops io_queue_depth_ops = {
	.set = io_queue_depth_set,
	.get = param_get_int,
};

static int io_queue_depth = 1024;
module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644);
MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2");

76 77
struct nvme_dev;
struct nvme_queue;
78

79
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
80

81 82 83 84
/*
 * Represents an NVM Express device.  Each nvme_dev is a PCI function.
 */
struct nvme_dev {
85
	struct nvme_queue *queues;
86 87 88 89 90 91 92 93
	struct blk_mq_tag_set tagset;
	struct blk_mq_tag_set admin_tagset;
	u32 __iomem *dbs;
	struct device *dev;
	struct dma_pool *prp_page_pool;
	struct dma_pool *prp_small_pool;
	unsigned online_queues;
	unsigned max_qid;
94
	unsigned int num_vecs;
95 96 97
	int q_depth;
	u32 db_stride;
	void __iomem *bar;
98
	unsigned long bar_mapped_size;
99
	struct work_struct remove_work;
100
	struct mutex shutdown_lock;
101 102
	bool subsystem;
	void __iomem *cmb;
103
	pci_bus_addr_t cmb_bus_addr;
104 105
	u64 cmb_size;
	u32 cmbsz;
106
	u32 cmbloc;
107
	struct nvme_ctrl ctrl;
K
Keith Busch 已提交
108
	struct completion ioq_wait;
109

110 111
	mempool_t *iod_mempool;

112
	/* shadow doorbell buffer support: */
113 114 115 116
	u32 *dbbuf_dbs;
	dma_addr_t dbbuf_dbs_dma_addr;
	u32 *dbbuf_eis;
	dma_addr_t dbbuf_eis_dma_addr;
117 118 119 120

	/* host memory buffer support: */
	u64 host_mem_size;
	u32 nr_host_mem_descs;
121
	dma_addr_t host_mem_descs_dma;
122 123
	struct nvme_host_mem_buf_desc *host_mem_descs;
	void **host_mem_desc_bufs;
K
Keith Busch 已提交
124
};
125

126 127 128 129 130 131 132 133 134 135 136
static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
{
	int n = 0, ret;

	ret = kstrtoint(val, 10, &n);
	if (ret != 0 || n < 2)
		return -EINVAL;

	return param_set_int(val, kp);
}

137 138 139 140 141 142 143 144 145 146
static inline unsigned int sq_idx(unsigned int qid, u32 stride)
{
	return qid * 2 * stride;
}

static inline unsigned int cq_idx(unsigned int qid, u32 stride)
{
	return (qid * 2 + 1) * stride;
}

147 148 149 150 151
static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
{
	return container_of(ctrl, struct nvme_dev, ctrl);
}

M
Matthew Wilcox 已提交
152 153 154 155 156 157
/*
 * An NVM Express queue.  Each device has at least two (one for admin
 * commands and one for I/O commands).
 */
struct nvme_queue {
	struct device *q_dmadev;
M
Matthew Wilcox 已提交
158
	struct nvme_dev *dev;
159
	spinlock_t sq_lock;
M
Matthew Wilcox 已提交
160
	struct nvme_command *sq_cmds;
161
	struct nvme_command __iomem *sq_cmds_io;
162
	spinlock_t cq_lock ____cacheline_aligned_in_smp;
M
Matthew Wilcox 已提交
163
	volatile struct nvme_completion *cqes;
164
	struct blk_mq_tags **tags;
M
Matthew Wilcox 已提交
165 166 167 168
	dma_addr_t sq_dma_addr;
	dma_addr_t cq_dma_addr;
	u32 __iomem *q_db;
	u16 q_depth;
J
Jens Axboe 已提交
169
	s16 cq_vector;
M
Matthew Wilcox 已提交
170 171
	u16 sq_tail;
	u16 cq_head;
172
	u16 last_cq_head;
K
Keith Busch 已提交
173
	u16 qid;
174
	u8 cq_phase;
175 176 177 178
	u32 *dbbuf_sq_db;
	u32 *dbbuf_cq_db;
	u32 *dbbuf_sq_ei;
	u32 *dbbuf_cq_ei;
M
Matthew Wilcox 已提交
179 180
};

181 182 183
/*
 * The nvme_iod describes the data in an I/O, including the list of PRP
 * entries.  You can't see it in this data structure because C doesn't let
C
Christoph Hellwig 已提交
184
 * me express that.  Use nvme_init_iod to ensure there's enough space
185 186 187
 * allocated to store the PRP list.
 */
struct nvme_iod {
188
	struct nvme_request req;
C
Christoph Hellwig 已提交
189
	struct nvme_queue *nvmeq;
C
Chaitanya Kulkarni 已提交
190
	bool use_sgl;
C
Christoph Hellwig 已提交
191
	int aborted;
192 193 194 195
	int npages;		/* In the PRP list. 0 means small pool in use */
	int nents;		/* Used in scatterlist */
	int length;		/* Of data, in bytes */
	dma_addr_t first_dma;
196
	struct scatterlist meta_sg; /* metadata requires single contiguous buffer */
C
Christoph Hellwig 已提交
197 198
	struct scatterlist *sg;
	struct scatterlist inline_sg[0];
M
Matthew Wilcox 已提交
199 200 201 202 203 204 205 206 207 208 209 210
};

/*
 * Check we didin't inadvertently grow the command struct
 */
static inline void _nvme_check_size(void)
{
	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
211
	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
K
Keith Busch 已提交
212
	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
M
Matthew Wilcox 已提交
213
	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
214 215
	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
M
Matthew Wilcox 已提交
216
	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
K
Keith Busch 已提交
217
	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291
	BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
}

static inline unsigned int nvme_dbbuf_size(u32 stride)
{
	return ((num_possible_cpus() + 1) * 8 * stride);
}

static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
{
	unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);

	if (dev->dbbuf_dbs)
		return 0;

	dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
					    &dev->dbbuf_dbs_dma_addr,
					    GFP_KERNEL);
	if (!dev->dbbuf_dbs)
		return -ENOMEM;
	dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
					    &dev->dbbuf_eis_dma_addr,
					    GFP_KERNEL);
	if (!dev->dbbuf_eis) {
		dma_free_coherent(dev->dev, mem_size,
				  dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
		dev->dbbuf_dbs = NULL;
		return -ENOMEM;
	}

	return 0;
}

static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
{
	unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);

	if (dev->dbbuf_dbs) {
		dma_free_coherent(dev->dev, mem_size,
				  dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
		dev->dbbuf_dbs = NULL;
	}
	if (dev->dbbuf_eis) {
		dma_free_coherent(dev->dev, mem_size,
				  dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
		dev->dbbuf_eis = NULL;
	}
}

static void nvme_dbbuf_init(struct nvme_dev *dev,
			    struct nvme_queue *nvmeq, int qid)
{
	if (!dev->dbbuf_dbs || !qid)
		return;

	nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
	nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
	nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
	nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
}

static void nvme_dbbuf_set(struct nvme_dev *dev)
{
	struct nvme_command c;

	if (!dev->dbbuf_dbs)
		return;

	memset(&c, 0, sizeof(c));
	c.dbbuf.opcode = nvme_admin_dbbuf;
	c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
	c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);

	if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
292
		dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323
		/* Free memory and continue on */
		nvme_dbbuf_dma_free(dev);
	}
}

static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
{
	return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
}

/* Update dbbuf and return true if an MMIO is required */
static bool nvme_dbbuf_update_and_check_event(u16 value, u32 *dbbuf_db,
					      volatile u32 *dbbuf_ei)
{
	if (dbbuf_db) {
		u16 old_value;

		/*
		 * Ensure that the queue is written before updating
		 * the doorbell in memory
		 */
		wmb();

		old_value = *dbbuf_db;
		*dbbuf_db = value;

		if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value))
			return false;
	}

	return true;
M
Matthew Wilcox 已提交
324 325
}

326 327 328 329
/*
 * Max size of iod being embedded in the request payload
 */
#define NVME_INT_PAGES		2
330
#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->ctrl.page_size)
331 332 333 334 335 336 337 338

/*
 * Will slightly overestimate the number of pages needed.  This is OK
 * as it only leads to a small amount of wasted memory for the lifetime of
 * the I/O.
 */
static int nvme_npages(unsigned size, struct nvme_dev *dev)
{
339 340
	unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
				      dev->ctrl.page_size);
341 342 343
	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}

C
Chaitanya Kulkarni 已提交
344 345 346 347 348
/*
 * Calculates the number of pages needed for the SGL segments. For example a 4k
 * page can accommodate 256 SGL descriptors.
 */
static int nvme_pci_npages_sgl(unsigned int num_seg)
349
{
C
Chaitanya Kulkarni 已提交
350
	return DIV_ROUND_UP(num_seg * sizeof(struct nvme_sgl_desc), PAGE_SIZE);
C
Christoph Hellwig 已提交
351
}
352

C
Chaitanya Kulkarni 已提交
353 354
static unsigned int nvme_pci_iod_alloc_size(struct nvme_dev *dev,
		unsigned int size, unsigned int nseg, bool use_sgl)
C
Christoph Hellwig 已提交
355
{
C
Chaitanya Kulkarni 已提交
356 357 358 359 360 361 362 363
	size_t alloc_size;

	if (use_sgl)
		alloc_size = sizeof(__le64 *) * nvme_pci_npages_sgl(nseg);
	else
		alloc_size = sizeof(__le64 *) * nvme_npages(size, dev);

	return alloc_size + sizeof(struct scatterlist) * nseg;
C
Christoph Hellwig 已提交
364
}
365

C
Chaitanya Kulkarni 已提交
366
static unsigned int nvme_pci_cmd_size(struct nvme_dev *dev, bool use_sgl)
C
Christoph Hellwig 已提交
367
{
C
Chaitanya Kulkarni 已提交
368 369 370 371 372
	unsigned int alloc_size = nvme_pci_iod_alloc_size(dev,
				    NVME_INT_BYTES(dev), NVME_INT_PAGES,
				    use_sgl);

	return sizeof(struct nvme_iod) + alloc_size;
373 374
}

M
Matias Bjørling 已提交
375 376
static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
				unsigned int hctx_idx)
377
{
M
Matias Bjørling 已提交
378
	struct nvme_dev *dev = data;
379
	struct nvme_queue *nvmeq = &dev->queues[0];
M
Matias Bjørling 已提交
380

381 382 383 384
	WARN_ON(hctx_idx != 0);
	WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
	WARN_ON(nvmeq->tags);

M
Matias Bjørling 已提交
385
	hctx->driver_data = nvmeq;
386
	nvmeq->tags = &dev->admin_tagset.tags[0];
M
Matias Bjørling 已提交
387
	return 0;
388 389
}

390 391 392 393 394 395 396
static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	nvmeq->tags = NULL;
}

M
Matias Bjørling 已提交
397 398
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			  unsigned int hctx_idx)
M
Matthew Wilcox 已提交
399
{
M
Matias Bjørling 已提交
400
	struct nvme_dev *dev = data;
401
	struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1];
M
Matias Bjørling 已提交
402

403 404
	if (!nvmeq->tags)
		nvmeq->tags = &dev->tagset.tags[hctx_idx];
M
Matthew Wilcox 已提交
405

406
	WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
M
Matias Bjørling 已提交
407 408
	hctx->driver_data = nvmeq;
	return 0;
M
Matthew Wilcox 已提交
409 410
}

411 412
static int nvme_init_request(struct blk_mq_tag_set *set, struct request *req,
		unsigned int hctx_idx, unsigned int numa_node)
M
Matthew Wilcox 已提交
413
{
414
	struct nvme_dev *dev = set->driver_data;
C
Christoph Hellwig 已提交
415
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
416
	int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0;
417
	struct nvme_queue *nvmeq = &dev->queues[queue_idx];
M
Matias Bjørling 已提交
418 419

	BUG_ON(!nvmeq);
C
Christoph Hellwig 已提交
420
	iod->nvmeq = nvmeq;
421 422

	nvme_req(req)->ctrl = &dev->ctrl;
M
Matias Bjørling 已提交
423 424 425
	return 0;
}

426 427 428 429
static int nvme_pci_map_queues(struct blk_mq_tag_set *set)
{
	struct nvme_dev *dev = set->driver_data;

430 431
	return blk_mq_pci_map_queues(set, to_pci_dev(dev->dev),
			dev->num_vecs > 1 ? 1 /* admin queue */ : 0);
432 433
}

M
Matthew Wilcox 已提交
434
/**
435
 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
M
Matthew Wilcox 已提交
436 437 438
 * @nvmeq: The queue to use
 * @cmd: The command to send
 */
439
static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
M
Matthew Wilcox 已提交
440
{
441
	spin_lock(&nvmeq->sq_lock);
442
	if (nvmeq->sq_cmds_io)
443 444
		memcpy_toio(&nvmeq->sq_cmds_io[nvmeq->sq_tail], cmd,
				sizeof(*cmd));
445
	else
446
		memcpy(&nvmeq->sq_cmds[nvmeq->sq_tail], cmd, sizeof(*cmd));
447

448 449 450 451 452 453
	if (++nvmeq->sq_tail == nvmeq->q_depth)
		nvmeq->sq_tail = 0;
	if (nvme_dbbuf_update_and_check_event(nvmeq->sq_tail,
			nvmeq->dbbuf_sq_db, nvmeq->dbbuf_sq_ei))
		writel(nvmeq->sq_tail, nvmeq->q_db);
	spin_unlock(&nvmeq->sq_lock);
M
Matthew Wilcox 已提交
454 455
}

C
Chaitanya Kulkarni 已提交
456
static void **nvme_pci_iod_list(struct request *req)
M
Matthew Wilcox 已提交
457
{
C
Christoph Hellwig 已提交
458
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
C
Chaitanya Kulkarni 已提交
459
	return (void **)(iod->sg + blk_rq_nr_phys_segments(req));
M
Matthew Wilcox 已提交
460 461
}

462 463 464
static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
465
	int nseg = blk_rq_nr_phys_segments(req);
466 467
	unsigned int avg_seg_size;

468 469 470 471
	if (nseg == 0)
		return false;

	avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg);
472 473 474 475 476 477 478 479 480 481

	if (!(dev->ctrl.sgls & ((1 << 0) | (1 << 1))))
		return false;
	if (!iod->nvmeq->qid)
		return false;
	if (!sgl_threshold || avg_seg_size < sgl_threshold)
		return false;
	return true;
}

482
static blk_status_t nvme_init_iod(struct request *rq, struct nvme_dev *dev)
483
{
C
Christoph Hellwig 已提交
484
	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq);
485
	int nseg = blk_rq_nr_phys_segments(rq);
486
	unsigned int size = blk_rq_payload_bytes(rq);
487

488 489
	iod->use_sgl = nvme_pci_use_sgls(dev, rq);

C
Christoph Hellwig 已提交
490
	if (nseg > NVME_INT_PAGES || size > NVME_INT_BYTES(dev)) {
491
		iod->sg = mempool_alloc(dev->iod_mempool, GFP_ATOMIC);
C
Christoph Hellwig 已提交
492
		if (!iod->sg)
493
			return BLK_STS_RESOURCE;
C
Christoph Hellwig 已提交
494 495
	} else {
		iod->sg = iod->inline_sg;
496 497
	}

C
Christoph Hellwig 已提交
498 499 500 501
	iod->aborted = 0;
	iod->npages = -1;
	iod->nents = 0;
	iod->length = size;
K
Keith Busch 已提交
502

503
	return BLK_STS_OK;
504 505
}

C
Christoph Hellwig 已提交
506
static void nvme_free_iod(struct nvme_dev *dev, struct request *req)
M
Matthew Wilcox 已提交
507
{
C
Christoph Hellwig 已提交
508
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
C
Chaitanya Kulkarni 已提交
509 510 511
	const int last_prp = dev->ctrl.page_size / sizeof(__le64) - 1;
	dma_addr_t dma_addr = iod->first_dma, next_dma_addr;

512 513 514
	int i;

	if (iod->npages == 0)
C
Chaitanya Kulkarni 已提交
515 516 517
		dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
			dma_addr);

518
	for (i = 0; i < iod->npages; i++) {
C
Chaitanya Kulkarni 已提交
519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
		void *addr = nvme_pci_iod_list(req)[i];

		if (iod->use_sgl) {
			struct nvme_sgl_desc *sg_list = addr;

			next_dma_addr =
			    le64_to_cpu((sg_list[SGES_PER_PAGE - 1]).addr);
		} else {
			__le64 *prp_list = addr;

			next_dma_addr = le64_to_cpu(prp_list[last_prp]);
		}

		dma_pool_free(dev->prp_page_pool, addr, dma_addr);
		dma_addr = next_dma_addr;
534
	}
535

C
Christoph Hellwig 已提交
536
	if (iod->sg != iod->inline_sg)
537
		mempool_free(iod->sg, dev->iod_mempool);
K
Keith Busch 已提交
538 539
}

540 541 542 543 544 545 546 547 548 549 550 551 552 553
static void nvme_print_sgl(struct scatterlist *sgl, int nents)
{
	int i;
	struct scatterlist *sg;

	for_each_sg(sgl, sg, nents, i) {
		dma_addr_t phys = sg_phys(sg);
		pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d "
			"dma_address:%pad dma_length:%d\n",
			i, &phys, sg->offset, sg->length, &sg_dma_address(sg),
			sg_dma_len(sg));
	}
}

C
Chaitanya Kulkarni 已提交
554 555
static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
		struct request *req, struct nvme_rw_command *cmnd)
M
Matthew Wilcox 已提交
556
{
C
Christoph Hellwig 已提交
557
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
558
	struct dma_pool *pool;
559
	int length = blk_rq_payload_bytes(req);
560
	struct scatterlist *sg = iod->sg;
M
Matthew Wilcox 已提交
561 562
	int dma_len = sg_dma_len(sg);
	u64 dma_addr = sg_dma_address(sg);
563
	u32 page_size = dev->ctrl.page_size;
564
	int offset = dma_addr & (page_size - 1);
565
	__le64 *prp_list;
C
Chaitanya Kulkarni 已提交
566
	void **list = nvme_pci_iod_list(req);
567
	dma_addr_t prp_dma;
568
	int nprps, i;
M
Matthew Wilcox 已提交
569

570
	length -= (page_size - offset);
571 572
	if (length <= 0) {
		iod->first_dma = 0;
C
Chaitanya Kulkarni 已提交
573
		goto done;
574
	}
M
Matthew Wilcox 已提交
575

576
	dma_len -= (page_size - offset);
M
Matthew Wilcox 已提交
577
	if (dma_len) {
578
		dma_addr += (page_size - offset);
M
Matthew Wilcox 已提交
579 580 581 582 583 584
	} else {
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
	}

585
	if (length <= page_size) {
586
		iod->first_dma = dma_addr;
C
Chaitanya Kulkarni 已提交
587
		goto done;
588 589
	}

590
	nprps = DIV_ROUND_UP(length, page_size);
591 592
	if (nprps <= (256 / 8)) {
		pool = dev->prp_small_pool;
593
		iod->npages = 0;
594 595
	} else {
		pool = dev->prp_page_pool;
596
		iod->npages = 1;
597 598
	}

599
	prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
600
	if (!prp_list) {
601
		iod->first_dma = dma_addr;
602
		iod->npages = -1;
603
		return BLK_STS_RESOURCE;
604
	}
605 606
	list[0] = prp_list;
	iod->first_dma = prp_dma;
607 608
	i = 0;
	for (;;) {
609
		if (i == page_size >> 3) {
610
			__le64 *old_prp_list = prp_list;
611
			prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
612
			if (!prp_list)
613
				return BLK_STS_RESOURCE;
614
			list[iod->npages++] = prp_list;
615 616 617
			prp_list[0] = old_prp_list[i - 1];
			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
			i = 1;
618 619
		}
		prp_list[i++] = cpu_to_le64(dma_addr);
620 621 622
		dma_len -= page_size;
		dma_addr += page_size;
		length -= page_size;
623 624 625 626
		if (length <= 0)
			break;
		if (dma_len > 0)
			continue;
627 628
		if (unlikely(dma_len < 0))
			goto bad_sgl;
629 630 631
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
M
Matthew Wilcox 已提交
632 633
	}

C
Chaitanya Kulkarni 已提交
634 635 636 637
done:
	cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
	cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);

638 639 640
	return BLK_STS_OK;

 bad_sgl:
641 642 643
	WARN(DO_ONCE(nvme_print_sgl, iod->sg, iod->nents),
			"Invalid SGL for payload:%d nents:%d\n",
			blk_rq_payload_bytes(req), iod->nents);
644
	return BLK_STS_IOERR;
M
Matthew Wilcox 已提交
645 646
}

C
Chaitanya Kulkarni 已提交
647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668
static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge,
		struct scatterlist *sg)
{
	sge->addr = cpu_to_le64(sg_dma_address(sg));
	sge->length = cpu_to_le32(sg_dma_len(sg));
	sge->type = NVME_SGL_FMT_DATA_DESC << 4;
}

static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge,
		dma_addr_t dma_addr, int entries)
{
	sge->addr = cpu_to_le64(dma_addr);
	if (entries < SGES_PER_PAGE) {
		sge->length = cpu_to_le32(entries * sizeof(*sge));
		sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4;
	} else {
		sge->length = cpu_to_le32(PAGE_SIZE);
		sge->type = NVME_SGL_FMT_SEG_DESC << 4;
	}
}

static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
669
		struct request *req, struct nvme_rw_command *cmd, int entries)
C
Chaitanya Kulkarni 已提交
670 671 672 673 674 675
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct dma_pool *pool;
	struct nvme_sgl_desc *sg_list;
	struct scatterlist *sg = iod->sg;
	dma_addr_t sgl_dma;
676
	int i = 0;
C
Chaitanya Kulkarni 已提交
677 678 679 680

	/* setting the transfer type as SGL */
	cmd->flags = NVME_CMD_SGL_METABUF;

681
	if (entries == 1) {
C
Chaitanya Kulkarni 已提交
682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721
		nvme_pci_sgl_set_data(&cmd->dptr.sgl, sg);
		return BLK_STS_OK;
	}

	if (entries <= (256 / sizeof(struct nvme_sgl_desc))) {
		pool = dev->prp_small_pool;
		iod->npages = 0;
	} else {
		pool = dev->prp_page_pool;
		iod->npages = 1;
	}

	sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
	if (!sg_list) {
		iod->npages = -1;
		return BLK_STS_RESOURCE;
	}

	nvme_pci_iod_list(req)[0] = sg_list;
	iod->first_dma = sgl_dma;

	nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries);

	do {
		if (i == SGES_PER_PAGE) {
			struct nvme_sgl_desc *old_sg_desc = sg_list;
			struct nvme_sgl_desc *link = &old_sg_desc[i - 1];

			sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
			if (!sg_list)
				return BLK_STS_RESOURCE;

			i = 0;
			nvme_pci_iod_list(req)[iod->npages++] = sg_list;
			sg_list[i++] = *link;
			nvme_pci_sgl_set_seg(link, sgl_dma, entries);
		}

		nvme_pci_sgl_set_data(&sg_list[i++], sg);
		sg = sg_next(sg);
722
	} while (--entries > 0);
C
Chaitanya Kulkarni 已提交
723 724 725 726

	return BLK_STS_OK;
}

727
static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
728
		struct nvme_command *cmnd)
729
{
C
Christoph Hellwig 已提交
730
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
C
Christoph Hellwig 已提交
731 732 733
	struct request_queue *q = req->q;
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;
734
	blk_status_t ret = BLK_STS_IOERR;
735
	int nr_mapped;
736

737
	sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
C
Christoph Hellwig 已提交
738 739 740
	iod->nents = blk_rq_map_sg(q, req, iod->sg);
	if (!iod->nents)
		goto out;
741

742
	ret = BLK_STS_RESOURCE;
743 744 745
	nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, dma_dir,
			DMA_ATTR_NO_WARN);
	if (!nr_mapped)
C
Christoph Hellwig 已提交
746
		goto out;
747

748
	if (iod->use_sgl)
749
		ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw, nr_mapped);
C
Chaitanya Kulkarni 已提交
750 751 752
	else
		ret = nvme_pci_setup_prps(dev, req, &cmnd->rw);

753
	if (ret != BLK_STS_OK)
C
Christoph Hellwig 已提交
754
		goto out_unmap;
755

756
	ret = BLK_STS_IOERR;
C
Christoph Hellwig 已提交
757 758 759
	if (blk_integrity_rq(req)) {
		if (blk_rq_count_integrity_sg(q, req->bio) != 1)
			goto out_unmap;
760

761 762
		sg_init_table(&iod->meta_sg, 1);
		if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1)
C
Christoph Hellwig 已提交
763
			goto out_unmap;
764

765
		if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir))
C
Christoph Hellwig 已提交
766
			goto out_unmap;
767
	}
M
Matthew Wilcox 已提交
768

C
Christoph Hellwig 已提交
769
	if (blk_integrity_rq(req))
770
		cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg));
771
	return BLK_STS_OK;
M
Matthew Wilcox 已提交
772

C
Christoph Hellwig 已提交
773 774 775 776
out_unmap:
	dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
out:
	return ret;
M
Matthew Wilcox 已提交
777 778
}

C
Christoph Hellwig 已提交
779
static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
M
Matthew Wilcox 已提交
780
{
C
Christoph Hellwig 已提交
781
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
782 783 784 785 786
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;

	if (iod->nents) {
		dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
787
		if (blk_integrity_rq(req))
788
			dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir);
789
	}
K
Keith Busch 已提交
790

791
	nvme_cleanup_cmd(req);
C
Christoph Hellwig 已提交
792
	nvme_free_iod(dev, req);
793
}
M
Matthew Wilcox 已提交
794

795 796 797
/*
 * NOTE: ns is NULL when called on the admin queue.
 */
798
static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
M
Matias Bjørling 已提交
799
			 const struct blk_mq_queue_data *bd)
800
{
M
Matias Bjørling 已提交
801 802
	struct nvme_ns *ns = hctx->queue->queuedata;
	struct nvme_queue *nvmeq = hctx->driver_data;
803
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
804
	struct request *req = bd->rq;
C
Christoph Hellwig 已提交
805
	struct nvme_command cmnd;
806
	blk_status_t ret;
K
Keith Busch 已提交
807

808 809 810 811 812 813 814
	/*
	 * We should not need to do this, but we're still using this to
	 * ensure we can drain requests on a dying queue.
	 */
	if (unlikely(nvmeq->cq_vector < 0))
		return BLK_STS_IOERR;

815
	ret = nvme_setup_cmd(ns, req, &cmnd);
816
	if (ret)
C
Christoph Hellwig 已提交
817
		return ret;
M
Matias Bjørling 已提交
818

819
	ret = nvme_init_iod(req, dev);
820
	if (ret)
821
		goto out_free_cmd;
M
Matias Bjørling 已提交
822

823
	if (blk_rq_nr_phys_segments(req)) {
824
		ret = nvme_map_data(dev, req, &cmnd);
825 826 827
		if (ret)
			goto out_cleanup_iod;
	}
M
Matias Bjørling 已提交
828

829
	blk_mq_start_request(req);
830
	nvme_submit_cmd(nvmeq, &cmnd);
831
	return BLK_STS_OK;
832
out_cleanup_iod:
C
Christoph Hellwig 已提交
833
	nvme_free_iod(dev, req);
834 835
out_free_cmd:
	nvme_cleanup_cmd(req);
C
Christoph Hellwig 已提交
836
	return ret;
M
Matthew Wilcox 已提交
837
}
K
Keith Busch 已提交
838

839
static void nvme_pci_complete_rq(struct request *req)
840
{
C
Christoph Hellwig 已提交
841
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
M
Matias Bjørling 已提交
842

843 844
	nvme_unmap_data(iod->nvmeq->dev, req);
	nvme_complete_rq(req);
M
Matthew Wilcox 已提交
845 846
}

847
/* We read the CQE phase first to check if the rest of the entry is valid */
848
static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
849
{
850 851
	return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
			nvmeq->cq_phase;
852 853
}

854
static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
855
{
856
	u16 head = nvmeq->cq_head;
857

858 859 860
	if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
					      nvmeq->dbbuf_cq_ei))
		writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
861
}
862

863
static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
864
{
865
	volatile struct nvme_completion *cqe = &nvmeq->cqes[idx];
866
	struct request *req;
867

868 869 870 871 872
	if (unlikely(cqe->command_id >= nvmeq->q_depth)) {
		dev_warn(nvmeq->dev->ctrl.device,
			"invalid id %d completed on queue %d\n",
			cqe->command_id, le16_to_cpu(cqe->sq_id));
		return;
M
Matthew Wilcox 已提交
873 874
	}

875 876 877 878 879 880 881
	/*
	 * AEN requests are special as they don't time out and can
	 * survive any kind of queue freeze and often don't respond to
	 * aborts.  We don't even bother to allocate a struct request
	 * for them but rather special case them here.
	 */
	if (unlikely(nvmeq->qid == 0 &&
K
Keith Busch 已提交
882
			cqe->command_id >= NVME_AQ_BLK_MQ_DEPTH)) {
883 884
		nvme_complete_async_event(&nvmeq->dev->ctrl,
				cqe->status, &cqe->result);
J
Jens Axboe 已提交
885
		return;
886
	}
M
Matthew Wilcox 已提交
887

888 889 890
	req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
	nvme_end_request(req, cqe->status, cqe->result);
}
M
Matthew Wilcox 已提交
891

892
static void nvme_complete_cqes(struct nvme_queue *nvmeq, u16 start, u16 end)
M
Matthew Wilcox 已提交
893
{
894 895 896 897 898 899
	while (start != end) {
		nvme_handle_cqe(nvmeq, start);
		if (++start == nvmeq->q_depth)
			start = 0;
	}
}
900

901 902 903 904 905
static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
{
	if (++nvmeq->cq_head == nvmeq->q_depth) {
		nvmeq->cq_head = 0;
		nvmeq->cq_phase = !nvmeq->cq_phase;
M
Matthew Wilcox 已提交
906
	}
J
Jens Axboe 已提交
907 908
}

909 910
static inline bool nvme_process_cq(struct nvme_queue *nvmeq, u16 *start,
		u16 *end, int tag)
J
Jens Axboe 已提交
911
{
912
	bool found = false;
M
Matthew Wilcox 已提交
913

914 915 916 917 918
	*start = nvmeq->cq_head;
	while (!found && nvme_cqe_pending(nvmeq)) {
		if (nvmeq->cqes[nvmeq->cq_head].command_id == tag)
			found = true;
		nvme_update_cq_head(nvmeq);
919
	}
920
	*end = nvmeq->cq_head;
921

922
	if (*start != *end)
923
		nvme_ring_cq_doorbell(nvmeq);
924
	return found;
M
Matthew Wilcox 已提交
925 926 927
}

static irqreturn_t nvme_irq(int irq, void *data)
928 929
{
	struct nvme_queue *nvmeq = data;
930
	irqreturn_t ret = IRQ_NONE;
931 932
	u16 start, end;

933
	spin_lock(&nvmeq->cq_lock);
934 935
	if (nvmeq->cq_head != nvmeq->last_cq_head)
		ret = IRQ_HANDLED;
936
	nvme_process_cq(nvmeq, &start, &end, -1);
937
	nvmeq->last_cq_head = nvmeq->cq_head;
938
	spin_unlock(&nvmeq->cq_lock);
939

940 941 942 943 944 945
	if (start != end) {
		nvme_complete_cqes(nvmeq, start, end);
		return IRQ_HANDLED;
	}

	return ret;
946 947 948 949 950
}

static irqreturn_t nvme_irq_check(int irq, void *data)
{
	struct nvme_queue *nvmeq = data;
951
	if (nvme_cqe_pending(nvmeq))
952 953
		return IRQ_WAKE_THREAD;
	return IRQ_NONE;
954 955
}

K
Keith Busch 已提交
956
static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag)
J
Jens Axboe 已提交
957
{
958 959
	u16 start, end;
	bool found;
J
Jens Axboe 已提交
960

961
	if (!nvme_cqe_pending(nvmeq))
962
		return 0;
J
Jens Axboe 已提交
963

964
	spin_lock_irq(&nvmeq->cq_lock);
965
	found = nvme_process_cq(nvmeq, &start, &end, tag);
966
	spin_unlock_irq(&nvmeq->cq_lock);
967

968
	nvme_complete_cqes(nvmeq, start, end);
969
	return found;
J
Jens Axboe 已提交
970 971
}

K
Keith Busch 已提交
972 973 974 975 976 977 978
static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	return __nvme_poll(nvmeq, tag);
}

979
static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl)
M
Matthew Wilcox 已提交
980
{
981
	struct nvme_dev *dev = to_nvme_dev(ctrl);
982
	struct nvme_queue *nvmeq = &dev->queues[0];
M
Matias Bjørling 已提交
983
	struct nvme_command c;
M
Matthew Wilcox 已提交
984

M
Matias Bjørling 已提交
985 986
	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_async_event;
987
	c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
988
	nvme_submit_cmd(nvmeq, &c);
989 990
}

M
Matthew Wilcox 已提交
991
static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
992
{
M
Matthew Wilcox 已提交
993 994 995 996 997 998
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(id);

999
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1000 1001 1002
}

static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
1003
		struct nvme_queue *nvmeq, s16 vector)
M
Matthew Wilcox 已提交
1004 1005 1006 1007
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;

1008
	/*
M
Minwoo Im 已提交
1009
	 * Note: we (ab)use the fact that the prp fields survive if no data
1010 1011
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
1012 1013 1014 1015 1016 1017
	memset(&c, 0, sizeof(c));
	c.create_cq.opcode = nvme_admin_create_cq;
	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
	c.create_cq.cqid = cpu_to_le16(qid);
	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_cq.cq_flags = cpu_to_le16(flags);
1018
	c.create_cq.irq_vector = cpu_to_le16(vector);
M
Matthew Wilcox 已提交
1019

1020
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1021 1022 1023 1024 1025
}

static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
1026
	struct nvme_ctrl *ctrl = &dev->ctrl;
M
Matthew Wilcox 已提交
1027
	struct nvme_command c;
1028
	int flags = NVME_QUEUE_PHYS_CONTIG;
M
Matthew Wilcox 已提交
1029

1030 1031 1032 1033 1034 1035 1036 1037
	/*
	 * Some drives have a bug that auto-enables WRRU if MEDIUM isn't
	 * set. Since URGENT priority is zeroes, it makes all queues
	 * URGENT.
	 */
	if (ctrl->quirks & NVME_QUIRK_MEDIUM_PRIO_SQ)
		flags |= NVME_SQ_PRIO_MEDIUM;

1038
	/*
M
Minwoo Im 已提交
1039
	 * Note: we (ab)use the fact that the prp fields survive if no data
1040 1041
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
1042 1043 1044 1045 1046 1047 1048 1049
	memset(&c, 0, sizeof(c));
	c.create_sq.opcode = nvme_admin_create_sq;
	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
	c.create_sq.sqid = cpu_to_le16(qid);
	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_sq.sq_flags = cpu_to_le16(flags);
	c.create_sq.cqid = cpu_to_le16(qid);

1050
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062
}

static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
}

static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
}

1063
static void abort_endio(struct request *req, blk_status_t error)
1064
{
C
Christoph Hellwig 已提交
1065 1066
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = iod->nvmeq;
1067

1068 1069
	dev_warn(nvmeq->dev->ctrl.device,
		 "Abort status: 0x%x", nvme_req(req)->status);
1070 1071
	atomic_inc(&nvmeq->dev->ctrl.abort_limit);
	blk_mq_free_request(req);
1072 1073
}

K
Keith Busch 已提交
1074 1075 1076 1077 1078 1079 1080 1081
static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
{

	/* If true, indicates loss of adapter communication, possibly by a
	 * NVMe Subsystem reset.
	 */
	bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);

1082 1083 1084
	/* If there is a reset/reinit ongoing, we shouldn't reset again. */
	switch (dev->ctrl.state) {
	case NVME_CTRL_RESETTING:
1085
	case NVME_CTRL_CONNECTING:
K
Keith Busch 已提交
1086
		return false;
1087 1088 1089
	default:
		break;
	}
K
Keith Busch 已提交
1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117

	/* We shouldn't reset unless the controller is on fatal error state
	 * _or_ if we lost the communication with it.
	 */
	if (!(csts & NVME_CSTS_CFS) && !nssro)
		return false;

	return true;
}

static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
{
	/* Read a config register to help see what died. */
	u16 pci_status;
	int result;

	result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
				      &pci_status);
	if (result == PCIBIOS_SUCCESSFUL)
		dev_warn(dev->ctrl.device,
			 "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
			 csts, pci_status);
	else
		dev_warn(dev->ctrl.device,
			 "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
			 csts, result);
}

1118
static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
K
Keith Busch 已提交
1119
{
C
Christoph Hellwig 已提交
1120 1121
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = iod->nvmeq;
K
Keith Busch 已提交
1122
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
1123 1124
	struct request *abort_req;
	struct nvme_command cmd;
K
Keith Busch 已提交
1125 1126
	u32 csts = readl(dev->bar + NVME_REG_CSTS);

W
Wen Xiong 已提交
1127 1128 1129 1130 1131 1132 1133
	/* If PCI error recovery process is happening, we cannot reset or
	 * the recovery mechanism will surely fail.
	 */
	mb();
	if (pci_channel_offline(to_pci_dev(dev->dev)))
		return BLK_EH_RESET_TIMER;

K
Keith Busch 已提交
1134 1135 1136 1137 1138 1139
	/*
	 * Reset immediately if the controller is failed
	 */
	if (nvme_should_reset(dev, csts)) {
		nvme_warn_reset(dev, csts);
		nvme_dev_disable(dev, false);
1140
		nvme_reset_ctrl(&dev->ctrl);
1141
		return BLK_EH_DONE;
K
Keith Busch 已提交
1142
	}
K
Keith Busch 已提交
1143

K
Keith Busch 已提交
1144 1145 1146 1147 1148 1149 1150
	/*
	 * Did we miss an interrupt?
	 */
	if (__nvme_poll(nvmeq, req->tag)) {
		dev_warn(dev->ctrl.device,
			 "I/O %d QID %d timeout, completion polled\n",
			 req->tag, nvmeq->qid);
1151
		return BLK_EH_DONE;
K
Keith Busch 已提交
1152 1153
	}

1154
	/*
1155 1156 1157
	 * Shutdown immediately if controller times out while starting. The
	 * reset work will see the pci device disabled when it gets the forced
	 * cancellation error. All outstanding requests are completed on
1158
	 * shutdown, so we return BLK_EH_DONE.
1159
	 */
1160 1161 1162
	switch (dev->ctrl.state) {
	case NVME_CTRL_CONNECTING:
	case NVME_CTRL_RESETTING:
1163
		dev_warn_ratelimited(dev->ctrl.device,
1164 1165
			 "I/O %d QID %d timeout, disable controller\n",
			 req->tag, nvmeq->qid);
1166
		nvme_dev_disable(dev, false);
1167
		nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1168
		return BLK_EH_DONE;
1169 1170
	default:
		break;
K
Keith Busch 已提交
1171 1172
	}

1173 1174 1175 1176
	/*
 	 * Shutdown the controller immediately and schedule a reset if the
 	 * command was already aborted once before and still hasn't been
 	 * returned to the driver, or if this is the admin queue.
1177
	 */
C
Christoph Hellwig 已提交
1178
	if (!nvmeq->qid || iod->aborted) {
1179
		dev_warn(dev->ctrl.device,
1180 1181
			 "I/O %d QID %d timeout, reset controller\n",
			 req->tag, nvmeq->qid);
1182
		nvme_dev_disable(dev, false);
1183
		nvme_reset_ctrl(&dev->ctrl);
K
Keith Busch 已提交
1184

1185
		nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1186
		return BLK_EH_DONE;
K
Keith Busch 已提交
1187 1188
	}

1189
	if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1190
		atomic_inc(&dev->ctrl.abort_limit);
1191
		return BLK_EH_RESET_TIMER;
1192
	}
1193
	iod->aborted = 1;
M
Matias Bjørling 已提交
1194

K
Keith Busch 已提交
1195 1196
	memset(&cmd, 0, sizeof(cmd));
	cmd.abort.opcode = nvme_admin_abort_cmd;
M
Matias Bjørling 已提交
1197
	cmd.abort.cid = req->tag;
K
Keith Busch 已提交
1198 1199
	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);

1200 1201 1202
	dev_warn(nvmeq->dev->ctrl.device,
		"I/O %d QID %d timeout, aborting\n",
		 req->tag, nvmeq->qid);
1203 1204

	abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
1205
			BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
1206 1207 1208 1209 1210 1211 1212 1213
	if (IS_ERR(abort_req)) {
		atomic_inc(&dev->ctrl.abort_limit);
		return BLK_EH_RESET_TIMER;
	}

	abort_req->timeout = ADMIN_TIMEOUT;
	abort_req->end_io_data = NULL;
	blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
K
Keith Busch 已提交
1214

1215 1216 1217 1218 1219 1220
	/*
	 * The aborted req will be completed on receiving the abort req.
	 * We enable the timer again. If hit twice, it'll cause a device reset,
	 * as the device then is in a faulty state.
	 */
	return BLK_EH_RESET_TIMER;
K
Keith Busch 已提交
1221 1222
}

M
Matias Bjørling 已提交
1223 1224
static void nvme_free_queue(struct nvme_queue *nvmeq)
{
1225 1226
	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1227 1228
	if (nvmeq->sq_cmds)
		dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1229 1230 1231
					nvmeq->sq_cmds, nvmeq->sq_dma_addr);
}

1232
static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1233 1234 1235
{
	int i;

1236 1237
	for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
		dev->ctrl.queue_count--;
1238
		nvme_free_queue(&dev->queues[i]);
1239
	}
1240 1241
}

K
Keith Busch 已提交
1242 1243 1244 1245 1246
/**
 * nvme_suspend_queue - put queue into suspended state
 * @nvmeq - queue to suspend
 */
static int nvme_suspend_queue(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
1247
{
K
Keith Busch 已提交
1248
	int vector;
M
Matthew Wilcox 已提交
1249

1250
	spin_lock_irq(&nvmeq->cq_lock);
K
Keith Busch 已提交
1251
	if (nvmeq->cq_vector == -1) {
1252
		spin_unlock_irq(&nvmeq->cq_lock);
K
Keith Busch 已提交
1253 1254
		return 1;
	}
1255
	vector = nvmeq->cq_vector;
K
Keith Busch 已提交
1256
	nvmeq->dev->online_queues--;
K
Keith Busch 已提交
1257
	nvmeq->cq_vector = -1;
1258
	spin_unlock_irq(&nvmeq->cq_lock);
1259

1260 1261 1262 1263 1264
	/*
	 * Ensure that nvme_queue_rq() sees it ->cq_vector == -1 without
	 * having to grab the lock.
	 */
	mb();
1265

1266
	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1267
		blk_mq_quiesce_queue(nvmeq->dev->ctrl.admin_q);
1268

1269
	pci_free_irq(to_pci_dev(nvmeq->dev->dev), vector, nvmeq);
M
Matthew Wilcox 已提交
1270

K
Keith Busch 已提交
1271 1272
	return 0;
}
M
Matthew Wilcox 已提交
1273

1274
static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
K
Keith Busch 已提交
1275
{
1276
	struct nvme_queue *nvmeq = &dev->queues[0];
1277
	u16 start, end;
K
Keith Busch 已提交
1278

1279 1280 1281
	if (shutdown)
		nvme_shutdown_ctrl(&dev->ctrl);
	else
1282
		nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
1283

1284
	spin_lock_irq(&nvmeq->cq_lock);
1285
	nvme_process_cq(nvmeq, &start, &end, -1);
1286
	spin_unlock_irq(&nvmeq->cq_lock);
1287 1288

	nvme_complete_cqes(nvmeq, start, end);
M
Matthew Wilcox 已提交
1289 1290
}

1291 1292 1293 1294
static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
				int entry_size)
{
	int q_depth = dev->q_depth;
1295 1296
	unsigned q_size_aligned = roundup(q_depth * entry_size,
					  dev->ctrl.page_size);
1297 1298

	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1299
		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1300
		mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
1301
		q_depth = div_u64(mem_per_q, entry_size);
1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317

		/*
		 * Ensure the reduced q_depth is above some threshold where it
		 * would be better to map queues in system memory with the
		 * original depth
		 */
		if (q_depth < 64)
			return -ENOMEM;
	}

	return q_depth;
}

static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
				int qid, int depth)
{
1318 1319 1320
	/* CMB SQEs will be mapped before creation */
	if (qid && dev->cmb && use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS))
		return 0;
1321

1322 1323 1324 1325
	nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
					    &nvmeq->sq_dma_addr, GFP_KERNEL);
	if (!nvmeq->sq_cmds)
		return -ENOMEM;
1326 1327 1328
	return 0;
}

1329
static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
M
Matthew Wilcox 已提交
1330
{
1331
	struct nvme_queue *nvmeq = &dev->queues[qid];
M
Matthew Wilcox 已提交
1332

1333 1334
	if (dev->ctrl.queue_count > qid)
		return 0;
M
Matthew Wilcox 已提交
1335

1336
	nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
J
Joe Perches 已提交
1337
					  &nvmeq->cq_dma_addr, GFP_KERNEL);
M
Matthew Wilcox 已提交
1338 1339 1340
	if (!nvmeq->cqes)
		goto free_nvmeq;

1341
	if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
M
Matthew Wilcox 已提交
1342 1343
		goto free_cqdma;

1344
	nvmeq->q_dmadev = dev->dev;
M
Matthew Wilcox 已提交
1345
	nvmeq->dev = dev;
1346 1347
	spin_lock_init(&nvmeq->sq_lock);
	spin_lock_init(&nvmeq->cq_lock);
M
Matthew Wilcox 已提交
1348
	nvmeq->cq_head = 0;
M
Matthew Wilcox 已提交
1349
	nvmeq->cq_phase = 1;
1350
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
M
Matthew Wilcox 已提交
1351
	nvmeq->q_depth = depth;
K
Keith Busch 已提交
1352
	nvmeq->qid = qid;
1353
	nvmeq->cq_vector = -1;
1354
	dev->ctrl.queue_count++;
1355

1356
	return 0;
M
Matthew Wilcox 已提交
1357 1358

 free_cqdma:
1359
	dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
M
Matthew Wilcox 已提交
1360 1361
							nvmeq->cq_dma_addr);
 free_nvmeq:
1362
	return -ENOMEM;
M
Matthew Wilcox 已提交
1363 1364
}

1365
static int queue_request_irq(struct nvme_queue *nvmeq)
1366
{
1367 1368 1369 1370 1371 1372 1373 1374 1375 1376
	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
	int nr = nvmeq->dev->ctrl.instance;

	if (use_threaded_interrupts) {
		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
				nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
	} else {
		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
				NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
	}
1377 1378
}

1379
static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
M
Matthew Wilcox 已提交
1380
{
1381
	struct nvme_dev *dev = nvmeq->dev;
M
Matthew Wilcox 已提交
1382

1383
	spin_lock_irq(&nvmeq->cq_lock);
1384 1385 1386
	nvmeq->sq_tail = 0;
	nvmeq->cq_head = 0;
	nvmeq->cq_phase = 1;
1387
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1388
	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
1389
	nvme_dbbuf_init(dev, nvmeq, qid);
K
Keith Busch 已提交
1390
	dev->online_queues++;
1391
	spin_unlock_irq(&nvmeq->cq_lock);
1392 1393 1394 1395 1396 1397
}

static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
{
	struct nvme_dev *dev = nvmeq->dev;
	int result;
1398
	s16 vector;
1399

1400 1401 1402 1403 1404 1405 1406
	if (dev->cmb && use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) {
		unsigned offset = (qid - 1) * roundup(SQ_SIZE(nvmeq->q_depth),
						      dev->ctrl.page_size);
		nvmeq->sq_dma_addr = dev->cmb_bus_addr + offset;
		nvmeq->sq_cmds_io = dev->cmb + offset;
	}

1407 1408 1409 1410
	/*
	 * A queue's vector matches the queue identifier unless the controller
	 * has only one vector available.
	 */
1411 1412
	vector = dev->num_vecs == 1 ? 0 : qid;
	result = adapter_alloc_cq(dev, qid, nvmeq, vector);
K
Keith Busch 已提交
1413 1414
	if (result)
		return result;
M
Matthew Wilcox 已提交
1415 1416 1417

	result = adapter_alloc_sq(dev, qid, nvmeq);
	if (result < 0)
K
Keith Busch 已提交
1418 1419
		return result;
	else if (result)
M
Matthew Wilcox 已提交
1420 1421
		goto release_cq;

1422 1423 1424 1425 1426 1427
	/*
	 * Set cq_vector after alloc cq/sq, otherwise nvme_suspend_queue will
	 * invoke free_irq for it and cause a 'Trying to free already-free IRQ
	 * xxx' warning if the create CQ/SQ command times out.
	 */
	nvmeq->cq_vector = vector;
1428
	nvme_init_queue(nvmeq, qid);
1429
	result = queue_request_irq(nvmeq);
M
Matthew Wilcox 已提交
1430 1431 1432
	if (result < 0)
		goto release_sq;

1433
	return result;
M
Matthew Wilcox 已提交
1434

1435 1436
release_sq:
	nvmeq->cq_vector = -1;
1437
	dev->online_queues--;
M
Matthew Wilcox 已提交
1438
	adapter_delete_sq(dev, qid);
1439
release_cq:
M
Matthew Wilcox 已提交
1440
	adapter_delete_cq(dev, qid);
1441
	return result;
M
Matthew Wilcox 已提交
1442 1443
}

1444
static const struct blk_mq_ops nvme_mq_admin_ops = {
1445
	.queue_rq	= nvme_queue_rq,
1446
	.complete	= nvme_pci_complete_rq,
M
Matias Bjørling 已提交
1447
	.init_hctx	= nvme_admin_init_hctx,
1448
	.exit_hctx      = nvme_admin_exit_hctx,
1449
	.init_request	= nvme_init_request,
M
Matias Bjørling 已提交
1450 1451 1452
	.timeout	= nvme_timeout,
};

1453
static const struct blk_mq_ops nvme_mq_ops = {
M
Matias Bjørling 已提交
1454
	.queue_rq	= nvme_queue_rq,
1455
	.complete	= nvme_pci_complete_rq,
M
Matias Bjørling 已提交
1456 1457
	.init_hctx	= nvme_init_hctx,
	.init_request	= nvme_init_request,
1458
	.map_queues	= nvme_pci_map_queues,
M
Matias Bjørling 已提交
1459
	.timeout	= nvme_timeout,
J
Jens Axboe 已提交
1460
	.poll		= nvme_poll,
M
Matias Bjørling 已提交
1461 1462
};

1463 1464
static void nvme_dev_remove_admin(struct nvme_dev *dev)
{
1465
	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
1466 1467 1468 1469 1470
		/*
		 * If the controller was reset during removal, it's possible
		 * user requests may be waiting on a stopped queue. Start the
		 * queue to flush these to completion.
		 */
1471
		blk_mq_unquiesce_queue(dev->ctrl.admin_q);
1472
		blk_cleanup_queue(dev->ctrl.admin_q);
1473 1474 1475 1476
		blk_mq_free_tag_set(&dev->admin_tagset);
	}
}

M
Matias Bjørling 已提交
1477 1478
static int nvme_alloc_admin_tags(struct nvme_dev *dev)
{
1479
	if (!dev->ctrl.admin_q) {
M
Matias Bjørling 已提交
1480 1481
		dev->admin_tagset.ops = &nvme_mq_admin_ops;
		dev->admin_tagset.nr_hw_queues = 1;
K
Keith Busch 已提交
1482

K
Keith Busch 已提交
1483
		dev->admin_tagset.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
M
Matias Bjørling 已提交
1484
		dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1485
		dev->admin_tagset.numa_node = dev_to_node(dev->dev);
C
Chaitanya Kulkarni 已提交
1486
		dev->admin_tagset.cmd_size = nvme_pci_cmd_size(dev, false);
1487
		dev->admin_tagset.flags = BLK_MQ_F_NO_SCHED;
M
Matias Bjørling 已提交
1488 1489 1490 1491
		dev->admin_tagset.driver_data = dev;

		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
			return -ENOMEM;
1492
		dev->ctrl.admin_tagset = &dev->admin_tagset;
M
Matias Bjørling 已提交
1493

1494 1495
		dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
		if (IS_ERR(dev->ctrl.admin_q)) {
M
Matias Bjørling 已提交
1496 1497 1498
			blk_mq_free_tag_set(&dev->admin_tagset);
			return -ENOMEM;
		}
1499
		if (!blk_get_queue(dev->ctrl.admin_q)) {
1500
			nvme_dev_remove_admin(dev);
1501
			dev->ctrl.admin_q = NULL;
1502 1503
			return -ENODEV;
		}
K
Keith Busch 已提交
1504
	} else
1505
		blk_mq_unquiesce_queue(dev->ctrl.admin_q);
M
Matias Bjørling 已提交
1506 1507 1508 1509

	return 0;
}

1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
{
	return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
}

static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
{
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	if (size <= dev->bar_mapped_size)
		return 0;
	if (size > pci_resource_len(pdev, 0))
		return -ENOMEM;
	if (dev->bar)
		iounmap(dev->bar);
	dev->bar = ioremap(pci_resource_start(pdev, 0), size);
	if (!dev->bar) {
		dev->bar_mapped_size = 0;
		return -ENOMEM;
	}
	dev->bar_mapped_size = size;
	dev->dbs = dev->bar + NVME_REG_DBS;

	return 0;
}

1536
static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1537
{
1538
	int result;
M
Matthew Wilcox 已提交
1539 1540 1541
	u32 aqa;
	struct nvme_queue *nvmeq;

1542 1543 1544 1545
	result = nvme_remap_bar(dev, db_bar_size(dev, 0));
	if (result < 0)
		return result;

1546
	dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
1547
				NVME_CAP_NSSRC(dev->ctrl.cap) : 0;
1548

1549 1550 1551
	if (dev->subsystem &&
	    (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
		writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1552

1553
	result = nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
1554 1555
	if (result < 0)
		return result;
M
Matthew Wilcox 已提交
1556

1557
	result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1558 1559
	if (result)
		return result;
M
Matthew Wilcox 已提交
1560

1561
	nvmeq = &dev->queues[0];
M
Matthew Wilcox 已提交
1562 1563 1564
	aqa = nvmeq->q_depth - 1;
	aqa |= aqa << 16;

1565 1566 1567
	writel(aqa, dev->bar + NVME_REG_AQA);
	lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
	lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
M
Matthew Wilcox 已提交
1568

1569
	result = nvme_enable_ctrl(&dev->ctrl, dev->ctrl.cap);
1570
	if (result)
K
Keith Busch 已提交
1571
		return result;
M
Matias Bjørling 已提交
1572

K
Keith Busch 已提交
1573
	nvmeq->cq_vector = 0;
1574
	nvme_init_queue(nvmeq, 0);
1575
	result = queue_request_irq(nvmeq);
1576 1577
	if (result) {
		nvmeq->cq_vector = -1;
K
Keith Busch 已提交
1578
		return result;
1579
	}
1580

M
Matthew Wilcox 已提交
1581 1582 1583
	return result;
}

1584
static int nvme_create_io_queues(struct nvme_dev *dev)
K
Keith Busch 已提交
1585
{
1586
	unsigned i, max;
1587
	int ret = 0;
K
Keith Busch 已提交
1588

1589
	for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
1590
		if (nvme_alloc_queue(dev, i, dev->q_depth)) {
1591
			ret = -ENOMEM;
K
Keith Busch 已提交
1592
			break;
1593 1594
		}
	}
K
Keith Busch 已提交
1595

1596
	max = min(dev->max_qid, dev->ctrl.queue_count - 1);
1597
	for (i = dev->online_queues; i <= max; i++) {
1598
		ret = nvme_create_queue(&dev->queues[i], i);
K
Keith Busch 已提交
1599
		if (ret)
K
Keith Busch 已提交
1600
			break;
M
Matthew Wilcox 已提交
1601
	}
1602 1603 1604

	/*
	 * Ignore failing Create SQ/CQ commands, we can continue with less
1605 1606
	 * than the desired amount of queues, and even a controller without
	 * I/O queues can still be used to issue admin commands.  This might
1607 1608 1609
	 * be useful to upgrade a buggy firmware for example.
	 */
	return ret >= 0 ? 0 : ret;
M
Matthew Wilcox 已提交
1610 1611
}

1612 1613 1614 1615 1616 1617
static ssize_t nvme_cmb_show(struct device *dev,
			     struct device_attribute *attr,
			     char *buf)
{
	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));

1618
	return scnprintf(buf, PAGE_SIZE, "cmbloc : x%08x\ncmbsz  : x%08x\n",
1619 1620 1621 1622
		       ndev->cmbloc, ndev->cmbsz);
}
static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL);

1623
static u64 nvme_cmb_size_unit(struct nvme_dev *dev)
1624
{
1625 1626 1627 1628 1629 1630 1631 1632 1633 1634
	u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK;

	return 1ULL << (12 + 4 * szu);
}

static u32 nvme_cmb_size(struct nvme_dev *dev)
{
	return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK;
}

1635
static void nvme_map_cmb(struct nvme_dev *dev)
1636
{
1637
	u64 size, offset;
1638 1639
	resource_size_t bar_size;
	struct pci_dev *pdev = to_pci_dev(dev->dev);
1640
	int bar;
1641

1642
	dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1643 1644
	if (!dev->cmbsz)
		return;
1645
	dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1646

1647
	if (!use_cmb_sqes)
1648
		return;
1649

1650 1651
	size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev);
	offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc);
1652 1653
	bar = NVME_CMB_BIR(dev->cmbloc);
	bar_size = pci_resource_len(pdev, bar);
1654 1655

	if (offset > bar_size)
1656
		return;
1657 1658 1659 1660 1661 1662 1663 1664 1665

	/*
	 * Controllers may support a CMB size larger than their BAR,
	 * for example, due to being behind a bridge. Reduce the CMB to
	 * the reported size of the BAR
	 */
	if (size > bar_size - offset)
		size = bar_size - offset;

1666 1667 1668
	dev->cmb = ioremap_wc(pci_resource_start(pdev, bar) + offset, size);
	if (!dev->cmb)
		return;
1669
	dev->cmb_bus_addr = pci_bus_address(pdev, bar) + offset;
1670
	dev->cmb_size = size;
1671 1672 1673 1674 1675

	if (sysfs_add_file_to_group(&dev->ctrl.device->kobj,
				    &dev_attr_cmb.attr, NULL))
		dev_warn(dev->ctrl.device,
			 "failed to add sysfs attribute for CMB\n");
1676 1677 1678 1679 1680 1681 1682
}

static inline void nvme_release_cmb(struct nvme_dev *dev)
{
	if (dev->cmb) {
		iounmap(dev->cmb);
		dev->cmb = NULL;
1683 1684 1685
		sysfs_remove_file_from_group(&dev->ctrl.device->kobj,
					     &dev_attr_cmb.attr, NULL);
		dev->cmbsz = 0;
1686 1687 1688
	}
}

1689 1690
static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
{
1691
	u64 dma_addr = dev->host_mem_descs_dma;
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725 1726 1727
	struct nvme_command c;
	int ret;

	memset(&c, 0, sizeof(c));
	c.features.opcode	= nvme_admin_set_features;
	c.features.fid		= cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
	c.features.dword11	= cpu_to_le32(bits);
	c.features.dword12	= cpu_to_le32(dev->host_mem_size >>
					      ilog2(dev->ctrl.page_size));
	c.features.dword13	= cpu_to_le32(lower_32_bits(dma_addr));
	c.features.dword14	= cpu_to_le32(upper_32_bits(dma_addr));
	c.features.dword15	= cpu_to_le32(dev->nr_host_mem_descs);

	ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
	if (ret) {
		dev_warn(dev->ctrl.device,
			 "failed to set host mem (err %d, flags %#x).\n",
			 ret, bits);
	}
	return ret;
}

static void nvme_free_host_mem(struct nvme_dev *dev)
{
	int i;

	for (i = 0; i < dev->nr_host_mem_descs; i++) {
		struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
		size_t size = le32_to_cpu(desc->size) * dev->ctrl.page_size;

		dma_free_coherent(dev->dev, size, dev->host_mem_desc_bufs[i],
				le64_to_cpu(desc->addr));
	}

	kfree(dev->host_mem_desc_bufs);
	dev->host_mem_desc_bufs = NULL;
1728 1729 1730
	dma_free_coherent(dev->dev,
			dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs),
			dev->host_mem_descs, dev->host_mem_descs_dma);
1731
	dev->host_mem_descs = NULL;
1732
	dev->nr_host_mem_descs = 0;
1733 1734
}

1735 1736
static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,
		u32 chunk_size)
K
Keith Busch 已提交
1737
{
1738
	struct nvme_host_mem_buf_desc *descs;
1739
	u32 max_entries, len;
1740
	dma_addr_t descs_dma;
1741
	int i = 0;
1742
	void **bufs;
1743
	u64 size, tmp;
1744 1745 1746 1747

	tmp = (preferred + chunk_size - 1);
	do_div(tmp, chunk_size);
	max_entries = tmp;
1748 1749 1750 1751

	if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
		max_entries = dev->ctrl.hmmaxd;

1752 1753
	descs = dma_zalloc_coherent(dev->dev, max_entries * sizeof(*descs),
			&descs_dma, GFP_KERNEL);
1754 1755 1756 1757 1758 1759 1760
	if (!descs)
		goto out;

	bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL);
	if (!bufs)
		goto out_free_descs;

1761
	for (size = 0; size < preferred && i < max_entries; size += len) {
1762 1763
		dma_addr_t dma_addr;

1764
		len = min_t(u64, chunk_size, preferred - size);
1765 1766 1767 1768 1769 1770 1771 1772 1773 1774
		bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL,
				DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
		if (!bufs[i])
			break;

		descs[i].addr = cpu_to_le64(dma_addr);
		descs[i].size = cpu_to_le32(len / dev->ctrl.page_size);
		i++;
	}

1775
	if (!size)
1776 1777 1778 1779 1780
		goto out_free_bufs;

	dev->nr_host_mem_descs = i;
	dev->host_mem_size = size;
	dev->host_mem_descs = descs;
1781
	dev->host_mem_descs_dma = descs_dma;
1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794
	dev->host_mem_desc_bufs = bufs;
	return 0;

out_free_bufs:
	while (--i >= 0) {
		size_t size = le32_to_cpu(descs[i].size) * dev->ctrl.page_size;

		dma_free_coherent(dev->dev, size, bufs[i],
				le64_to_cpu(descs[i].addr));
	}

	kfree(bufs);
out_free_descs:
1795 1796
	dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
			descs_dma);
1797 1798 1799 1800 1801
out:
	dev->host_mem_descs = NULL;
	return -ENOMEM;
}

1802 1803 1804 1805 1806
static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
{
	u32 chunk_size;

	/* start big and work our way down */
1807
	for (chunk_size = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
1808
	     chunk_size >= max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
	     chunk_size /= 2) {
		if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {
			if (!min || dev->host_mem_size >= min)
				return 0;
			nvme_free_host_mem(dev);
		}
	}

	return -ENOMEM;
}

1820
static int nvme_setup_host_mem(struct nvme_dev *dev)
1821 1822 1823 1824 1825
{
	u64 max = (u64)max_host_mem_size_mb * SZ_1M;
	u64 preferred = (u64)dev->ctrl.hmpre * 4096;
	u64 min = (u64)dev->ctrl.hmmin * 4096;
	u32 enable_bits = NVME_HOST_MEM_ENABLE;
1826
	int ret;
1827 1828 1829 1830 1831 1832 1833

	preferred = min(preferred, max);
	if (min > max) {
		dev_warn(dev->ctrl.device,
			"min host memory (%lld MiB) above limit (%d MiB).\n",
			min >> ilog2(SZ_1M), max_host_mem_size_mb);
		nvme_free_host_mem(dev);
1834
		return 0;
1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847
	}

	/*
	 * If we already have a buffer allocated check if we can reuse it.
	 */
	if (dev->host_mem_descs) {
		if (dev->host_mem_size >= min)
			enable_bits |= NVME_HOST_MEM_RETURN;
		else
			nvme_free_host_mem(dev);
	}

	if (!dev->host_mem_descs) {
1848 1849 1850
		if (nvme_alloc_host_mem(dev, min, preferred)) {
			dev_warn(dev->ctrl.device,
				"failed to allocate host memory buffer.\n");
1851
			return 0; /* controller must work without HMB */
1852 1853 1854 1855 1856
		}

		dev_info(dev->ctrl.device,
			"allocated %lld MiB host memory buffer.\n",
			dev->host_mem_size >> ilog2(SZ_1M));
1857 1858
	}

1859 1860
	ret = nvme_set_host_mem(dev, enable_bits);
	if (ret)
1861
		nvme_free_host_mem(dev);
1862
	return ret;
K
Keith Busch 已提交
1863 1864
}

1865
static int nvme_setup_io_queues(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1866
{
1867
	struct nvme_queue *adminq = &dev->queues[0];
1868
	struct pci_dev *pdev = to_pci_dev(dev->dev);
1869 1870
	int result, nr_io_queues;
	unsigned long size;
M
Matthew Wilcox 已提交
1871

1872 1873 1874 1875
	struct irq_affinity affd = {
		.pre_vectors = 1
	};

1876
	nr_io_queues = num_possible_cpus();
C
Christoph Hellwig 已提交
1877 1878
	result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
	if (result < 0)
M
Matthew Wilcox 已提交
1879
		return result;
C
Christoph Hellwig 已提交
1880

1881
	if (nr_io_queues == 0)
1882
		return 0;
M
Matthew Wilcox 已提交
1883

1884
	if (dev->cmb && (dev->cmbsz & NVME_CMBSZ_SQS)) {
1885 1886 1887 1888 1889 1890 1891 1892
		result = nvme_cmb_qdepth(dev, nr_io_queues,
				sizeof(struct nvme_command));
		if (result > 0)
			dev->q_depth = result;
		else
			nvme_release_cmb(dev);
	}

1893 1894 1895 1896 1897 1898 1899 1900 1901
	do {
		size = db_bar_size(dev, nr_io_queues);
		result = nvme_remap_bar(dev, size);
		if (!result)
			break;
		if (!--nr_io_queues)
			return -ENOMEM;
	} while (1);
	adminq->q_db = dev->dbs;
1902

K
Keith Busch 已提交
1903
	/* Deregister the admin queue's interrupt */
1904
	pci_free_irq(pdev, 0, adminq);
K
Keith Busch 已提交
1905

1906 1907 1908 1909
	/*
	 * If we enable msix early due to not intx, disable it again before
	 * setting up the full range we need.
	 */
1910
	pci_free_irq_vectors(pdev);
1911 1912 1913
	result = pci_alloc_irq_vectors_affinity(pdev, 1, nr_io_queues + 1,
			PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY, &affd);
	if (result <= 0)
1914
		return -EIO;
1915 1916
	dev->num_vecs = result;
	dev->max_qid = max(result - 1, 1);
R
Ramachandra Rao Gajula 已提交
1917

1918 1919 1920 1921 1922 1923 1924
	/*
	 * Should investigate if there's a performance win from allocating
	 * more queues than interrupt vectors; it might allow the submission
	 * path to scale better, even if the receive path is limited by the
	 * number of interrupts.
	 */

1925
	result = queue_request_irq(adminq);
1926 1927
	if (result) {
		adminq->cq_vector = -1;
K
Keith Busch 已提交
1928
		return result;
1929
	}
1930
	return nvme_create_io_queues(dev);
M
Matthew Wilcox 已提交
1931 1932
}

1933
static void nvme_del_queue_end(struct request *req, blk_status_t error)
K
Keith Busch 已提交
1934
{
K
Keith Busch 已提交
1935
	struct nvme_queue *nvmeq = req->end_io_data;
1936

K
Keith Busch 已提交
1937 1938
	blk_mq_free_request(req);
	complete(&nvmeq->dev->ioq_wait);
K
Keith Busch 已提交
1939 1940
}

1941
static void nvme_del_cq_end(struct request *req, blk_status_t error)
K
Keith Busch 已提交
1942
{
K
Keith Busch 已提交
1943
	struct nvme_queue *nvmeq = req->end_io_data;
1944
	u16 start, end;
K
Keith Busch 已提交
1945

K
Keith Busch 已提交
1946 1947 1948
	if (!error) {
		unsigned long flags;

1949
		spin_lock_irqsave(&nvmeq->cq_lock, flags);
1950
		nvme_process_cq(nvmeq, &start, &end, -1);
1951
		spin_unlock_irqrestore(&nvmeq->cq_lock, flags);
1952 1953

		nvme_complete_cqes(nvmeq, start, end);
K
Keith Busch 已提交
1954
	}
K
Keith Busch 已提交
1955 1956

	nvme_del_queue_end(req, error);
K
Keith Busch 已提交
1957 1958
}

K
Keith Busch 已提交
1959
static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
1960
{
K
Keith Busch 已提交
1961 1962 1963
	struct request_queue *q = nvmeq->dev->ctrl.admin_q;
	struct request *req;
	struct nvme_command cmd;
1964

K
Keith Busch 已提交
1965 1966 1967
	memset(&cmd, 0, sizeof(cmd));
	cmd.delete_queue.opcode = opcode;
	cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
1968

1969
	req = nvme_alloc_request(q, &cmd, BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
K
Keith Busch 已提交
1970 1971
	if (IS_ERR(req))
		return PTR_ERR(req);
1972

K
Keith Busch 已提交
1973 1974 1975 1976 1977 1978 1979
	req->timeout = ADMIN_TIMEOUT;
	req->end_io_data = nvmeq;

	blk_execute_rq_nowait(q, NULL, req, false,
			opcode == nvme_admin_delete_cq ?
				nvme_del_cq_end : nvme_del_queue_end);
	return 0;
1980 1981
}

1982
static void nvme_disable_io_queues(struct nvme_dev *dev)
K
Keith Busch 已提交
1983
{
1984
	int pass, queues = dev->online_queues - 1;
K
Keith Busch 已提交
1985 1986
	unsigned long timeout;
	u8 opcode = nvme_admin_delete_sq;
K
Keith Busch 已提交
1987

K
Keith Busch 已提交
1988
	for (pass = 0; pass < 2; pass++) {
K
Keith Busch 已提交
1989
		int sent = 0, i = queues;
K
Keith Busch 已提交
1990 1991 1992 1993

		reinit_completion(&dev->ioq_wait);
 retry:
		timeout = ADMIN_TIMEOUT;
1994
		for (; i > 0; i--, sent++)
1995
			if (nvme_delete_queue(&dev->queues[i], opcode))
K
Keith Busch 已提交
1996
				break;
1997

K
Keith Busch 已提交
1998 1999 2000 2001 2002 2003 2004 2005 2006
		while (sent--) {
			timeout = wait_for_completion_io_timeout(&dev->ioq_wait, timeout);
			if (timeout == 0)
				return;
			if (i)
				goto retry;
		}
		opcode = nvme_admin_delete_cq;
	}
K
Keith Busch 已提交
2007 2008
}

2009
/*
2010
 * return error value only when tagset allocation failed
2011
 */
2012
static int nvme_dev_add(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2013
{
2014 2015
	int ret;

2016
	if (!dev->ctrl.tagset) {
2017 2018 2019 2020 2021
		dev->tagset.ops = &nvme_mq_ops;
		dev->tagset.nr_hw_queues = dev->online_queues - 1;
		dev->tagset.timeout = NVME_IO_TIMEOUT;
		dev->tagset.numa_node = dev_to_node(dev->dev);
		dev->tagset.queue_depth =
M
Matias Bjørling 已提交
2022
				min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
C
Chaitanya Kulkarni 已提交
2023 2024 2025 2026 2027
		dev->tagset.cmd_size = nvme_pci_cmd_size(dev, false);
		if ((dev->ctrl.sgls & ((1 << 0) | (1 << 1))) && sgl_threshold) {
			dev->tagset.cmd_size = max(dev->tagset.cmd_size,
					nvme_pci_cmd_size(dev, true));
		}
2028 2029
		dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
		dev->tagset.driver_data = dev;
M
Matthew Wilcox 已提交
2030

2031 2032 2033 2034 2035 2036
		ret = blk_mq_alloc_tag_set(&dev->tagset);
		if (ret) {
			dev_warn(dev->ctrl.device,
				"IO queues tagset allocation failed %d\n", ret);
			return ret;
		}
2037
		dev->ctrl.tagset = &dev->tagset;
2038 2039

		nvme_dbbuf_set(dev);
2040 2041 2042 2043 2044
	} else {
		blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);

		/* Free previously allocated queues that are no longer usable */
		nvme_free_queues(dev, dev->online_queues);
2045
	}
2046

K
Keith Busch 已提交
2047
	return 0;
M
Matthew Wilcox 已提交
2048 2049
}

2050
static int nvme_pci_enable(struct nvme_dev *dev)
2051
{
2052
	int result = -ENOMEM;
2053
	struct pci_dev *pdev = to_pci_dev(dev->dev);
2054 2055 2056 2057 2058 2059

	if (pci_enable_device_mem(pdev))
		return result;

	pci_set_master(pdev);

2060 2061
	if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
	    dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
2062
		goto disable;
2063

2064
	if (readl(dev->bar + NVME_REG_CSTS) == -1) {
K
Keith Busch 已提交
2065
		result = -ENODEV;
2066
		goto disable;
K
Keith Busch 已提交
2067
	}
2068 2069

	/*
2070 2071 2072
	 * Some devices and/or platforms don't advertise or work with INTx
	 * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
	 * adjust this later.
2073
	 */
2074 2075 2076
	result = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
	if (result < 0)
		return result;
2077

2078
	dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
2079

2080
	dev->q_depth = min_t(int, NVME_CAP_MQES(dev->ctrl.cap) + 1,
2081
				io_queue_depth);
2082
	dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
2083
	dev->dbs = dev->bar + 4096;
2084 2085 2086 2087 2088 2089 2090

	/*
	 * Temporary fix for the Apple controller found in the MacBook8,1 and
	 * some MacBook7,1 to avoid controller resets and data loss.
	 */
	if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) {
		dev->q_depth = 2;
2091 2092
		dev_warn(dev->ctrl.device, "detected Apple NVMe controller, "
			"set queue depth=%u to work around controller resets\n",
2093
			dev->q_depth);
2094 2095
	} else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG &&
		   (pdev->device == 0xa821 || pdev->device == 0xa822) &&
2096
		   NVME_CAP_MQES(dev->ctrl.cap) == 0) {
2097 2098 2099
		dev->q_depth = 64;
		dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
                        "set queue depth=%u\n", dev->q_depth);
2100 2101
	}

2102
	nvme_map_cmb(dev);
2103

K
Keith Busch 已提交
2104 2105
	pci_enable_pcie_error_reporting(pdev);
	pci_save_state(pdev);
2106 2107 2108 2109 2110 2111 2112 2113
	return 0;

 disable:
	pci_disable_device(pdev);
	return result;
}

static void nvme_dev_unmap(struct nvme_dev *dev)
2114 2115 2116
{
	if (dev->bar)
		iounmap(dev->bar);
2117
	pci_release_mem_regions(to_pci_dev(dev->dev));
2118 2119 2120
}

static void nvme_pci_disable(struct nvme_dev *dev)
2121
{
2122 2123
	struct pci_dev *pdev = to_pci_dev(dev->dev);

2124
	nvme_release_cmb(dev);
2125
	pci_free_irq_vectors(pdev);
2126

K
Keith Busch 已提交
2127 2128
	if (pci_is_enabled(pdev)) {
		pci_disable_pcie_error_reporting(pdev);
2129
		pci_disable_device(pdev);
K
Keith Busch 已提交
2130 2131 2132
	}
}

2133
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
M
Matthew Wilcox 已提交
2134
{
2135
	int i;
K
Keith Busch 已提交
2136 2137
	bool dead = true;
	struct pci_dev *pdev = to_pci_dev(dev->dev);
2138

2139
	mutex_lock(&dev->shutdown_lock);
K
Keith Busch 已提交
2140 2141 2142
	if (pci_is_enabled(pdev)) {
		u32 csts = readl(dev->bar + NVME_REG_CSTS);

K
Keith Busch 已提交
2143 2144
		if (dev->ctrl.state == NVME_CTRL_LIVE ||
		    dev->ctrl.state == NVME_CTRL_RESETTING)
K
Keith Busch 已提交
2145 2146 2147
			nvme_start_freeze(&dev->ctrl);
		dead = !!((csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY) ||
			pdev->error_state  != pci_channel_io_normal);
2148
	}
2149

K
Keith Busch 已提交
2150 2151 2152 2153
	/*
	 * Give the controller a chance to complete all entered requests if
	 * doing a safe shutdown.
	 */
2154 2155 2156
	if (!dead) {
		if (shutdown)
			nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
2157 2158 2159
	}

	nvme_stop_queues(&dev->ctrl);
2160

2161
	if (!dead && dev->ctrl.queue_count > 0) {
2162
		nvme_disable_io_queues(dev);
2163
		nvme_disable_admin_queue(dev, shutdown);
K
Keith Busch 已提交
2164
	}
2165 2166 2167
	for (i = dev->ctrl.queue_count - 1; i >= 0; i--)
		nvme_suspend_queue(&dev->queues[i]);

2168
	nvme_pci_disable(dev);
2169

2170 2171
	blk_mq_tagset_busy_iter(&dev->tagset, nvme_cancel_request, &dev->ctrl);
	blk_mq_tagset_busy_iter(&dev->admin_tagset, nvme_cancel_request, &dev->ctrl);
K
Keith Busch 已提交
2172 2173 2174 2175 2176 2177 2178 2179

	/*
	 * The driver will not be starting up queues again if shutting down so
	 * must flush all entered requests to their failed completion to avoid
	 * deadlocking blk-mq hot-cpu notifier.
	 */
	if (shutdown)
		nvme_start_queues(&dev->ctrl);
2180
	mutex_unlock(&dev->shutdown_lock);
M
Matthew Wilcox 已提交
2181 2182
}

M
Matthew Wilcox 已提交
2183 2184
static int nvme_setup_prp_pools(struct nvme_dev *dev)
{
2185
	dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
M
Matthew Wilcox 已提交
2186 2187 2188 2189
						PAGE_SIZE, PAGE_SIZE, 0);
	if (!dev->prp_page_pool)
		return -ENOMEM;

2190
	/* Optimisation for I/Os between 4k and 128k */
2191
	dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2192 2193 2194 2195 2196
						256, 256, 0);
	if (!dev->prp_small_pool) {
		dma_pool_destroy(dev->prp_page_pool);
		return -ENOMEM;
	}
M
Matthew Wilcox 已提交
2197 2198 2199 2200 2201 2202
	return 0;
}

static void nvme_release_prp_pools(struct nvme_dev *dev)
{
	dma_pool_destroy(dev->prp_page_pool);
2203
	dma_pool_destroy(dev->prp_small_pool);
M
Matthew Wilcox 已提交
2204 2205
}

2206
static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
2207
{
2208
	struct nvme_dev *dev = to_nvme_dev(ctrl);
2209

2210
	nvme_dbbuf_dma_free(dev);
2211
	put_device(dev->dev);
2212 2213
	if (dev->tagset.tags)
		blk_mq_free_tag_set(&dev->tagset);
2214 2215
	if (dev->ctrl.admin_q)
		blk_put_queue(dev->ctrl.admin_q);
2216
	kfree(dev->queues);
2217
	free_opal_dev(dev->ctrl.opal_dev);
2218
	mempool_destroy(dev->iod_mempool);
2219 2220 2221
	kfree(dev);
}

2222 2223
static void nvme_remove_dead_ctrl(struct nvme_dev *dev, int status)
{
2224
	dev_warn(dev->ctrl.device, "Removing after probe failure status: %d\n", status);
2225

2226
	nvme_get_ctrl(&dev->ctrl);
2227
	nvme_dev_disable(dev, false);
2228
	nvme_kill_queues(&dev->ctrl);
2229
	if (!queue_work(nvme_wq, &dev->remove_work))
2230 2231 2232
		nvme_put_ctrl(&dev->ctrl);
}

2233
static void nvme_reset_work(struct work_struct *work)
2234
{
2235 2236
	struct nvme_dev *dev =
		container_of(work, struct nvme_dev, ctrl.reset_work);
2237
	bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
2238
	int result = -ENODEV;
2239
	enum nvme_ctrl_state new_state = NVME_CTRL_LIVE;
2240

2241
	if (WARN_ON(dev->ctrl.state != NVME_CTRL_RESETTING))
2242
		goto out;
2243

2244 2245 2246 2247
	/*
	 * If we're called to reset a live controller first shut it down before
	 * moving on.
	 */
2248
	if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
2249
		nvme_dev_disable(dev, false);
2250

2251
	/*
2252
	 * Introduce CONNECTING state from nvme-fc/rdma transports to mark the
2253 2254
	 * initializing procedure here.
	 */
2255
	if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
2256
		dev_warn(dev->ctrl.device,
2257
			"failed to mark controller CONNECTING\n");
2258 2259 2260
		goto out;
	}

2261
	result = nvme_pci_enable(dev);
2262
	if (result)
2263
		goto out;
2264

2265
	result = nvme_pci_configure_admin_queue(dev);
2266
	if (result)
2267
		goto out;
2268

K
Keith Busch 已提交
2269 2270
	result = nvme_alloc_admin_tags(dev);
	if (result)
2271
		goto out;
2272

2273 2274 2275 2276 2277 2278 2279
	/*
	 * Limit the max command size to prevent iod->sg allocations going
	 * over a single page.
	 */
	dev->ctrl.max_hw_sectors = NVME_MAX_KB_SZ << 1;
	dev->ctrl.max_segments = NVME_MAX_SEGS;

2280 2281
	result = nvme_init_identify(&dev->ctrl);
	if (result)
2282
		goto out;
2283

2284 2285 2286 2287 2288 2289 2290 2291 2292
	if (dev->ctrl.oacs & NVME_CTRL_OACS_SEC_SUPP) {
		if (!dev->ctrl.opal_dev)
			dev->ctrl.opal_dev =
				init_opal_dev(&dev->ctrl, &nvme_sec_submit);
		else if (was_suspend)
			opal_unlock_from_suspend(dev->ctrl.opal_dev);
	} else {
		free_opal_dev(dev->ctrl.opal_dev);
		dev->ctrl.opal_dev = NULL;
2293
	}
2294

2295 2296 2297 2298 2299 2300 2301
	if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) {
		result = nvme_dbbuf_dma_alloc(dev);
		if (result)
			dev_warn(dev->dev,
				 "unable to allocate dma for dbbuf\n");
	}

2302 2303 2304 2305 2306
	if (dev->ctrl.hmpre) {
		result = nvme_setup_host_mem(dev);
		if (result < 0)
			goto out;
	}
2307

2308
	result = nvme_setup_io_queues(dev);
2309
	if (result)
2310
		goto out;
2311

2312 2313 2314 2315
	/*
	 * Keep the controller around but remove all namespaces if we don't have
	 * any working I/O queue.
	 */
2316
	if (dev->online_queues < 2) {
2317
		dev_warn(dev->ctrl.device, "IO queues not created\n");
2318
		nvme_kill_queues(&dev->ctrl);
2319
		nvme_remove_namespaces(&dev->ctrl);
2320
		new_state = NVME_CTRL_ADMIN_ONLY;
2321
	} else {
2322
		nvme_start_queues(&dev->ctrl);
K
Keith Busch 已提交
2323
		nvme_wait_freeze(&dev->ctrl);
2324 2325 2326
		/* hit this only when allocate tagset fails */
		if (nvme_dev_add(dev))
			new_state = NVME_CTRL_ADMIN_ONLY;
K
Keith Busch 已提交
2327
		nvme_unfreeze(&dev->ctrl);
2328 2329
	}

2330 2331 2332 2333 2334 2335 2336
	/*
	 * If only admin queue live, keep it to do further investigation or
	 * recovery.
	 */
	if (!nvme_change_ctrl_state(&dev->ctrl, new_state)) {
		dev_warn(dev->ctrl.device,
			"failed to mark controller state %d\n", new_state);
2337 2338
		goto out;
	}
2339

2340
	nvme_start_ctrl(&dev->ctrl);
2341
	return;
2342

2343
 out:
2344
	nvme_remove_dead_ctrl(dev, result);
2345 2346
}

2347
static void nvme_remove_dead_ctrl_work(struct work_struct *work)
K
Keith Busch 已提交
2348
{
2349
	struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
2350
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
2351 2352

	if (pci_get_drvdata(pdev))
K
Keith Busch 已提交
2353
		device_release_driver(&pdev->dev);
2354
	nvme_put_ctrl(&dev->ctrl);
K
Keith Busch 已提交
2355 2356
}

2357
static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
T
Tejun Heo 已提交
2358
{
2359
	*val = readl(to_nvme_dev(ctrl)->bar + off);
2360
	return 0;
T
Tejun Heo 已提交
2361 2362
}

2363
static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
2364
{
2365 2366 2367
	writel(val, to_nvme_dev(ctrl)->bar + off);
	return 0;
}
2368

2369 2370 2371 2372
static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
{
	*val = readq(to_nvme_dev(ctrl)->bar + off);
	return 0;
2373 2374
}

2375 2376 2377 2378 2379 2380 2381
static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
{
	struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);

	return snprintf(buf, size, "%s", dev_name(&pdev->dev));
}

2382
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
M
Ming Lin 已提交
2383
	.name			= "pcie",
2384
	.module			= THIS_MODULE,
2385
	.flags			= NVME_F_METADATA_SUPPORTED,
2386
	.reg_read32		= nvme_pci_reg_read32,
2387
	.reg_write32		= nvme_pci_reg_write32,
2388
	.reg_read64		= nvme_pci_reg_read64,
2389
	.free_ctrl		= nvme_pci_free_ctrl,
2390
	.submit_async_event	= nvme_pci_submit_async_event,
2391
	.get_address		= nvme_pci_get_address,
2392
};
2393

2394 2395 2396 2397
static int nvme_dev_map(struct nvme_dev *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev->dev);

2398
	if (pci_request_mem_regions(pdev, "nvme"))
2399 2400
		return -ENODEV;

2401
	if (nvme_remap_bar(dev, NVME_REG_DBS + 4096))
2402 2403
		goto release;

M
Max Gurtovoy 已提交
2404
	return 0;
2405
  release:
M
Max Gurtovoy 已提交
2406 2407
	pci_release_mem_regions(pdev);
	return -ENODEV;
2408 2409
}

2410
static unsigned long check_vendor_combination_bug(struct pci_dev *pdev)
2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424
{
	if (pdev->vendor == 0x144d && pdev->device == 0xa802) {
		/*
		 * Several Samsung devices seem to drop off the PCIe bus
		 * randomly when APST is on and uses the deepest sleep state.
		 * This has been observed on a Samsung "SM951 NVMe SAMSUNG
		 * 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD
		 * 950 PRO 256GB", but it seems to be restricted to two Dell
		 * laptops.
		 */
		if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") &&
		    (dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") ||
		     dmi_match(DMI_PRODUCT_NAME, "Precision 5510")))
			return NVME_QUIRK_NO_DEEPEST_PS;
2425 2426 2427
	} else if (pdev->vendor == 0x144d && pdev->device == 0xa804) {
		/*
		 * Samsung SSD 960 EVO drops off the PCIe bus after system
2428 2429 2430
		 * suspend on a Ryzen board, ASUS PRIME B350M-A, as well as
		 * within few minutes after bootup on a Coffee Lake board -
		 * ASUS PRIME Z370-A
2431 2432
		 */
		if (dmi_match(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC.") &&
2433 2434
		    (dmi_match(DMI_BOARD_NAME, "PRIME B350M-A") ||
		     dmi_match(DMI_BOARD_NAME, "PRIME Z370-A")))
2435
			return NVME_QUIRK_NO_APST;
2436 2437 2438 2439 2440
	}

	return 0;
}

2441 2442 2443
static void nvme_async_probe(void *data, async_cookie_t cookie)
{
	struct nvme_dev *dev = data;
2444

2445 2446
	nvme_reset_ctrl_sync(&dev->ctrl);
	flush_work(&dev->ctrl.scan_work);
2447
	nvme_put_ctrl(&dev->ctrl);
2448 2449
}

2450
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
M
Matthew Wilcox 已提交
2451
{
M
Matias Bjørling 已提交
2452
	int node, result = -ENOMEM;
M
Matthew Wilcox 已提交
2453
	struct nvme_dev *dev;
2454
	unsigned long quirks = id->driver_data;
2455
	size_t alloc_size;
M
Matthew Wilcox 已提交
2456

M
Matias Bjørling 已提交
2457 2458
	node = dev_to_node(&pdev->dev);
	if (node == NUMA_NO_NODE)
2459
		set_dev_node(&pdev->dev, first_memory_node);
M
Matias Bjørling 已提交
2460 2461

	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2462 2463
	if (!dev)
		return -ENOMEM;
2464 2465 2466

	dev->queues = kcalloc_node(num_possible_cpus() + 1,
			sizeof(struct nvme_queue), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2467 2468 2469
	if (!dev->queues)
		goto free;

2470
	dev->dev = get_device(&pdev->dev);
K
Keith Busch 已提交
2471
	pci_set_drvdata(pdev, dev);
2472

2473 2474
	result = nvme_dev_map(dev);
	if (result)
2475
		goto put_pci;
2476

2477
	INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
2478
	INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
2479
	mutex_init(&dev->shutdown_lock);
K
Keith Busch 已提交
2480
	init_completion(&dev->ioq_wait);
M
Matthew Wilcox 已提交
2481

M
Matthew Wilcox 已提交
2482 2483
	result = nvme_setup_prp_pools(dev);
	if (result)
2484
		goto unmap;
2485

2486
	quirks |= check_vendor_combination_bug(pdev);
2487

2488 2489 2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504
	/*
	 * Double check that our mempool alloc size will cover the biggest
	 * command we support.
	 */
	alloc_size = nvme_pci_iod_alloc_size(dev, NVME_MAX_KB_SZ,
						NVME_MAX_SEGS, true);
	WARN_ON_ONCE(alloc_size > PAGE_SIZE);

	dev->iod_mempool = mempool_create_node(1, mempool_kmalloc,
						mempool_kfree,
						(void *) alloc_size,
						GFP_KERNEL, node);
	if (!dev->iod_mempool) {
		result = -ENOMEM;
		goto release_pools;
	}

2505 2506 2507 2508 2509
	result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
			quirks);
	if (result)
		goto release_mempool;

2510 2511
	dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));

2512
	nvme_get_ctrl(&dev->ctrl);
2513
	async_schedule(nvme_async_probe, dev);
2514

M
Matthew Wilcox 已提交
2515 2516
	return 0;

2517 2518
 release_mempool:
	mempool_destroy(dev->iod_mempool);
2519
 release_pools:
M
Matthew Wilcox 已提交
2520
	nvme_release_prp_pools(dev);
2521 2522
 unmap:
	nvme_dev_unmap(dev);
K
Keith Busch 已提交
2523
 put_pci:
2524
	put_device(dev->dev);
M
Matthew Wilcox 已提交
2525 2526 2527 2528 2529 2530
 free:
	kfree(dev->queues);
	kfree(dev);
	return result;
}

2531
static void nvme_reset_prepare(struct pci_dev *pdev)
2532
{
K
Keith Busch 已提交
2533
	struct nvme_dev *dev = pci_get_drvdata(pdev);
2534
	nvme_dev_disable(dev, false);
2535
}
2536

2537 2538
static void nvme_reset_done(struct pci_dev *pdev)
{
2539
	struct nvme_dev *dev = pci_get_drvdata(pdev);
S
Sagi Grimberg 已提交
2540
	nvme_reset_ctrl_sync(&dev->ctrl);
2541 2542
}

2543 2544 2545
static void nvme_shutdown(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
2546
	nvme_dev_disable(dev, true);
2547 2548
}

2549 2550 2551 2552 2553
/*
 * The driver's remove may be called on a device in a partially initialized
 * state. This function must not have any dependencies on the device state in
 * order to proceed.
 */
2554
static void nvme_remove(struct pci_dev *pdev)
M
Matthew Wilcox 已提交
2555 2556
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
K
Keith Busch 已提交
2557

2558 2559
	nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);

2560
	cancel_work_sync(&dev->ctrl.reset_work);
K
Keith Busch 已提交
2561
	pci_set_drvdata(pdev, NULL);
2562

2563
	if (!pci_device_is_present(pdev)) {
2564
		nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
2565
		nvme_dev_disable(dev, true);
2566
	}
2567

2568
	flush_work(&dev->ctrl.reset_work);
2569 2570
	nvme_stop_ctrl(&dev->ctrl);
	nvme_remove_namespaces(&dev->ctrl);
2571
	nvme_dev_disable(dev, true);
2572
	nvme_free_host_mem(dev);
M
Matias Bjørling 已提交
2573
	nvme_dev_remove_admin(dev);
2574
	nvme_free_queues(dev, 0);
2575
	nvme_uninit_ctrl(&dev->ctrl);
K
Keith Busch 已提交
2576
	nvme_release_prp_pools(dev);
2577
	nvme_dev_unmap(dev);
2578
	nvme_put_ctrl(&dev->ctrl);
M
Matthew Wilcox 已提交
2579 2580
}

2581
#ifdef CONFIG_PM_SLEEP
2582 2583 2584 2585 2586
static int nvme_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

2587
	nvme_dev_disable(ndev, true);
2588 2589 2590 2591 2592 2593 2594 2595
	return 0;
}

static int nvme_resume(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

2596
	nvme_reset_ctrl(&ndev->ctrl);
K
Keith Busch 已提交
2597
	return 0;
2598
}
2599
#endif
2600 2601

static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
M
Matthew Wilcox 已提交
2602

K
Keith Busch 已提交
2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616
static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
						pci_channel_state_t state)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);

	/*
	 * A frozen channel requires a reset. When detected, this method will
	 * shutdown the controller to quiesce. The controller will be restarted
	 * after the slot reset through driver's slot_reset callback.
	 */
	switch (state) {
	case pci_channel_io_normal:
		return PCI_ERS_RESULT_CAN_RECOVER;
	case pci_channel_io_frozen:
K
Keith Busch 已提交
2617 2618
		dev_warn(dev->ctrl.device,
			"frozen state error detected, reset controller\n");
2619
		nvme_dev_disable(dev, false);
K
Keith Busch 已提交
2620 2621
		return PCI_ERS_RESULT_NEED_RESET;
	case pci_channel_io_perm_failure:
K
Keith Busch 已提交
2622 2623
		dev_warn(dev->ctrl.device,
			"failure state error detected, request disconnect\n");
K
Keith Busch 已提交
2624 2625 2626 2627 2628 2629 2630 2631 2632
		return PCI_ERS_RESULT_DISCONNECT;
	}
	return PCI_ERS_RESULT_NEED_RESET;
}

static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);

2633
	dev_info(dev->ctrl.device, "restart after slot reset\n");
K
Keith Busch 已提交
2634
	pci_restore_state(pdev);
2635
	nvme_reset_ctrl(&dev->ctrl);
K
Keith Busch 已提交
2636 2637 2638 2639 2640
	return PCI_ERS_RESULT_RECOVERED;
}

static void nvme_error_resume(struct pci_dev *pdev)
{
K
Keith Busch 已提交
2641 2642 2643
	struct nvme_dev *dev = pci_get_drvdata(pdev);

	flush_work(&dev->ctrl.reset_work);
K
Keith Busch 已提交
2644 2645 2646
	pci_cleanup_aer_uncorrect_error_status(pdev);
}

2647
static const struct pci_error_handlers nvme_err_handler = {
M
Matthew Wilcox 已提交
2648 2649 2650
	.error_detected	= nvme_error_detected,
	.slot_reset	= nvme_slot_reset,
	.resume		= nvme_error_resume,
2651 2652
	.reset_prepare	= nvme_reset_prepare,
	.reset_done	= nvme_reset_done,
M
Matthew Wilcox 已提交
2653 2654
};

2655
static const struct pci_device_id nvme_id_table[] = {
2656
	{ PCI_VDEVICE(INTEL, 0x0953),
2657
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
2658
				NVME_QUIRK_DEALLOCATE_ZEROES, },
2659 2660
	{ PCI_VDEVICE(INTEL, 0x0a53),
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
2661
				NVME_QUIRK_DEALLOCATE_ZEROES, },
2662 2663
	{ PCI_VDEVICE(INTEL, 0x0a54),
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
2664
				NVME_QUIRK_DEALLOCATE_ZEROES, },
2665 2666 2667
	{ PCI_VDEVICE(INTEL, 0x0a55),
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
				NVME_QUIRK_DEALLOCATE_ZEROES, },
2668
	{ PCI_VDEVICE(INTEL, 0xf1a5),	/* Intel 600P/P3100 */
2669 2670
		.driver_data = NVME_QUIRK_NO_DEEPEST_PS |
				NVME_QUIRK_MEDIUM_PRIO_SQ },
2671 2672
	{ PCI_VDEVICE(INTEL, 0x5845),	/* Qemu emulated controller */
		.driver_data = NVME_QUIRK_IDENTIFY_CNS, },
2673 2674
	{ PCI_DEVICE(0x1bb1, 0x0100),   /* Seagate Nytro Flash Storage */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2675 2676
	{ PCI_DEVICE(0x1c58, 0x0003),	/* HGST adapter */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2677 2678
	{ PCI_DEVICE(0x1c58, 0x0023),	/* WDC SN200 adapter */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2679 2680
	{ PCI_DEVICE(0x1c5f, 0x0540),	/* Memblaze Pblaze4 adapter */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2681 2682 2683 2684
	{ PCI_DEVICE(0x144d, 0xa821),   /* Samsung PM1725 */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
	{ PCI_DEVICE(0x144d, 0xa822),   /* Samsung PM1725a */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
C
Christoph Hellwig 已提交
2685 2686 2687 2688
	{ PCI_DEVICE(0x1d1d, 0x1f1f),	/* LighNVM qemu device */
		.driver_data = NVME_QUIRK_LIGHTNVM, },
	{ PCI_DEVICE(0x1d1d, 0x2807),	/* CNEX WL */
		.driver_data = NVME_QUIRK_LIGHTNVM, },
W
Wei Xu 已提交
2689 2690
	{ PCI_DEVICE(0x1d1d, 0x2601),	/* CNEX Granby */
		.driver_data = NVME_QUIRK_LIGHTNVM, },
M
Matthew Wilcox 已提交
2691
	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2692
	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
2693
	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
M
Matthew Wilcox 已提交
2694 2695 2696 2697 2698 2699 2700 2701
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, nvme_id_table);

static struct pci_driver nvme_driver = {
	.name		= "nvme",
	.id_table	= nvme_id_table,
	.probe		= nvme_probe,
2702
	.remove		= nvme_remove,
2703
	.shutdown	= nvme_shutdown,
2704 2705 2706
	.driver		= {
		.pm	= &nvme_dev_pm_ops,
	},
2707
	.sriov_configure = pci_sriov_configure_simple,
M
Matthew Wilcox 已提交
2708 2709 2710 2711 2712
	.err_handler	= &nvme_err_handler,
};

static int __init nvme_init(void)
{
2713
	return pci_register_driver(&nvme_driver);
M
Matthew Wilcox 已提交
2714 2715 2716 2717 2718
}

static void __exit nvme_exit(void)
{
	pci_unregister_driver(&nvme_driver);
2719
	flush_workqueue(nvme_wq);
2720
	_nvme_check_size();
M
Matthew Wilcox 已提交
2721 2722 2723 2724
}

MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
MODULE_LICENSE("GPL");
2725
MODULE_VERSION("1.0");
M
Matthew Wilcox 已提交
2726 2727
module_init(nvme_init);
module_exit(nvme_exit);