pci.c 70.4 KB
Newer Older
M
Matthew Wilcox 已提交
1 2
/*
 * NVM Express device driver
3
 * Copyright (c) 2011-2014, Intel Corporation.
M
Matthew Wilcox 已提交
4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

K
Keith Busch 已提交
15
#include <linux/aer.h>
16
#include <linux/async.h>
M
Matthew Wilcox 已提交
17
#include <linux/blkdev.h>
M
Matias Bjørling 已提交
18
#include <linux/blk-mq.h>
19
#include <linux/blk-mq-pci.h>
20
#include <linux/dmi.h>
M
Matthew Wilcox 已提交
21 22 23 24 25
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/mm.h>
#include <linux/module.h>
26
#include <linux/mutex.h>
27
#include <linux/once.h>
M
Matthew Wilcox 已提交
28
#include <linux/pci.h>
K
Keith Busch 已提交
29
#include <linux/t10-pi.h>
M
Matthew Wilcox 已提交
30
#include <linux/types.h>
31
#include <linux/io-64-nonatomic-lo-hi.h>
32
#include <linux/sed-opal.h>
33

34 35
#include "nvme.h"

M
Matthew Wilcox 已提交
36 37
#define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))
#define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))
38

C
Chaitanya Kulkarni 已提交
39
#define SGES_PER_PAGE	(PAGE_SIZE / sizeof(struct nvme_sgl_desc))
40

41 42 43
static int use_threaded_interrupts;
module_param(use_threaded_interrupts, int, 0);

44 45 46 47
static bool use_cmb_sqes = true;
module_param(use_cmb_sqes, bool, 0644);
MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");

48 49 50 51
static unsigned int max_host_mem_size_mb = 128;
module_param(max_host_mem_size_mb, uint, 0444);
MODULE_PARM_DESC(max_host_mem_size_mb,
	"Maximum Host Memory Buffer (HMB) size per controller (in MiB)");
52

C
Chaitanya Kulkarni 已提交
53 54 55 56 57 58
static unsigned int sgl_threshold = SZ_32K;
module_param(sgl_threshold, uint, 0644);
MODULE_PARM_DESC(sgl_threshold,
		"Use SGLs when average request segment size is larger or equal to "
		"this size. Use 0 to disable SGLs.");

59 60 61 62 63 64 65 66 67 68
static int io_queue_depth_set(const char *val, const struct kernel_param *kp);
static const struct kernel_param_ops io_queue_depth_ops = {
	.set = io_queue_depth_set,
	.get = param_get_int,
};

static int io_queue_depth = 1024;
module_param_cb(io_queue_depth, &io_queue_depth_ops, &io_queue_depth, 0644);
MODULE_PARM_DESC(io_queue_depth, "set io queue depth, should >= 2");

69 70
struct nvme_dev;
struct nvme_queue;
71

72
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
73

74 75 76 77
/*
 * Represents an NVM Express device.  Each nvme_dev is a PCI function.
 */
struct nvme_dev {
78
	struct nvme_queue *queues;
79 80 81 82 83 84 85 86
	struct blk_mq_tag_set tagset;
	struct blk_mq_tag_set admin_tagset;
	u32 __iomem *dbs;
	struct device *dev;
	struct dma_pool *prp_page_pool;
	struct dma_pool *prp_small_pool;
	unsigned online_queues;
	unsigned max_qid;
87
	unsigned int num_vecs;
88 89 90
	int q_depth;
	u32 db_stride;
	void __iomem *bar;
91
	unsigned long bar_mapped_size;
92
	struct work_struct remove_work;
93
	struct mutex shutdown_lock;
94 95
	bool subsystem;
	void __iomem *cmb;
96
	pci_bus_addr_t cmb_bus_addr;
97 98
	u64 cmb_size;
	u32 cmbsz;
99
	u32 cmbloc;
100
	struct nvme_ctrl ctrl;
K
Keith Busch 已提交
101
	struct completion ioq_wait;
102 103

	/* shadow doorbell buffer support: */
104 105 106 107
	u32 *dbbuf_dbs;
	dma_addr_t dbbuf_dbs_dma_addr;
	u32 *dbbuf_eis;
	dma_addr_t dbbuf_eis_dma_addr;
108 109 110 111

	/* host memory buffer support: */
	u64 host_mem_size;
	u32 nr_host_mem_descs;
112
	dma_addr_t host_mem_descs_dma;
113 114
	struct nvme_host_mem_buf_desc *host_mem_descs;
	void **host_mem_desc_bufs;
K
Keith Busch 已提交
115
};
116

117 118 119 120 121 122 123 124 125 126 127
static int io_queue_depth_set(const char *val, const struct kernel_param *kp)
{
	int n = 0, ret;

	ret = kstrtoint(val, 10, &n);
	if (ret != 0 || n < 2)
		return -EINVAL;

	return param_set_int(val, kp);
}

128 129 130 131 132 133 134 135 136 137
static inline unsigned int sq_idx(unsigned int qid, u32 stride)
{
	return qid * 2 * stride;
}

static inline unsigned int cq_idx(unsigned int qid, u32 stride)
{
	return (qid * 2 + 1) * stride;
}

138 139 140 141 142
static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
{
	return container_of(ctrl, struct nvme_dev, ctrl);
}

M
Matthew Wilcox 已提交
143 144 145 146 147 148
/*
 * An NVM Express queue.  Each device has at least two (one for admin
 * commands and one for I/O commands).
 */
struct nvme_queue {
	struct device *q_dmadev;
M
Matthew Wilcox 已提交
149
	struct nvme_dev *dev;
M
Matthew Wilcox 已提交
150 151
	spinlock_t q_lock;
	struct nvme_command *sq_cmds;
152
	struct nvme_command __iomem *sq_cmds_io;
M
Matthew Wilcox 已提交
153
	volatile struct nvme_completion *cqes;
154
	struct blk_mq_tags **tags;
M
Matthew Wilcox 已提交
155 156 157 158
	dma_addr_t sq_dma_addr;
	dma_addr_t cq_dma_addr;
	u32 __iomem *q_db;
	u16 q_depth;
J
Jens Axboe 已提交
159
	s16 cq_vector;
M
Matthew Wilcox 已提交
160 161
	u16 sq_tail;
	u16 cq_head;
K
Keith Busch 已提交
162
	u16 qid;
163
	u8 cq_phase;
164 165 166 167
	u32 *dbbuf_sq_db;
	u32 *dbbuf_cq_db;
	u32 *dbbuf_sq_ei;
	u32 *dbbuf_cq_ei;
M
Matthew Wilcox 已提交
168 169
};

170 171 172
/*
 * The nvme_iod describes the data in an I/O, including the list of PRP
 * entries.  You can't see it in this data structure because C doesn't let
C
Christoph Hellwig 已提交
173
 * me express that.  Use nvme_init_iod to ensure there's enough space
174 175 176
 * allocated to store the PRP list.
 */
struct nvme_iod {
177
	struct nvme_request req;
C
Christoph Hellwig 已提交
178
	struct nvme_queue *nvmeq;
C
Chaitanya Kulkarni 已提交
179
	bool use_sgl;
C
Christoph Hellwig 已提交
180
	int aborted;
181 182 183 184
	int npages;		/* In the PRP list. 0 means small pool in use */
	int nents;		/* Used in scatterlist */
	int length;		/* Of data, in bytes */
	dma_addr_t first_dma;
185
	struct scatterlist meta_sg; /* metadata requires single contiguous buffer */
C
Christoph Hellwig 已提交
186 187
	struct scatterlist *sg;
	struct scatterlist inline_sg[0];
M
Matthew Wilcox 已提交
188 189 190 191 192 193 194 195 196 197 198 199
};

/*
 * Check we didin't inadvertently grow the command struct
 */
static inline void _nvme_check_size(void)
{
	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
200
	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
K
Keith Busch 已提交
201
	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
M
Matthew Wilcox 已提交
202
	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
203 204
	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != NVME_IDENTIFY_DATA_SIZE);
	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != NVME_IDENTIFY_DATA_SIZE);
M
Matthew Wilcox 已提交
205
	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
K
Keith Busch 已提交
206
	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280
	BUILD_BUG_ON(sizeof(struct nvme_dbbuf) != 64);
}

static inline unsigned int nvme_dbbuf_size(u32 stride)
{
	return ((num_possible_cpus() + 1) * 8 * stride);
}

static int nvme_dbbuf_dma_alloc(struct nvme_dev *dev)
{
	unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);

	if (dev->dbbuf_dbs)
		return 0;

	dev->dbbuf_dbs = dma_alloc_coherent(dev->dev, mem_size,
					    &dev->dbbuf_dbs_dma_addr,
					    GFP_KERNEL);
	if (!dev->dbbuf_dbs)
		return -ENOMEM;
	dev->dbbuf_eis = dma_alloc_coherent(dev->dev, mem_size,
					    &dev->dbbuf_eis_dma_addr,
					    GFP_KERNEL);
	if (!dev->dbbuf_eis) {
		dma_free_coherent(dev->dev, mem_size,
				  dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
		dev->dbbuf_dbs = NULL;
		return -ENOMEM;
	}

	return 0;
}

static void nvme_dbbuf_dma_free(struct nvme_dev *dev)
{
	unsigned int mem_size = nvme_dbbuf_size(dev->db_stride);

	if (dev->dbbuf_dbs) {
		dma_free_coherent(dev->dev, mem_size,
				  dev->dbbuf_dbs, dev->dbbuf_dbs_dma_addr);
		dev->dbbuf_dbs = NULL;
	}
	if (dev->dbbuf_eis) {
		dma_free_coherent(dev->dev, mem_size,
				  dev->dbbuf_eis, dev->dbbuf_eis_dma_addr);
		dev->dbbuf_eis = NULL;
	}
}

static void nvme_dbbuf_init(struct nvme_dev *dev,
			    struct nvme_queue *nvmeq, int qid)
{
	if (!dev->dbbuf_dbs || !qid)
		return;

	nvmeq->dbbuf_sq_db = &dev->dbbuf_dbs[sq_idx(qid, dev->db_stride)];
	nvmeq->dbbuf_cq_db = &dev->dbbuf_dbs[cq_idx(qid, dev->db_stride)];
	nvmeq->dbbuf_sq_ei = &dev->dbbuf_eis[sq_idx(qid, dev->db_stride)];
	nvmeq->dbbuf_cq_ei = &dev->dbbuf_eis[cq_idx(qid, dev->db_stride)];
}

static void nvme_dbbuf_set(struct nvme_dev *dev)
{
	struct nvme_command c;

	if (!dev->dbbuf_dbs)
		return;

	memset(&c, 0, sizeof(c));
	c.dbbuf.opcode = nvme_admin_dbbuf;
	c.dbbuf.prp1 = cpu_to_le64(dev->dbbuf_dbs_dma_addr);
	c.dbbuf.prp2 = cpu_to_le64(dev->dbbuf_eis_dma_addr);

	if (nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0)) {
281
		dev_warn(dev->ctrl.device, "unable to set dbbuf\n");
282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312
		/* Free memory and continue on */
		nvme_dbbuf_dma_free(dev);
	}
}

static inline int nvme_dbbuf_need_event(u16 event_idx, u16 new_idx, u16 old)
{
	return (u16)(new_idx - event_idx - 1) < (u16)(new_idx - old);
}

/* Update dbbuf and return true if an MMIO is required */
static bool nvme_dbbuf_update_and_check_event(u16 value, u32 *dbbuf_db,
					      volatile u32 *dbbuf_ei)
{
	if (dbbuf_db) {
		u16 old_value;

		/*
		 * Ensure that the queue is written before updating
		 * the doorbell in memory
		 */
		wmb();

		old_value = *dbbuf_db;
		*dbbuf_db = value;

		if (!nvme_dbbuf_need_event(*dbbuf_ei, value, old_value))
			return false;
	}

	return true;
M
Matthew Wilcox 已提交
313 314
}

315 316 317 318
/*
 * Max size of iod being embedded in the request payload
 */
#define NVME_INT_PAGES		2
319
#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->ctrl.page_size)
320 321 322 323 324 325 326 327

/*
 * Will slightly overestimate the number of pages needed.  This is OK
 * as it only leads to a small amount of wasted memory for the lifetime of
 * the I/O.
 */
static int nvme_npages(unsigned size, struct nvme_dev *dev)
{
328 329
	unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
				      dev->ctrl.page_size);
330 331 332
	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}

C
Chaitanya Kulkarni 已提交
333 334 335 336 337
/*
 * Calculates the number of pages needed for the SGL segments. For example a 4k
 * page can accommodate 256 SGL descriptors.
 */
static int nvme_pci_npages_sgl(unsigned int num_seg)
338
{
C
Chaitanya Kulkarni 已提交
339
	return DIV_ROUND_UP(num_seg * sizeof(struct nvme_sgl_desc), PAGE_SIZE);
C
Christoph Hellwig 已提交
340
}
341

C
Chaitanya Kulkarni 已提交
342 343
static unsigned int nvme_pci_iod_alloc_size(struct nvme_dev *dev,
		unsigned int size, unsigned int nseg, bool use_sgl)
C
Christoph Hellwig 已提交
344
{
C
Chaitanya Kulkarni 已提交
345 346 347 348 349 350 351 352
	size_t alloc_size;

	if (use_sgl)
		alloc_size = sizeof(__le64 *) * nvme_pci_npages_sgl(nseg);
	else
		alloc_size = sizeof(__le64 *) * nvme_npages(size, dev);

	return alloc_size + sizeof(struct scatterlist) * nseg;
C
Christoph Hellwig 已提交
353
}
354

C
Chaitanya Kulkarni 已提交
355
static unsigned int nvme_pci_cmd_size(struct nvme_dev *dev, bool use_sgl)
C
Christoph Hellwig 已提交
356
{
C
Chaitanya Kulkarni 已提交
357 358 359 360 361
	unsigned int alloc_size = nvme_pci_iod_alloc_size(dev,
				    NVME_INT_BYTES(dev), NVME_INT_PAGES,
				    use_sgl);

	return sizeof(struct nvme_iod) + alloc_size;
362 363
}

M
Matias Bjørling 已提交
364 365
static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
				unsigned int hctx_idx)
366
{
M
Matias Bjørling 已提交
367
	struct nvme_dev *dev = data;
368
	struct nvme_queue *nvmeq = &dev->queues[0];
M
Matias Bjørling 已提交
369

370 371 372 373
	WARN_ON(hctx_idx != 0);
	WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
	WARN_ON(nvmeq->tags);

M
Matias Bjørling 已提交
374
	hctx->driver_data = nvmeq;
375
	nvmeq->tags = &dev->admin_tagset.tags[0];
M
Matias Bjørling 已提交
376
	return 0;
377 378
}

379 380 381 382 383 384 385
static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	nvmeq->tags = NULL;
}

M
Matias Bjørling 已提交
386 387
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			  unsigned int hctx_idx)
M
Matthew Wilcox 已提交
388
{
M
Matias Bjørling 已提交
389
	struct nvme_dev *dev = data;
390
	struct nvme_queue *nvmeq = &dev->queues[hctx_idx + 1];
M
Matias Bjørling 已提交
391

392 393
	if (!nvmeq->tags)
		nvmeq->tags = &dev->tagset.tags[hctx_idx];
M
Matthew Wilcox 已提交
394

395
	WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
M
Matias Bjørling 已提交
396 397
	hctx->driver_data = nvmeq;
	return 0;
M
Matthew Wilcox 已提交
398 399
}

400 401
static int nvme_init_request(struct blk_mq_tag_set *set, struct request *req,
		unsigned int hctx_idx, unsigned int numa_node)
M
Matthew Wilcox 已提交
402
{
403
	struct nvme_dev *dev = set->driver_data;
C
Christoph Hellwig 已提交
404
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
405
	int queue_idx = (set == &dev->tagset) ? hctx_idx + 1 : 0;
406
	struct nvme_queue *nvmeq = &dev->queues[queue_idx];
M
Matias Bjørling 已提交
407 408

	BUG_ON(!nvmeq);
C
Christoph Hellwig 已提交
409
	iod->nvmeq = nvmeq;
M
Matias Bjørling 已提交
410 411 412
	return 0;
}

413 414 415 416
static int nvme_pci_map_queues(struct blk_mq_tag_set *set)
{
	struct nvme_dev *dev = set->driver_data;

417 418
	return blk_mq_pci_map_queues(set, to_pci_dev(dev->dev),
			dev->num_vecs > 1 ? 1 /* admin queue */ : 0);
419 420
}

M
Matthew Wilcox 已提交
421
/**
422
 * __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
M
Matthew Wilcox 已提交
423 424 425 426 427
 * @nvmeq: The queue to use
 * @cmd: The command to send
 *
 * Safe to use from interrupt context
 */
428 429
static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
						struct nvme_command *cmd)
M
Matthew Wilcox 已提交
430
{
M
Matias Bjørling 已提交
431 432
	u16 tail = nvmeq->sq_tail;

433 434 435 436 437
	if (nvmeq->sq_cmds_io)
		memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd));
	else
		memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));

M
Matthew Wilcox 已提交
438 439
	if (++tail == nvmeq->q_depth)
		tail = 0;
440 441 442
	if (nvme_dbbuf_update_and_check_event(tail, nvmeq->dbbuf_sq_db,
					      nvmeq->dbbuf_sq_ei))
		writel(tail, nvmeq->q_db);
M
Matthew Wilcox 已提交
443 444 445
	nvmeq->sq_tail = tail;
}

C
Chaitanya Kulkarni 已提交
446
static void **nvme_pci_iod_list(struct request *req)
M
Matthew Wilcox 已提交
447
{
C
Christoph Hellwig 已提交
448
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
C
Chaitanya Kulkarni 已提交
449
	return (void **)(iod->sg + blk_rq_nr_phys_segments(req));
M
Matthew Wilcox 已提交
450 451
}

452 453 454
static inline bool nvme_pci_use_sgls(struct nvme_dev *dev, struct request *req)
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
455
	int nseg = blk_rq_nr_phys_segments(req);
456 457
	unsigned int avg_seg_size;

458 459 460 461
	if (nseg == 0)
		return false;

	avg_seg_size = DIV_ROUND_UP(blk_rq_payload_bytes(req), nseg);
462 463 464 465 466 467 468 469 470 471

	if (!(dev->ctrl.sgls & ((1 << 0) | (1 << 1))))
		return false;
	if (!iod->nvmeq->qid)
		return false;
	if (!sgl_threshold || avg_seg_size < sgl_threshold)
		return false;
	return true;
}

472
static blk_status_t nvme_init_iod(struct request *rq, struct nvme_dev *dev)
473
{
C
Christoph Hellwig 已提交
474
	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq);
475
	int nseg = blk_rq_nr_phys_segments(rq);
476
	unsigned int size = blk_rq_payload_bytes(rq);
477

478 479
	iod->use_sgl = nvme_pci_use_sgls(dev, rq);

C
Christoph Hellwig 已提交
480
	if (nseg > NVME_INT_PAGES || size > NVME_INT_BYTES(dev)) {
C
Chaitanya Kulkarni 已提交
481 482 483 484
		size_t alloc_size = nvme_pci_iod_alloc_size(dev, size, nseg,
				iod->use_sgl);

		iod->sg = kmalloc(alloc_size, GFP_ATOMIC);
C
Christoph Hellwig 已提交
485
		if (!iod->sg)
486
			return BLK_STS_RESOURCE;
C
Christoph Hellwig 已提交
487 488
	} else {
		iod->sg = iod->inline_sg;
489 490
	}

C
Christoph Hellwig 已提交
491 492 493 494
	iod->aborted = 0;
	iod->npages = -1;
	iod->nents = 0;
	iod->length = size;
K
Keith Busch 已提交
495

496
	return BLK_STS_OK;
497 498
}

C
Christoph Hellwig 已提交
499
static void nvme_free_iod(struct nvme_dev *dev, struct request *req)
M
Matthew Wilcox 已提交
500
{
C
Christoph Hellwig 已提交
501
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
C
Chaitanya Kulkarni 已提交
502 503 504
	const int last_prp = dev->ctrl.page_size / sizeof(__le64) - 1;
	dma_addr_t dma_addr = iod->first_dma, next_dma_addr;

505 506 507
	int i;

	if (iod->npages == 0)
C
Chaitanya Kulkarni 已提交
508 509 510
		dma_pool_free(dev->prp_small_pool, nvme_pci_iod_list(req)[0],
			dma_addr);

511
	for (i = 0; i < iod->npages; i++) {
C
Chaitanya Kulkarni 已提交
512 513 514 515 516 517 518 519 520 521 522 523 524 525 526
		void *addr = nvme_pci_iod_list(req)[i];

		if (iod->use_sgl) {
			struct nvme_sgl_desc *sg_list = addr;

			next_dma_addr =
			    le64_to_cpu((sg_list[SGES_PER_PAGE - 1]).addr);
		} else {
			__le64 *prp_list = addr;

			next_dma_addr = le64_to_cpu(prp_list[last_prp]);
		}

		dma_pool_free(dev->prp_page_pool, addr, dma_addr);
		dma_addr = next_dma_addr;
527
	}
528

C
Christoph Hellwig 已提交
529 530
	if (iod->sg != iod->inline_sg)
		kfree(iod->sg);
K
Keith Busch 已提交
531 532
}

533
#ifdef CONFIG_BLK_DEV_INTEGRITY
K
Keith Busch 已提交
534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == v)
		pi->ref_tag = cpu_to_be32(p);
}

static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == p)
		pi->ref_tag = cpu_to_be32(v);
}

/**
 * nvme_dif_remap - remaps ref tags to bip seed and physical lba
 *
 * The virtual start sector is the one that was originally submitted by the
 * block layer.	Due to partitioning, MD/DM cloning, etc. the actual physical
 * start sector may be different. Remap protection information to match the
 * physical LBA on writes, and back to the original seed on reads.
 *
 * Type 0 and 3 do not have a ref tag, so no remapping required.
 */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
	struct nvme_ns *ns = req->rq_disk->private_data;
	struct bio_integrity_payload *bip;
	struct t10_pi_tuple *pi;
	void *p, *pmap;
	u32 i, nlb, ts, phys, virt;

	if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
		return;

	bip = bio_integrity(req->bio);
	if (!bip)
		return;

	pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;

	p = pmap;
	virt = bip_get_seed(bip);
	phys = nvme_block_nr(ns, blk_rq_pos(req));
	nlb = (blk_rq_bytes(req) >> ns->lba_shift);
578
	ts = ns->disk->queue->integrity.tuple_size;
K
Keith Busch 已提交
579 580 581 582 583 584 585 586

	for (i = 0; i < nlb; i++, virt++, phys++) {
		pi = (struct t10_pi_tuple *)p;
		dif_swap(phys, virt, pi);
		p += ts;
	}
	kunmap_atomic(pmap);
}
587 588 589 590 591 592 593 594 595 596 597 598 599
#else /* CONFIG_BLK_DEV_INTEGRITY */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
}
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
#endif

600 601 602 603 604 605 606 607 608 609 610 611 612 613
static void nvme_print_sgl(struct scatterlist *sgl, int nents)
{
	int i;
	struct scatterlist *sg;

	for_each_sg(sgl, sg, nents, i) {
		dma_addr_t phys = sg_phys(sg);
		pr_warn("sg[%d] phys_addr:%pad offset:%d length:%d "
			"dma_address:%pad dma_length:%d\n",
			i, &phys, sg->offset, sg->length, &sg_dma_address(sg),
			sg_dma_len(sg));
	}
}

C
Chaitanya Kulkarni 已提交
614 615
static blk_status_t nvme_pci_setup_prps(struct nvme_dev *dev,
		struct request *req, struct nvme_rw_command *cmnd)
M
Matthew Wilcox 已提交
616
{
C
Christoph Hellwig 已提交
617
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
618
	struct dma_pool *pool;
619
	int length = blk_rq_payload_bytes(req);
620
	struct scatterlist *sg = iod->sg;
M
Matthew Wilcox 已提交
621 622
	int dma_len = sg_dma_len(sg);
	u64 dma_addr = sg_dma_address(sg);
623
	u32 page_size = dev->ctrl.page_size;
624
	int offset = dma_addr & (page_size - 1);
625
	__le64 *prp_list;
C
Chaitanya Kulkarni 已提交
626
	void **list = nvme_pci_iod_list(req);
627
	dma_addr_t prp_dma;
628
	int nprps, i;
M
Matthew Wilcox 已提交
629

630
	length -= (page_size - offset);
631 632
	if (length <= 0) {
		iod->first_dma = 0;
C
Chaitanya Kulkarni 已提交
633
		goto done;
634
	}
M
Matthew Wilcox 已提交
635

636
	dma_len -= (page_size - offset);
M
Matthew Wilcox 已提交
637
	if (dma_len) {
638
		dma_addr += (page_size - offset);
M
Matthew Wilcox 已提交
639 640 641 642 643 644
	} else {
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
	}

645
	if (length <= page_size) {
646
		iod->first_dma = dma_addr;
C
Chaitanya Kulkarni 已提交
647
		goto done;
648 649
	}

650
	nprps = DIV_ROUND_UP(length, page_size);
651 652
	if (nprps <= (256 / 8)) {
		pool = dev->prp_small_pool;
653
		iod->npages = 0;
654 655
	} else {
		pool = dev->prp_page_pool;
656
		iod->npages = 1;
657 658
	}

659
	prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
660
	if (!prp_list) {
661
		iod->first_dma = dma_addr;
662
		iod->npages = -1;
663
		return BLK_STS_RESOURCE;
664
	}
665 666
	list[0] = prp_list;
	iod->first_dma = prp_dma;
667 668
	i = 0;
	for (;;) {
669
		if (i == page_size >> 3) {
670
			__le64 *old_prp_list = prp_list;
671
			prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
672
			if (!prp_list)
673
				return BLK_STS_RESOURCE;
674
			list[iod->npages++] = prp_list;
675 676 677
			prp_list[0] = old_prp_list[i - 1];
			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
			i = 1;
678 679
		}
		prp_list[i++] = cpu_to_le64(dma_addr);
680 681 682
		dma_len -= page_size;
		dma_addr += page_size;
		length -= page_size;
683 684 685 686
		if (length <= 0)
			break;
		if (dma_len > 0)
			continue;
687 688
		if (unlikely(dma_len < 0))
			goto bad_sgl;
689 690 691
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
M
Matthew Wilcox 已提交
692 693
	}

C
Chaitanya Kulkarni 已提交
694 695 696 697
done:
	cmnd->dptr.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
	cmnd->dptr.prp2 = cpu_to_le64(iod->first_dma);

698 699 700
	return BLK_STS_OK;

 bad_sgl:
701 702 703
	WARN(DO_ONCE(nvme_print_sgl, iod->sg, iod->nents),
			"Invalid SGL for payload:%d nents:%d\n",
			blk_rq_payload_bytes(req), iod->nents);
704
	return BLK_STS_IOERR;
M
Matthew Wilcox 已提交
705 706
}

C
Chaitanya Kulkarni 已提交
707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728
static void nvme_pci_sgl_set_data(struct nvme_sgl_desc *sge,
		struct scatterlist *sg)
{
	sge->addr = cpu_to_le64(sg_dma_address(sg));
	sge->length = cpu_to_le32(sg_dma_len(sg));
	sge->type = NVME_SGL_FMT_DATA_DESC << 4;
}

static void nvme_pci_sgl_set_seg(struct nvme_sgl_desc *sge,
		dma_addr_t dma_addr, int entries)
{
	sge->addr = cpu_to_le64(dma_addr);
	if (entries < SGES_PER_PAGE) {
		sge->length = cpu_to_le32(entries * sizeof(*sge));
		sge->type = NVME_SGL_FMT_LAST_SEG_DESC << 4;
	} else {
		sge->length = cpu_to_le32(PAGE_SIZE);
		sge->type = NVME_SGL_FMT_SEG_DESC << 4;
	}
}

static blk_status_t nvme_pci_setup_sgls(struct nvme_dev *dev,
729
		struct request *req, struct nvme_rw_command *cmd, int entries)
C
Chaitanya Kulkarni 已提交
730 731 732 733 734 735
{
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct dma_pool *pool;
	struct nvme_sgl_desc *sg_list;
	struct scatterlist *sg = iod->sg;
	dma_addr_t sgl_dma;
736
	int i = 0;
C
Chaitanya Kulkarni 已提交
737 738 739 740

	/* setting the transfer type as SGL */
	cmd->flags = NVME_CMD_SGL_METABUF;

741
	if (entries == 1) {
C
Chaitanya Kulkarni 已提交
742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781
		nvme_pci_sgl_set_data(&cmd->dptr.sgl, sg);
		return BLK_STS_OK;
	}

	if (entries <= (256 / sizeof(struct nvme_sgl_desc))) {
		pool = dev->prp_small_pool;
		iod->npages = 0;
	} else {
		pool = dev->prp_page_pool;
		iod->npages = 1;
	}

	sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
	if (!sg_list) {
		iod->npages = -1;
		return BLK_STS_RESOURCE;
	}

	nvme_pci_iod_list(req)[0] = sg_list;
	iod->first_dma = sgl_dma;

	nvme_pci_sgl_set_seg(&cmd->dptr.sgl, sgl_dma, entries);

	do {
		if (i == SGES_PER_PAGE) {
			struct nvme_sgl_desc *old_sg_desc = sg_list;
			struct nvme_sgl_desc *link = &old_sg_desc[i - 1];

			sg_list = dma_pool_alloc(pool, GFP_ATOMIC, &sgl_dma);
			if (!sg_list)
				return BLK_STS_RESOURCE;

			i = 0;
			nvme_pci_iod_list(req)[iod->npages++] = sg_list;
			sg_list[i++] = *link;
			nvme_pci_sgl_set_seg(link, sgl_dma, entries);
		}

		nvme_pci_sgl_set_data(&sg_list[i++], sg);
		sg = sg_next(sg);
782
	} while (--entries > 0);
C
Chaitanya Kulkarni 已提交
783 784 785 786

	return BLK_STS_OK;
}

787
static blk_status_t nvme_map_data(struct nvme_dev *dev, struct request *req,
788
		struct nvme_command *cmnd)
789
{
C
Christoph Hellwig 已提交
790
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
C
Christoph Hellwig 已提交
791 792 793
	struct request_queue *q = req->q;
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;
794
	blk_status_t ret = BLK_STS_IOERR;
795
	int nr_mapped;
796

797
	sg_init_table(iod->sg, blk_rq_nr_phys_segments(req));
C
Christoph Hellwig 已提交
798 799 800
	iod->nents = blk_rq_map_sg(q, req, iod->sg);
	if (!iod->nents)
		goto out;
801

802
	ret = BLK_STS_RESOURCE;
803 804 805
	nr_mapped = dma_map_sg_attrs(dev->dev, iod->sg, iod->nents, dma_dir,
			DMA_ATTR_NO_WARN);
	if (!nr_mapped)
C
Christoph Hellwig 已提交
806
		goto out;
807

808
	if (iod->use_sgl)
809
		ret = nvme_pci_setup_sgls(dev, req, &cmnd->rw, nr_mapped);
C
Chaitanya Kulkarni 已提交
810 811 812
	else
		ret = nvme_pci_setup_prps(dev, req, &cmnd->rw);

813
	if (ret != BLK_STS_OK)
C
Christoph Hellwig 已提交
814
		goto out_unmap;
815

816
	ret = BLK_STS_IOERR;
C
Christoph Hellwig 已提交
817 818 819
	if (blk_integrity_rq(req)) {
		if (blk_rq_count_integrity_sg(q, req->bio) != 1)
			goto out_unmap;
820

821 822
		sg_init_table(&iod->meta_sg, 1);
		if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1)
C
Christoph Hellwig 已提交
823
			goto out_unmap;
824

825
		if (req_op(req) == REQ_OP_WRITE)
C
Christoph Hellwig 已提交
826
			nvme_dif_remap(req, nvme_dif_prep);
827

828
		if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir))
C
Christoph Hellwig 已提交
829
			goto out_unmap;
830
	}
M
Matthew Wilcox 已提交
831

C
Christoph Hellwig 已提交
832
	if (blk_integrity_rq(req))
833
		cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg));
834
	return BLK_STS_OK;
M
Matthew Wilcox 已提交
835

C
Christoph Hellwig 已提交
836 837 838 839
out_unmap:
	dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
out:
	return ret;
M
Matthew Wilcox 已提交
840 841
}

C
Christoph Hellwig 已提交
842
static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
M
Matthew Wilcox 已提交
843
{
C
Christoph Hellwig 已提交
844
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
845 846 847 848 849 850
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;

	if (iod->nents) {
		dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
		if (blk_integrity_rq(req)) {
851
			if (req_op(req) == REQ_OP_READ)
852
				nvme_dif_remap(req, nvme_dif_complete);
853
			dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir);
K
Keith Busch 已提交
854
		}
855
	}
K
Keith Busch 已提交
856

857
	nvme_cleanup_cmd(req);
C
Christoph Hellwig 已提交
858
	nvme_free_iod(dev, req);
859
}
M
Matthew Wilcox 已提交
860

861 862 863
/*
 * NOTE: ns is NULL when called on the admin queue.
 */
864
static blk_status_t nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
M
Matias Bjørling 已提交
865
			 const struct blk_mq_queue_data *bd)
866
{
M
Matias Bjørling 已提交
867 868
	struct nvme_ns *ns = hctx->queue->queuedata;
	struct nvme_queue *nvmeq = hctx->driver_data;
869
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
870
	struct request *req = bd->rq;
C
Christoph Hellwig 已提交
871
	struct nvme_command cmnd;
872
	blk_status_t ret;
K
Keith Busch 已提交
873

874 875 876 877 878 879 880
	/*
	 * We should not need to do this, but we're still using this to
	 * ensure we can drain requests on a dying queue.
	 */
	if (unlikely(nvmeq->cq_vector < 0))
		return BLK_STS_IOERR;

881
	ret = nvme_setup_cmd(ns, req, &cmnd);
882
	if (ret)
C
Christoph Hellwig 已提交
883
		return ret;
M
Matias Bjørling 已提交
884

885
	ret = nvme_init_iod(req, dev);
886
	if (ret)
887
		goto out_free_cmd;
M
Matias Bjørling 已提交
888

889
	if (blk_rq_nr_phys_segments(req)) {
890
		ret = nvme_map_data(dev, req, &cmnd);
891 892 893
		if (ret)
			goto out_cleanup_iod;
	}
M
Matias Bjørling 已提交
894

895
	blk_mq_start_request(req);
M
Matias Bjørling 已提交
896

C
Christoph Hellwig 已提交
897 898
	spin_lock_irq(&nvmeq->q_lock);
	__nvme_submit_cmd(nvmeq, &cmnd);
M
Matias Bjørling 已提交
899
	spin_unlock_irq(&nvmeq->q_lock);
900
	return BLK_STS_OK;
901
out_cleanup_iod:
C
Christoph Hellwig 已提交
902
	nvme_free_iod(dev, req);
903 904
out_free_cmd:
	nvme_cleanup_cmd(req);
C
Christoph Hellwig 已提交
905
	return ret;
M
Matthew Wilcox 已提交
906
}
K
Keith Busch 已提交
907

908
static void nvme_pci_complete_rq(struct request *req)
909
{
C
Christoph Hellwig 已提交
910
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
M
Matias Bjørling 已提交
911

912 913
	nvme_unmap_data(iod->nvmeq->dev, req);
	nvme_complete_rq(req);
M
Matthew Wilcox 已提交
914 915
}

916
/* We read the CQE phase first to check if the rest of the entry is valid */
917
static inline bool nvme_cqe_pending(struct nvme_queue *nvmeq)
918
{
919 920
	return (le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
			nvmeq->cq_phase;
921 922
}

923
static inline void nvme_ring_cq_doorbell(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
924
{
925
	u16 head = nvmeq->cq_head;
926

927 928 929 930 931 932
	if (likely(nvmeq->cq_vector >= 0)) {
		if (nvme_dbbuf_update_and_check_event(head, nvmeq->dbbuf_cq_db,
						      nvmeq->dbbuf_cq_ei))
			writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
	}
}
933

934
static inline void nvme_handle_cqe(struct nvme_queue *nvmeq, u16 idx)
935
{
936
	volatile struct nvme_completion *cqe = &nvmeq->cqes[idx];
937
	struct request *req;
938

939 940 941 942 943
	if (unlikely(cqe->command_id >= nvmeq->q_depth)) {
		dev_warn(nvmeq->dev->ctrl.device,
			"invalid id %d completed on queue %d\n",
			cqe->command_id, le16_to_cpu(cqe->sq_id));
		return;
M
Matthew Wilcox 已提交
944 945
	}

946 947 948 949 950 951 952
	/*
	 * AEN requests are special as they don't time out and can
	 * survive any kind of queue freeze and often don't respond to
	 * aborts.  We don't even bother to allocate a struct request
	 * for them but rather special case them here.
	 */
	if (unlikely(nvmeq->qid == 0 &&
K
Keith Busch 已提交
953
			cqe->command_id >= NVME_AQ_BLK_MQ_DEPTH)) {
954 955
		nvme_complete_async_event(&nvmeq->dev->ctrl,
				cqe->status, &cqe->result);
J
Jens Axboe 已提交
956
		return;
957
	}
M
Matthew Wilcox 已提交
958

959 960 961
	req = blk_mq_tag_to_rq(*nvmeq->tags, cqe->command_id);
	nvme_end_request(req, cqe->status, cqe->result);
}
M
Matthew Wilcox 已提交
962

963
static void nvme_complete_cqes(struct nvme_queue *nvmeq, u16 start, u16 end)
M
Matthew Wilcox 已提交
964
{
965 966 967 968 969 970
	while (start != end) {
		nvme_handle_cqe(nvmeq, start);
		if (++start == nvmeq->q_depth)
			start = 0;
	}
}
971

972 973 974 975 976
static inline void nvme_update_cq_head(struct nvme_queue *nvmeq)
{
	if (++nvmeq->cq_head == nvmeq->q_depth) {
		nvmeq->cq_head = 0;
		nvmeq->cq_phase = !nvmeq->cq_phase;
M
Matthew Wilcox 已提交
977
	}
J
Jens Axboe 已提交
978 979
}

980 981
static inline bool nvme_process_cq(struct nvme_queue *nvmeq, u16 *start,
		u16 *end, int tag)
J
Jens Axboe 已提交
982
{
983
	bool found = false;
M
Matthew Wilcox 已提交
984

985 986 987 988 989
	*start = nvmeq->cq_head;
	while (!found && nvme_cqe_pending(nvmeq)) {
		if (nvmeq->cqes[nvmeq->cq_head].command_id == tag)
			found = true;
		nvme_update_cq_head(nvmeq);
990
	}
991
	*end = nvmeq->cq_head;
992

993
	if (*start != *end)
994
		nvme_ring_cq_doorbell(nvmeq);
995
	return found;
M
Matthew Wilcox 已提交
996 997 998
}

static irqreturn_t nvme_irq(int irq, void *data)
999 1000
{
	struct nvme_queue *nvmeq = data;
1001 1002
	u16 start, end;

1003
	spin_lock(&nvmeq->q_lock);
1004
	nvme_process_cq(nvmeq, &start, &end, -1);
1005
	spin_unlock(&nvmeq->q_lock);
1006 1007 1008 1009 1010

	if (start == end)
		return IRQ_NONE;
	nvme_complete_cqes(nvmeq, start, end);
	return IRQ_HANDLED;
1011 1012 1013 1014 1015
}

static irqreturn_t nvme_irq_check(int irq, void *data)
{
	struct nvme_queue *nvmeq = data;
1016
	if (nvme_cqe_pending(nvmeq))
1017 1018
		return IRQ_WAKE_THREAD;
	return IRQ_NONE;
1019 1020
}

K
Keith Busch 已提交
1021
static int __nvme_poll(struct nvme_queue *nvmeq, unsigned int tag)
J
Jens Axboe 已提交
1022
{
1023 1024
	u16 start, end;
	bool found;
J
Jens Axboe 已提交
1025

1026
	if (!nvme_cqe_pending(nvmeq))
1027
		return 0;
J
Jens Axboe 已提交
1028

1029
	spin_lock_irq(&nvmeq->q_lock);
1030
	found = nvme_process_cq(nvmeq, &start, &end, tag);
1031 1032
	spin_unlock_irq(&nvmeq->q_lock);

1033
	nvme_complete_cqes(nvmeq, start, end);
1034
	return found;
J
Jens Axboe 已提交
1035 1036
}

K
Keith Busch 已提交
1037 1038 1039 1040 1041 1042 1043
static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	return __nvme_poll(nvmeq, tag);
}

1044
static void nvme_pci_submit_async_event(struct nvme_ctrl *ctrl)
M
Matthew Wilcox 已提交
1045
{
1046
	struct nvme_dev *dev = to_nvme_dev(ctrl);
1047
	struct nvme_queue *nvmeq = &dev->queues[0];
M
Matias Bjørling 已提交
1048
	struct nvme_command c;
M
Matthew Wilcox 已提交
1049

M
Matias Bjørling 已提交
1050 1051
	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_async_event;
1052
	c.common.command_id = NVME_AQ_BLK_MQ_DEPTH;
1053

1054
	spin_lock_irq(&nvmeq->q_lock);
1055
	__nvme_submit_cmd(nvmeq, &c);
1056
	spin_unlock_irq(&nvmeq->q_lock);
1057 1058
}

M
Matthew Wilcox 已提交
1059
static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
1060
{
M
Matthew Wilcox 已提交
1061 1062 1063 1064 1065 1066
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(id);

1067
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1068 1069 1070 1071 1072 1073 1074 1075
}

static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;

1076
	/*
M
Minwoo Im 已提交
1077
	 * Note: we (ab)use the fact that the prp fields survive if no data
1078 1079
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
1080 1081 1082 1083 1084 1085 1086 1087
	memset(&c, 0, sizeof(c));
	c.create_cq.opcode = nvme_admin_create_cq;
	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
	c.create_cq.cqid = cpu_to_le16(qid);
	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_cq.cq_flags = cpu_to_le16(flags);
	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);

1088
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1089 1090 1091 1092 1093 1094
}

static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
1095
	int flags = NVME_QUEUE_PHYS_CONTIG;
M
Matthew Wilcox 已提交
1096

1097
	/*
M
Minwoo Im 已提交
1098
	 * Note: we (ab)use the fact that the prp fields survive if no data
1099 1100
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
1101 1102 1103 1104 1105 1106 1107 1108
	memset(&c, 0, sizeof(c));
	c.create_sq.opcode = nvme_admin_create_sq;
	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
	c.create_sq.sqid = cpu_to_le16(qid);
	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_sq.sq_flags = cpu_to_le16(flags);
	c.create_sq.cqid = cpu_to_le16(qid);

1109
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
}

static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
}

static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
}

1122
static void abort_endio(struct request *req, blk_status_t error)
1123
{
C
Christoph Hellwig 已提交
1124 1125
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = iod->nvmeq;
1126

1127 1128
	dev_warn(nvmeq->dev->ctrl.device,
		 "Abort status: 0x%x", nvme_req(req)->status);
1129 1130
	atomic_inc(&nvmeq->dev->ctrl.abort_limit);
	blk_mq_free_request(req);
1131 1132
}

K
Keith Busch 已提交
1133 1134 1135 1136 1137 1138 1139 1140
static bool nvme_should_reset(struct nvme_dev *dev, u32 csts)
{

	/* If true, indicates loss of adapter communication, possibly by a
	 * NVMe Subsystem reset.
	 */
	bool nssro = dev->subsystem && (csts & NVME_CSTS_NSSRO);

1141 1142 1143
	/* If there is a reset/reinit ongoing, we shouldn't reset again. */
	switch (dev->ctrl.state) {
	case NVME_CTRL_RESETTING:
1144
	case NVME_CTRL_CONNECTING:
K
Keith Busch 已提交
1145
		return false;
1146 1147 1148
	default:
		break;
	}
K
Keith Busch 已提交
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176

	/* We shouldn't reset unless the controller is on fatal error state
	 * _or_ if we lost the communication with it.
	 */
	if (!(csts & NVME_CSTS_CFS) && !nssro)
		return false;

	return true;
}

static void nvme_warn_reset(struct nvme_dev *dev, u32 csts)
{
	/* Read a config register to help see what died. */
	u16 pci_status;
	int result;

	result = pci_read_config_word(to_pci_dev(dev->dev), PCI_STATUS,
				      &pci_status);
	if (result == PCIBIOS_SUCCESSFUL)
		dev_warn(dev->ctrl.device,
			 "controller is down; will reset: CSTS=0x%x, PCI_STATUS=0x%hx\n",
			 csts, pci_status);
	else
		dev_warn(dev->ctrl.device,
			 "controller is down; will reset: CSTS=0x%x, PCI_STATUS read failed (%d)\n",
			 csts, result);
}

1177
static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
K
Keith Busch 已提交
1178
{
C
Christoph Hellwig 已提交
1179 1180
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = iod->nvmeq;
K
Keith Busch 已提交
1181
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
1182 1183
	struct request *abort_req;
	struct nvme_command cmd;
K
Keith Busch 已提交
1184 1185
	u32 csts = readl(dev->bar + NVME_REG_CSTS);

W
Wen Xiong 已提交
1186 1187 1188 1189 1190 1191 1192
	/* If PCI error recovery process is happening, we cannot reset or
	 * the recovery mechanism will surely fail.
	 */
	mb();
	if (pci_channel_offline(to_pci_dev(dev->dev)))
		return BLK_EH_RESET_TIMER;

K
Keith Busch 已提交
1193 1194 1195 1196 1197 1198
	/*
	 * Reset immediately if the controller is failed
	 */
	if (nvme_should_reset(dev, csts)) {
		nvme_warn_reset(dev, csts);
		nvme_dev_disable(dev, false);
1199
		nvme_reset_ctrl(&dev->ctrl);
K
Keith Busch 已提交
1200 1201
		return BLK_EH_HANDLED;
	}
K
Keith Busch 已提交
1202

K
Keith Busch 已提交
1203 1204 1205 1206 1207 1208 1209 1210 1211 1212
	/*
	 * Did we miss an interrupt?
	 */
	if (__nvme_poll(nvmeq, req->tag)) {
		dev_warn(dev->ctrl.device,
			 "I/O %d QID %d timeout, completion polled\n",
			 req->tag, nvmeq->qid);
		return BLK_EH_HANDLED;
	}

1213
	/*
1214 1215 1216 1217 1218
	 * Shutdown immediately if controller times out while starting. The
	 * reset work will see the pci device disabled when it gets the forced
	 * cancellation error. All outstanding requests are completed on
	 * shutdown, so we return BLK_EH_HANDLED.
	 */
1219 1220 1221
	switch (dev->ctrl.state) {
	case NVME_CTRL_CONNECTING:
	case NVME_CTRL_RESETTING:
1222
		dev_warn(dev->ctrl.device,
1223 1224
			 "I/O %d QID %d timeout, disable controller\n",
			 req->tag, nvmeq->qid);
1225
		nvme_dev_disable(dev, false);
1226
		nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1227
		return BLK_EH_HANDLED;
1228 1229
	default:
		break;
K
Keith Busch 已提交
1230 1231
	}

1232 1233 1234 1235
	/*
 	 * Shutdown the controller immediately and schedule a reset if the
 	 * command was already aborted once before and still hasn't been
 	 * returned to the driver, or if this is the admin queue.
1236
	 */
C
Christoph Hellwig 已提交
1237
	if (!nvmeq->qid || iod->aborted) {
1238
		dev_warn(dev->ctrl.device,
1239 1240
			 "I/O %d QID %d timeout, reset controller\n",
			 req->tag, nvmeq->qid);
1241
		nvme_dev_disable(dev, false);
1242
		nvme_reset_ctrl(&dev->ctrl);
K
Keith Busch 已提交
1243

1244 1245 1246 1247
		/*
		 * Mark the request as handled, since the inline shutdown
		 * forces all outstanding requests to complete.
		 */
1248
		nvme_req(req)->flags |= NVME_REQ_CANCELLED;
1249
		return BLK_EH_HANDLED;
K
Keith Busch 已提交
1250 1251
	}

1252
	if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1253
		atomic_inc(&dev->ctrl.abort_limit);
1254
		return BLK_EH_RESET_TIMER;
1255
	}
1256
	iod->aborted = 1;
M
Matias Bjørling 已提交
1257

K
Keith Busch 已提交
1258 1259
	memset(&cmd, 0, sizeof(cmd));
	cmd.abort.opcode = nvme_admin_abort_cmd;
M
Matias Bjørling 已提交
1260
	cmd.abort.cid = req->tag;
K
Keith Busch 已提交
1261 1262
	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);

1263 1264 1265
	dev_warn(nvmeq->dev->ctrl.device,
		"I/O %d QID %d timeout, aborting\n",
		 req->tag, nvmeq->qid);
1266 1267

	abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
1268
			BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
1269 1270 1271 1272 1273 1274 1275 1276
	if (IS_ERR(abort_req)) {
		atomic_inc(&dev->ctrl.abort_limit);
		return BLK_EH_RESET_TIMER;
	}

	abort_req->timeout = ADMIN_TIMEOUT;
	abort_req->end_io_data = NULL;
	blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
K
Keith Busch 已提交
1277

1278 1279 1280 1281 1282 1283
	/*
	 * The aborted req will be completed on receiving the abort req.
	 * We enable the timer again. If hit twice, it'll cause a device reset,
	 * as the device then is in a faulty state.
	 */
	return BLK_EH_RESET_TIMER;
K
Keith Busch 已提交
1284 1285
}

M
Matias Bjørling 已提交
1286 1287
static void nvme_free_queue(struct nvme_queue *nvmeq)
{
1288 1289
	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1290 1291
	if (nvmeq->sq_cmds)
		dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1292 1293 1294
					nvmeq->sq_cmds, nvmeq->sq_dma_addr);
}

1295
static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1296 1297 1298
{
	int i;

1299 1300
	for (i = dev->ctrl.queue_count - 1; i >= lowest; i--) {
		dev->ctrl.queue_count--;
1301
		nvme_free_queue(&dev->queues[i]);
1302
	}
1303 1304
}

K
Keith Busch 已提交
1305 1306 1307 1308 1309
/**
 * nvme_suspend_queue - put queue into suspended state
 * @nvmeq - queue to suspend
 */
static int nvme_suspend_queue(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
1310
{
K
Keith Busch 已提交
1311
	int vector;
M
Matthew Wilcox 已提交
1312

1313
	spin_lock_irq(&nvmeq->q_lock);
K
Keith Busch 已提交
1314 1315 1316 1317
	if (nvmeq->cq_vector == -1) {
		spin_unlock_irq(&nvmeq->q_lock);
		return 1;
	}
1318
	vector = nvmeq->cq_vector;
K
Keith Busch 已提交
1319
	nvmeq->dev->online_queues--;
K
Keith Busch 已提交
1320
	nvmeq->cq_vector = -1;
1321 1322
	spin_unlock_irq(&nvmeq->q_lock);

1323 1324 1325 1326 1327 1328
	/*
	 * Ensure that nvme_queue_rq() sees it ->cq_vector == -1 without
	 * having to grab the lock.
	 */
	mb();

1329
	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1330
		blk_mq_quiesce_queue(nvmeq->dev->ctrl.admin_q);
1331

1332
	pci_free_irq(to_pci_dev(nvmeq->dev->dev), vector, nvmeq);
M
Matthew Wilcox 已提交
1333

K
Keith Busch 已提交
1334 1335
	return 0;
}
M
Matthew Wilcox 已提交
1336

1337
static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
K
Keith Busch 已提交
1338
{
1339
	struct nvme_queue *nvmeq = &dev->queues[0];
1340
	u16 start, end;
K
Keith Busch 已提交
1341

1342 1343 1344
	if (shutdown)
		nvme_shutdown_ctrl(&dev->ctrl);
	else
1345
		nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
1346 1347

	spin_lock_irq(&nvmeq->q_lock);
1348
	nvme_process_cq(nvmeq, &start, &end, -1);
1349
	spin_unlock_irq(&nvmeq->q_lock);
1350 1351

	nvme_complete_cqes(nvmeq, start, end);
M
Matthew Wilcox 已提交
1352 1353
}

1354 1355 1356 1357
static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
				int entry_size)
{
	int q_depth = dev->q_depth;
1358 1359
	unsigned q_size_aligned = roundup(q_depth * entry_size,
					  dev->ctrl.page_size);
1360 1361

	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1362
		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1363
		mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
1364
		q_depth = div_u64(mem_per_q, entry_size);
1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380

		/*
		 * Ensure the reduced q_depth is above some threshold where it
		 * would be better to map queues in system memory with the
		 * original depth
		 */
		if (q_depth < 64)
			return -ENOMEM;
	}

	return q_depth;
}

static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
				int qid, int depth)
{
1381 1382 1383
	/* CMB SQEs will be mapped before creation */
	if (qid && dev->cmb && use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS))
		return 0;
1384

1385 1386 1387 1388
	nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
					    &nvmeq->sq_dma_addr, GFP_KERNEL);
	if (!nvmeq->sq_cmds)
		return -ENOMEM;
1389 1390 1391
	return 0;
}

1392
static int nvme_alloc_queue(struct nvme_dev *dev, int qid, int depth)
M
Matthew Wilcox 已提交
1393
{
1394
	struct nvme_queue *nvmeq = &dev->queues[qid];
M
Matthew Wilcox 已提交
1395

1396 1397
	if (dev->ctrl.queue_count > qid)
		return 0;
M
Matthew Wilcox 已提交
1398

1399
	nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
J
Joe Perches 已提交
1400
					  &nvmeq->cq_dma_addr, GFP_KERNEL);
M
Matthew Wilcox 已提交
1401 1402 1403
	if (!nvmeq->cqes)
		goto free_nvmeq;

1404
	if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
M
Matthew Wilcox 已提交
1405 1406
		goto free_cqdma;

1407
	nvmeq->q_dmadev = dev->dev;
M
Matthew Wilcox 已提交
1408
	nvmeq->dev = dev;
M
Matthew Wilcox 已提交
1409 1410
	spin_lock_init(&nvmeq->q_lock);
	nvmeq->cq_head = 0;
M
Matthew Wilcox 已提交
1411
	nvmeq->cq_phase = 1;
1412
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
M
Matthew Wilcox 已提交
1413
	nvmeq->q_depth = depth;
K
Keith Busch 已提交
1414
	nvmeq->qid = qid;
1415
	nvmeq->cq_vector = -1;
1416
	dev->ctrl.queue_count++;
1417

1418
	return 0;
M
Matthew Wilcox 已提交
1419 1420

 free_cqdma:
1421
	dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
M
Matthew Wilcox 已提交
1422 1423
							nvmeq->cq_dma_addr);
 free_nvmeq:
1424
	return -ENOMEM;
M
Matthew Wilcox 已提交
1425 1426
}

1427
static int queue_request_irq(struct nvme_queue *nvmeq)
1428
{
1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
	struct pci_dev *pdev = to_pci_dev(nvmeq->dev->dev);
	int nr = nvmeq->dev->ctrl.instance;

	if (use_threaded_interrupts) {
		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq_check,
				nvme_irq, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
	} else {
		return pci_request_irq(pdev, nvmeq->cq_vector, nvme_irq,
				NULL, nvmeq, "nvme%dq%d", nr, nvmeq->qid);
	}
1439 1440
}

1441
static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
M
Matthew Wilcox 已提交
1442
{
1443
	struct nvme_dev *dev = nvmeq->dev;
M
Matthew Wilcox 已提交
1444

1445
	spin_lock_irq(&nvmeq->q_lock);
1446 1447 1448
	nvmeq->sq_tail = 0;
	nvmeq->cq_head = 0;
	nvmeq->cq_phase = 1;
1449
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1450
	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
1451
	nvme_dbbuf_init(dev, nvmeq, qid);
K
Keith Busch 已提交
1452
	dev->online_queues++;
1453
	spin_unlock_irq(&nvmeq->q_lock);
1454 1455 1456 1457 1458 1459
}

static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
{
	struct nvme_dev *dev = nvmeq->dev;
	int result;
1460

1461 1462 1463 1464 1465 1466 1467
	if (dev->cmb && use_cmb_sqes && (dev->cmbsz & NVME_CMBSZ_SQS)) {
		unsigned offset = (qid - 1) * roundup(SQ_SIZE(nvmeq->q_depth),
						      dev->ctrl.page_size);
		nvmeq->sq_dma_addr = dev->cmb_bus_addr + offset;
		nvmeq->sq_cmds_io = dev->cmb + offset;
	}

1468 1469 1470 1471 1472
	/*
	 * A queue's vector matches the queue identifier unless the controller
	 * has only one vector available.
	 */
	nvmeq->cq_vector = dev->num_vecs == 1 ? 0 : qid;
M
Matthew Wilcox 已提交
1473 1474
	result = adapter_alloc_cq(dev, qid, nvmeq);
	if (result < 0)
1475
		goto release_vector;
M
Matthew Wilcox 已提交
1476 1477 1478 1479 1480

	result = adapter_alloc_sq(dev, qid, nvmeq);
	if (result < 0)
		goto release_cq;

1481
	nvme_init_queue(nvmeq, qid);
1482
	result = queue_request_irq(nvmeq);
M
Matthew Wilcox 已提交
1483 1484 1485
	if (result < 0)
		goto release_sq;

1486
	return result;
M
Matthew Wilcox 已提交
1487 1488

 release_sq:
1489
	dev->online_queues--;
M
Matthew Wilcox 已提交
1490 1491 1492
	adapter_delete_sq(dev, qid);
 release_cq:
	adapter_delete_cq(dev, qid);
1493 1494
 release_vector:
	nvmeq->cq_vector = -1;
1495
	return result;
M
Matthew Wilcox 已提交
1496 1497
}

1498
static const struct blk_mq_ops nvme_mq_admin_ops = {
1499
	.queue_rq	= nvme_queue_rq,
1500
	.complete	= nvme_pci_complete_rq,
M
Matias Bjørling 已提交
1501
	.init_hctx	= nvme_admin_init_hctx,
1502
	.exit_hctx      = nvme_admin_exit_hctx,
1503
	.init_request	= nvme_init_request,
M
Matias Bjørling 已提交
1504 1505 1506
	.timeout	= nvme_timeout,
};

1507
static const struct blk_mq_ops nvme_mq_ops = {
M
Matias Bjørling 已提交
1508
	.queue_rq	= nvme_queue_rq,
1509
	.complete	= nvme_pci_complete_rq,
M
Matias Bjørling 已提交
1510 1511
	.init_hctx	= nvme_init_hctx,
	.init_request	= nvme_init_request,
1512
	.map_queues	= nvme_pci_map_queues,
M
Matias Bjørling 已提交
1513
	.timeout	= nvme_timeout,
J
Jens Axboe 已提交
1514
	.poll		= nvme_poll,
M
Matias Bjørling 已提交
1515 1516
};

1517 1518
static void nvme_dev_remove_admin(struct nvme_dev *dev)
{
1519
	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
1520 1521 1522 1523 1524
		/*
		 * If the controller was reset during removal, it's possible
		 * user requests may be waiting on a stopped queue. Start the
		 * queue to flush these to completion.
		 */
1525
		blk_mq_unquiesce_queue(dev->ctrl.admin_q);
1526
		blk_cleanup_queue(dev->ctrl.admin_q);
1527 1528 1529 1530
		blk_mq_free_tag_set(&dev->admin_tagset);
	}
}

M
Matias Bjørling 已提交
1531 1532
static int nvme_alloc_admin_tags(struct nvme_dev *dev)
{
1533
	if (!dev->ctrl.admin_q) {
M
Matias Bjørling 已提交
1534 1535
		dev->admin_tagset.ops = &nvme_mq_admin_ops;
		dev->admin_tagset.nr_hw_queues = 1;
K
Keith Busch 已提交
1536

K
Keith Busch 已提交
1537
		dev->admin_tagset.queue_depth = NVME_AQ_MQ_TAG_DEPTH;
M
Matias Bjørling 已提交
1538
		dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1539
		dev->admin_tagset.numa_node = dev_to_node(dev->dev);
C
Chaitanya Kulkarni 已提交
1540
		dev->admin_tagset.cmd_size = nvme_pci_cmd_size(dev, false);
1541
		dev->admin_tagset.flags = BLK_MQ_F_NO_SCHED;
M
Matias Bjørling 已提交
1542 1543 1544 1545
		dev->admin_tagset.driver_data = dev;

		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
			return -ENOMEM;
1546
		dev->ctrl.admin_tagset = &dev->admin_tagset;
M
Matias Bjørling 已提交
1547

1548 1549
		dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
		if (IS_ERR(dev->ctrl.admin_q)) {
M
Matias Bjørling 已提交
1550 1551 1552
			blk_mq_free_tag_set(&dev->admin_tagset);
			return -ENOMEM;
		}
1553
		if (!blk_get_queue(dev->ctrl.admin_q)) {
1554
			nvme_dev_remove_admin(dev);
1555
			dev->ctrl.admin_q = NULL;
1556 1557
			return -ENODEV;
		}
K
Keith Busch 已提交
1558
	} else
1559
		blk_mq_unquiesce_queue(dev->ctrl.admin_q);
M
Matias Bjørling 已提交
1560 1561 1562 1563

	return 0;
}

1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586 1587 1588 1589
static unsigned long db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
{
	return NVME_REG_DBS + ((nr_io_queues + 1) * 8 * dev->db_stride);
}

static int nvme_remap_bar(struct nvme_dev *dev, unsigned long size)
{
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	if (size <= dev->bar_mapped_size)
		return 0;
	if (size > pci_resource_len(pdev, 0))
		return -ENOMEM;
	if (dev->bar)
		iounmap(dev->bar);
	dev->bar = ioremap(pci_resource_start(pdev, 0), size);
	if (!dev->bar) {
		dev->bar_mapped_size = 0;
		return -ENOMEM;
	}
	dev->bar_mapped_size = size;
	dev->dbs = dev->bar + NVME_REG_DBS;

	return 0;
}

1590
static int nvme_pci_configure_admin_queue(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1591
{
1592
	int result;
M
Matthew Wilcox 已提交
1593 1594 1595
	u32 aqa;
	struct nvme_queue *nvmeq;

1596 1597 1598 1599
	result = nvme_remap_bar(dev, db_bar_size(dev, 0));
	if (result < 0)
		return result;

1600
	dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1, 0) ?
1601
				NVME_CAP_NSSRC(dev->ctrl.cap) : 0;
1602

1603 1604 1605
	if (dev->subsystem &&
	    (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
		writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1606

1607
	result = nvme_disable_ctrl(&dev->ctrl, dev->ctrl.cap);
1608 1609
	if (result < 0)
		return result;
M
Matthew Wilcox 已提交
1610

1611
	result = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1612 1613
	if (result)
		return result;
M
Matthew Wilcox 已提交
1614

1615
	nvmeq = &dev->queues[0];
M
Matthew Wilcox 已提交
1616 1617 1618
	aqa = nvmeq->q_depth - 1;
	aqa |= aqa << 16;

1619 1620 1621
	writel(aqa, dev->bar + NVME_REG_AQA);
	lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
	lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
M
Matthew Wilcox 已提交
1622

1623
	result = nvme_enable_ctrl(&dev->ctrl, dev->ctrl.cap);
1624
	if (result)
K
Keith Busch 已提交
1625
		return result;
M
Matias Bjørling 已提交
1626

K
Keith Busch 已提交
1627
	nvmeq->cq_vector = 0;
1628
	nvme_init_queue(nvmeq, 0);
1629
	result = queue_request_irq(nvmeq);
1630 1631
	if (result) {
		nvmeq->cq_vector = -1;
K
Keith Busch 已提交
1632
		return result;
1633
	}
1634

M
Matthew Wilcox 已提交
1635 1636 1637
	return result;
}

1638
static int nvme_create_io_queues(struct nvme_dev *dev)
K
Keith Busch 已提交
1639
{
1640
	unsigned i, max;
1641
	int ret = 0;
K
Keith Busch 已提交
1642

1643
	for (i = dev->ctrl.queue_count; i <= dev->max_qid; i++) {
1644
		if (nvme_alloc_queue(dev, i, dev->q_depth)) {
1645
			ret = -ENOMEM;
K
Keith Busch 已提交
1646
			break;
1647 1648
		}
	}
K
Keith Busch 已提交
1649

1650
	max = min(dev->max_qid, dev->ctrl.queue_count - 1);
1651
	for (i = dev->online_queues; i <= max; i++) {
1652
		ret = nvme_create_queue(&dev->queues[i], i);
K
Keith Busch 已提交
1653
		if (ret)
K
Keith Busch 已提交
1654
			break;
M
Matthew Wilcox 已提交
1655
	}
1656 1657 1658

	/*
	 * Ignore failing Create SQ/CQ commands, we can continue with less
1659 1660
	 * than the desired amount of queues, and even a controller without
	 * I/O queues can still be used to issue admin commands.  This might
1661 1662 1663
	 * be useful to upgrade a buggy firmware for example.
	 */
	return ret >= 0 ? 0 : ret;
M
Matthew Wilcox 已提交
1664 1665
}

1666 1667 1668 1669 1670 1671
static ssize_t nvme_cmb_show(struct device *dev,
			     struct device_attribute *attr,
			     char *buf)
{
	struct nvme_dev *ndev = to_nvme_dev(dev_get_drvdata(dev));

1672
	return scnprintf(buf, PAGE_SIZE, "cmbloc : x%08x\ncmbsz  : x%08x\n",
1673 1674 1675 1676
		       ndev->cmbloc, ndev->cmbsz);
}
static DEVICE_ATTR(cmb, S_IRUGO, nvme_cmb_show, NULL);

1677
static u64 nvme_cmb_size_unit(struct nvme_dev *dev)
1678
{
1679 1680 1681 1682 1683 1684 1685 1686 1687 1688
	u8 szu = (dev->cmbsz >> NVME_CMBSZ_SZU_SHIFT) & NVME_CMBSZ_SZU_MASK;

	return 1ULL << (12 + 4 * szu);
}

static u32 nvme_cmb_size(struct nvme_dev *dev)
{
	return (dev->cmbsz >> NVME_CMBSZ_SZ_SHIFT) & NVME_CMBSZ_SZ_MASK;
}

1689
static void nvme_map_cmb(struct nvme_dev *dev)
1690
{
1691
	u64 size, offset;
1692 1693
	resource_size_t bar_size;
	struct pci_dev *pdev = to_pci_dev(dev->dev);
1694
	int bar;
1695

1696
	dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1697 1698
	if (!dev->cmbsz)
		return;
1699
	dev->cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1700

1701
	if (!use_cmb_sqes)
1702
		return;
1703

1704 1705
	size = nvme_cmb_size_unit(dev) * nvme_cmb_size(dev);
	offset = nvme_cmb_size_unit(dev) * NVME_CMB_OFST(dev->cmbloc);
1706 1707
	bar = NVME_CMB_BIR(dev->cmbloc);
	bar_size = pci_resource_len(pdev, bar);
1708 1709

	if (offset > bar_size)
1710
		return;
1711 1712 1713 1714 1715 1716 1717 1718 1719

	/*
	 * Controllers may support a CMB size larger than their BAR,
	 * for example, due to being behind a bridge. Reduce the CMB to
	 * the reported size of the BAR
	 */
	if (size > bar_size - offset)
		size = bar_size - offset;

1720 1721 1722
	dev->cmb = ioremap_wc(pci_resource_start(pdev, bar) + offset, size);
	if (!dev->cmb)
		return;
1723
	dev->cmb_bus_addr = pci_bus_address(pdev, bar) + offset;
1724
	dev->cmb_size = size;
1725 1726 1727 1728 1729

	if (sysfs_add_file_to_group(&dev->ctrl.device->kobj,
				    &dev_attr_cmb.attr, NULL))
		dev_warn(dev->ctrl.device,
			 "failed to add sysfs attribute for CMB\n");
1730 1731 1732 1733 1734 1735 1736
}

static inline void nvme_release_cmb(struct nvme_dev *dev)
{
	if (dev->cmb) {
		iounmap(dev->cmb);
		dev->cmb = NULL;
1737 1738 1739
		sysfs_remove_file_from_group(&dev->ctrl.device->kobj,
					     &dev_attr_cmb.attr, NULL);
		dev->cmbsz = 0;
1740 1741 1742
	}
}

1743 1744
static int nvme_set_host_mem(struct nvme_dev *dev, u32 bits)
{
1745
	u64 dma_addr = dev->host_mem_descs_dma;
1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780 1781
	struct nvme_command c;
	int ret;

	memset(&c, 0, sizeof(c));
	c.features.opcode	= nvme_admin_set_features;
	c.features.fid		= cpu_to_le32(NVME_FEAT_HOST_MEM_BUF);
	c.features.dword11	= cpu_to_le32(bits);
	c.features.dword12	= cpu_to_le32(dev->host_mem_size >>
					      ilog2(dev->ctrl.page_size));
	c.features.dword13	= cpu_to_le32(lower_32_bits(dma_addr));
	c.features.dword14	= cpu_to_le32(upper_32_bits(dma_addr));
	c.features.dword15	= cpu_to_le32(dev->nr_host_mem_descs);

	ret = nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
	if (ret) {
		dev_warn(dev->ctrl.device,
			 "failed to set host mem (err %d, flags %#x).\n",
			 ret, bits);
	}
	return ret;
}

static void nvme_free_host_mem(struct nvme_dev *dev)
{
	int i;

	for (i = 0; i < dev->nr_host_mem_descs; i++) {
		struct nvme_host_mem_buf_desc *desc = &dev->host_mem_descs[i];
		size_t size = le32_to_cpu(desc->size) * dev->ctrl.page_size;

		dma_free_coherent(dev->dev, size, dev->host_mem_desc_bufs[i],
				le64_to_cpu(desc->addr));
	}

	kfree(dev->host_mem_desc_bufs);
	dev->host_mem_desc_bufs = NULL;
1782 1783 1784
	dma_free_coherent(dev->dev,
			dev->nr_host_mem_descs * sizeof(*dev->host_mem_descs),
			dev->host_mem_descs, dev->host_mem_descs_dma);
1785
	dev->host_mem_descs = NULL;
1786
	dev->nr_host_mem_descs = 0;
1787 1788
}

1789 1790
static int __nvme_alloc_host_mem(struct nvme_dev *dev, u64 preferred,
		u32 chunk_size)
K
Keith Busch 已提交
1791
{
1792
	struct nvme_host_mem_buf_desc *descs;
1793
	u32 max_entries, len;
1794
	dma_addr_t descs_dma;
1795
	int i = 0;
1796
	void **bufs;
1797
	u64 size, tmp;
1798 1799 1800 1801

	tmp = (preferred + chunk_size - 1);
	do_div(tmp, chunk_size);
	max_entries = tmp;
1802 1803 1804 1805

	if (dev->ctrl.hmmaxd && dev->ctrl.hmmaxd < max_entries)
		max_entries = dev->ctrl.hmmaxd;

1806 1807
	descs = dma_zalloc_coherent(dev->dev, max_entries * sizeof(*descs),
			&descs_dma, GFP_KERNEL);
1808 1809 1810 1811 1812 1813 1814
	if (!descs)
		goto out;

	bufs = kcalloc(max_entries, sizeof(*bufs), GFP_KERNEL);
	if (!bufs)
		goto out_free_descs;

1815
	for (size = 0; size < preferred && i < max_entries; size += len) {
1816 1817
		dma_addr_t dma_addr;

1818
		len = min_t(u64, chunk_size, preferred - size);
1819 1820 1821 1822 1823 1824 1825 1826 1827 1828
		bufs[i] = dma_alloc_attrs(dev->dev, len, &dma_addr, GFP_KERNEL,
				DMA_ATTR_NO_KERNEL_MAPPING | DMA_ATTR_NO_WARN);
		if (!bufs[i])
			break;

		descs[i].addr = cpu_to_le64(dma_addr);
		descs[i].size = cpu_to_le32(len / dev->ctrl.page_size);
		i++;
	}

1829
	if (!size)
1830 1831 1832 1833 1834
		goto out_free_bufs;

	dev->nr_host_mem_descs = i;
	dev->host_mem_size = size;
	dev->host_mem_descs = descs;
1835
	dev->host_mem_descs_dma = descs_dma;
1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848
	dev->host_mem_desc_bufs = bufs;
	return 0;

out_free_bufs:
	while (--i >= 0) {
		size_t size = le32_to_cpu(descs[i].size) * dev->ctrl.page_size;

		dma_free_coherent(dev->dev, size, bufs[i],
				le64_to_cpu(descs[i].addr));
	}

	kfree(bufs);
out_free_descs:
1849 1850
	dma_free_coherent(dev->dev, max_entries * sizeof(*descs), descs,
			descs_dma);
1851 1852 1853 1854 1855
out:
	dev->host_mem_descs = NULL;
	return -ENOMEM;
}

1856 1857 1858 1859 1860
static int nvme_alloc_host_mem(struct nvme_dev *dev, u64 min, u64 preferred)
{
	u32 chunk_size;

	/* start big and work our way down */
1861
	for (chunk_size = min_t(u64, preferred, PAGE_SIZE * MAX_ORDER_NR_PAGES);
1862
	     chunk_size >= max_t(u32, dev->ctrl.hmminds * 4096, PAGE_SIZE * 2);
1863 1864 1865 1866 1867 1868 1869 1870 1871 1872 1873
	     chunk_size /= 2) {
		if (!__nvme_alloc_host_mem(dev, preferred, chunk_size)) {
			if (!min || dev->host_mem_size >= min)
				return 0;
			nvme_free_host_mem(dev);
		}
	}

	return -ENOMEM;
}

1874
static int nvme_setup_host_mem(struct nvme_dev *dev)
1875 1876 1877 1878 1879
{
	u64 max = (u64)max_host_mem_size_mb * SZ_1M;
	u64 preferred = (u64)dev->ctrl.hmpre * 4096;
	u64 min = (u64)dev->ctrl.hmmin * 4096;
	u32 enable_bits = NVME_HOST_MEM_ENABLE;
1880
	int ret;
1881 1882 1883 1884 1885 1886 1887

	preferred = min(preferred, max);
	if (min > max) {
		dev_warn(dev->ctrl.device,
			"min host memory (%lld MiB) above limit (%d MiB).\n",
			min >> ilog2(SZ_1M), max_host_mem_size_mb);
		nvme_free_host_mem(dev);
1888
		return 0;
1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901
	}

	/*
	 * If we already have a buffer allocated check if we can reuse it.
	 */
	if (dev->host_mem_descs) {
		if (dev->host_mem_size >= min)
			enable_bits |= NVME_HOST_MEM_RETURN;
		else
			nvme_free_host_mem(dev);
	}

	if (!dev->host_mem_descs) {
1902 1903 1904
		if (nvme_alloc_host_mem(dev, min, preferred)) {
			dev_warn(dev->ctrl.device,
				"failed to allocate host memory buffer.\n");
1905
			return 0; /* controller must work without HMB */
1906 1907 1908 1909 1910
		}

		dev_info(dev->ctrl.device,
			"allocated %lld MiB host memory buffer.\n",
			dev->host_mem_size >> ilog2(SZ_1M));
1911 1912
	}

1913 1914
	ret = nvme_set_host_mem(dev, enable_bits);
	if (ret)
1915
		nvme_free_host_mem(dev);
1916
	return ret;
K
Keith Busch 已提交
1917 1918
}

1919
static int nvme_setup_io_queues(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1920
{
1921
	struct nvme_queue *adminq = &dev->queues[0];
1922
	struct pci_dev *pdev = to_pci_dev(dev->dev);
1923 1924
	int result, nr_io_queues;
	unsigned long size;
M
Matthew Wilcox 已提交
1925

1926 1927 1928 1929
	struct irq_affinity affd = {
		.pre_vectors = 1
	};

1930
	nr_io_queues = num_possible_cpus();
C
Christoph Hellwig 已提交
1931 1932
	result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
	if (result < 0)
M
Matthew Wilcox 已提交
1933
		return result;
C
Christoph Hellwig 已提交
1934

1935
	if (nr_io_queues == 0)
1936
		return 0;
M
Matthew Wilcox 已提交
1937

1938
	if (dev->cmb && (dev->cmbsz & NVME_CMBSZ_SQS)) {
1939 1940 1941 1942 1943 1944 1945 1946
		result = nvme_cmb_qdepth(dev, nr_io_queues,
				sizeof(struct nvme_command));
		if (result > 0)
			dev->q_depth = result;
		else
			nvme_release_cmb(dev);
	}

1947 1948 1949 1950 1951 1952 1953 1954 1955
	do {
		size = db_bar_size(dev, nr_io_queues);
		result = nvme_remap_bar(dev, size);
		if (!result)
			break;
		if (!--nr_io_queues)
			return -ENOMEM;
	} while (1);
	adminq->q_db = dev->dbs;
1956

K
Keith Busch 已提交
1957
	/* Deregister the admin queue's interrupt */
1958
	pci_free_irq(pdev, 0, adminq);
K
Keith Busch 已提交
1959

1960 1961 1962 1963
	/*
	 * If we enable msix early due to not intx, disable it again before
	 * setting up the full range we need.
	 */
1964
	pci_free_irq_vectors(pdev);
1965 1966 1967
	result = pci_alloc_irq_vectors_affinity(pdev, 1, nr_io_queues + 1,
			PCI_IRQ_ALL_TYPES | PCI_IRQ_AFFINITY, &affd);
	if (result <= 0)
1968
		return -EIO;
1969 1970
	dev->num_vecs = result;
	dev->max_qid = max(result - 1, 1);
R
Ramachandra Rao Gajula 已提交
1971

1972 1973 1974 1975 1976 1977 1978
	/*
	 * Should investigate if there's a performance win from allocating
	 * more queues than interrupt vectors; it might allow the submission
	 * path to scale better, even if the receive path is limited by the
	 * number of interrupts.
	 */

1979
	result = queue_request_irq(adminq);
1980 1981
	if (result) {
		adminq->cq_vector = -1;
K
Keith Busch 已提交
1982
		return result;
1983
	}
1984
	return nvme_create_io_queues(dev);
M
Matthew Wilcox 已提交
1985 1986
}

1987
static void nvme_del_queue_end(struct request *req, blk_status_t error)
K
Keith Busch 已提交
1988
{
K
Keith Busch 已提交
1989
	struct nvme_queue *nvmeq = req->end_io_data;
1990

K
Keith Busch 已提交
1991 1992
	blk_mq_free_request(req);
	complete(&nvmeq->dev->ioq_wait);
K
Keith Busch 已提交
1993 1994
}

1995
static void nvme_del_cq_end(struct request *req, blk_status_t error)
K
Keith Busch 已提交
1996
{
K
Keith Busch 已提交
1997
	struct nvme_queue *nvmeq = req->end_io_data;
1998
	u16 start, end;
K
Keith Busch 已提交
1999

K
Keith Busch 已提交
2000 2001 2002
	if (!error) {
		unsigned long flags;

2003 2004 2005 2006 2007 2008 2009
		/*
		 * We might be called with the AQ q_lock held
		 * and the I/O queue q_lock should always
		 * nest inside the AQ one.
		 */
		spin_lock_irqsave_nested(&nvmeq->q_lock, flags,
					SINGLE_DEPTH_NESTING);
2010
		nvme_process_cq(nvmeq, &start, &end, -1);
K
Keith Busch 已提交
2011
		spin_unlock_irqrestore(&nvmeq->q_lock, flags);
2012 2013

		nvme_complete_cqes(nvmeq, start, end);
K
Keith Busch 已提交
2014
	}
K
Keith Busch 已提交
2015 2016

	nvme_del_queue_end(req, error);
K
Keith Busch 已提交
2017 2018
}

K
Keith Busch 已提交
2019
static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
2020
{
K
Keith Busch 已提交
2021 2022 2023
	struct request_queue *q = nvmeq->dev->ctrl.admin_q;
	struct request *req;
	struct nvme_command cmd;
2024

K
Keith Busch 已提交
2025 2026 2027
	memset(&cmd, 0, sizeof(cmd));
	cmd.delete_queue.opcode = opcode;
	cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
2028

2029
	req = nvme_alloc_request(q, &cmd, BLK_MQ_REQ_NOWAIT, NVME_QID_ANY);
K
Keith Busch 已提交
2030 2031
	if (IS_ERR(req))
		return PTR_ERR(req);
2032

K
Keith Busch 已提交
2033 2034 2035 2036 2037 2038 2039
	req->timeout = ADMIN_TIMEOUT;
	req->end_io_data = nvmeq;

	blk_execute_rq_nowait(q, NULL, req, false,
			opcode == nvme_admin_delete_cq ?
				nvme_del_cq_end : nvme_del_queue_end);
	return 0;
2040 2041
}

2042
static void nvme_disable_io_queues(struct nvme_dev *dev)
K
Keith Busch 已提交
2043
{
2044
	int pass, queues = dev->online_queues - 1;
K
Keith Busch 已提交
2045 2046
	unsigned long timeout;
	u8 opcode = nvme_admin_delete_sq;
K
Keith Busch 已提交
2047

K
Keith Busch 已提交
2048
	for (pass = 0; pass < 2; pass++) {
K
Keith Busch 已提交
2049
		int sent = 0, i = queues;
K
Keith Busch 已提交
2050 2051 2052 2053

		reinit_completion(&dev->ioq_wait);
 retry:
		timeout = ADMIN_TIMEOUT;
2054
		for (; i > 0; i--, sent++)
2055
			if (nvme_delete_queue(&dev->queues[i], opcode))
K
Keith Busch 已提交
2056
				break;
2057

K
Keith Busch 已提交
2058 2059 2060 2061 2062 2063 2064 2065 2066
		while (sent--) {
			timeout = wait_for_completion_io_timeout(&dev->ioq_wait, timeout);
			if (timeout == 0)
				return;
			if (i)
				goto retry;
		}
		opcode = nvme_admin_delete_cq;
	}
K
Keith Busch 已提交
2067 2068
}

2069
/*
2070
 * return error value only when tagset allocation failed
2071
 */
2072
static int nvme_dev_add(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2073
{
2074 2075
	int ret;

2076
	if (!dev->ctrl.tagset) {
2077 2078 2079 2080 2081
		dev->tagset.ops = &nvme_mq_ops;
		dev->tagset.nr_hw_queues = dev->online_queues - 1;
		dev->tagset.timeout = NVME_IO_TIMEOUT;
		dev->tagset.numa_node = dev_to_node(dev->dev);
		dev->tagset.queue_depth =
M
Matias Bjørling 已提交
2082
				min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
C
Chaitanya Kulkarni 已提交
2083 2084 2085 2086 2087
		dev->tagset.cmd_size = nvme_pci_cmd_size(dev, false);
		if ((dev->ctrl.sgls & ((1 << 0) | (1 << 1))) && sgl_threshold) {
			dev->tagset.cmd_size = max(dev->tagset.cmd_size,
					nvme_pci_cmd_size(dev, true));
		}
2088 2089
		dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
		dev->tagset.driver_data = dev;
M
Matthew Wilcox 已提交
2090

2091 2092 2093 2094 2095 2096
		ret = blk_mq_alloc_tag_set(&dev->tagset);
		if (ret) {
			dev_warn(dev->ctrl.device,
				"IO queues tagset allocation failed %d\n", ret);
			return ret;
		}
2097
		dev->ctrl.tagset = &dev->tagset;
2098 2099

		nvme_dbbuf_set(dev);
2100 2101 2102 2103 2104
	} else {
		blk_mq_update_nr_hw_queues(&dev->tagset, dev->online_queues - 1);

		/* Free previously allocated queues that are no longer usable */
		nvme_free_queues(dev, dev->online_queues);
2105
	}
2106

K
Keith Busch 已提交
2107
	return 0;
M
Matthew Wilcox 已提交
2108 2109
}

2110
static int nvme_pci_enable(struct nvme_dev *dev)
2111
{
2112
	int result = -ENOMEM;
2113
	struct pci_dev *pdev = to_pci_dev(dev->dev);
2114 2115 2116 2117 2118 2119

	if (pci_enable_device_mem(pdev))
		return result;

	pci_set_master(pdev);

2120 2121
	if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
	    dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
2122
		goto disable;
2123

2124
	if (readl(dev->bar + NVME_REG_CSTS) == -1) {
K
Keith Busch 已提交
2125
		result = -ENODEV;
2126
		goto disable;
K
Keith Busch 已提交
2127
	}
2128 2129

	/*
2130 2131 2132
	 * Some devices and/or platforms don't advertise or work with INTx
	 * interrupts. Pre-enable a single MSIX or MSI vec for setup. We'll
	 * adjust this later.
2133
	 */
2134 2135 2136
	result = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_ALL_TYPES);
	if (result < 0)
		return result;
2137

2138
	dev->ctrl.cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
2139

2140
	dev->q_depth = min_t(int, NVME_CAP_MQES(dev->ctrl.cap) + 1,
2141
				io_queue_depth);
2142
	dev->db_stride = 1 << NVME_CAP_STRIDE(dev->ctrl.cap);
2143
	dev->dbs = dev->bar + 4096;
2144 2145 2146 2147 2148 2149 2150

	/*
	 * Temporary fix for the Apple controller found in the MacBook8,1 and
	 * some MacBook7,1 to avoid controller resets and data loss.
	 */
	if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) {
		dev->q_depth = 2;
2151 2152
		dev_warn(dev->ctrl.device, "detected Apple NVMe controller, "
			"set queue depth=%u to work around controller resets\n",
2153
			dev->q_depth);
2154 2155
	} else if (pdev->vendor == PCI_VENDOR_ID_SAMSUNG &&
		   (pdev->device == 0xa821 || pdev->device == 0xa822) &&
2156
		   NVME_CAP_MQES(dev->ctrl.cap) == 0) {
2157 2158 2159
		dev->q_depth = 64;
		dev_err(dev->ctrl.device, "detected PM1725 NVMe controller, "
                        "set queue depth=%u\n", dev->q_depth);
2160 2161
	}

2162
	nvme_map_cmb(dev);
2163

K
Keith Busch 已提交
2164 2165
	pci_enable_pcie_error_reporting(pdev);
	pci_save_state(pdev);
2166 2167 2168 2169 2170 2171 2172 2173
	return 0;

 disable:
	pci_disable_device(pdev);
	return result;
}

static void nvme_dev_unmap(struct nvme_dev *dev)
2174 2175 2176
{
	if (dev->bar)
		iounmap(dev->bar);
2177
	pci_release_mem_regions(to_pci_dev(dev->dev));
2178 2179 2180
}

static void nvme_pci_disable(struct nvme_dev *dev)
2181
{
2182 2183
	struct pci_dev *pdev = to_pci_dev(dev->dev);

2184
	nvme_release_cmb(dev);
2185
	pci_free_irq_vectors(pdev);
2186

K
Keith Busch 已提交
2187 2188
	if (pci_is_enabled(pdev)) {
		pci_disable_pcie_error_reporting(pdev);
2189
		pci_disable_device(pdev);
K
Keith Busch 已提交
2190 2191 2192
	}
}

2193
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
M
Matthew Wilcox 已提交
2194
{
2195
	int i;
K
Keith Busch 已提交
2196 2197
	bool dead = true;
	struct pci_dev *pdev = to_pci_dev(dev->dev);
2198

2199
	mutex_lock(&dev->shutdown_lock);
K
Keith Busch 已提交
2200 2201 2202
	if (pci_is_enabled(pdev)) {
		u32 csts = readl(dev->bar + NVME_REG_CSTS);

K
Keith Busch 已提交
2203 2204
		if (dev->ctrl.state == NVME_CTRL_LIVE ||
		    dev->ctrl.state == NVME_CTRL_RESETTING)
K
Keith Busch 已提交
2205 2206 2207
			nvme_start_freeze(&dev->ctrl);
		dead = !!((csts & NVME_CSTS_CFS) || !(csts & NVME_CSTS_RDY) ||
			pdev->error_state  != pci_channel_io_normal);
2208
	}
2209

K
Keith Busch 已提交
2210 2211 2212 2213
	/*
	 * Give the controller a chance to complete all entered requests if
	 * doing a safe shutdown.
	 */
2214 2215 2216
	if (!dead) {
		if (shutdown)
			nvme_wait_freeze_timeout(&dev->ctrl, NVME_IO_TIMEOUT);
2217 2218 2219
	}

	nvme_stop_queues(&dev->ctrl);
2220

2221
	if (!dead && dev->ctrl.queue_count > 0) {
2222 2223 2224 2225 2226 2227 2228 2229
		/*
		 * If the controller is still alive tell it to stop using the
		 * host memory buffer.  In theory the shutdown / reset should
		 * make sure that it doesn't access the host memoery anymore,
		 * but I'd rather be safe than sorry..
		 */
		if (dev->host_mem_descs)
			nvme_set_host_mem(dev, 0);
2230
		nvme_disable_io_queues(dev);
2231
		nvme_disable_admin_queue(dev, shutdown);
K
Keith Busch 已提交
2232
	}
2233 2234 2235
	for (i = dev->ctrl.queue_count - 1; i >= 0; i--)
		nvme_suspend_queue(&dev->queues[i]);

2236
	nvme_pci_disable(dev);
2237

2238 2239
	blk_mq_tagset_busy_iter(&dev->tagset, nvme_cancel_request, &dev->ctrl);
	blk_mq_tagset_busy_iter(&dev->admin_tagset, nvme_cancel_request, &dev->ctrl);
K
Keith Busch 已提交
2240 2241 2242 2243 2244 2245 2246 2247

	/*
	 * The driver will not be starting up queues again if shutting down so
	 * must flush all entered requests to their failed completion to avoid
	 * deadlocking blk-mq hot-cpu notifier.
	 */
	if (shutdown)
		nvme_start_queues(&dev->ctrl);
2248
	mutex_unlock(&dev->shutdown_lock);
M
Matthew Wilcox 已提交
2249 2250
}

M
Matthew Wilcox 已提交
2251 2252
static int nvme_setup_prp_pools(struct nvme_dev *dev)
{
2253
	dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
M
Matthew Wilcox 已提交
2254 2255 2256 2257
						PAGE_SIZE, PAGE_SIZE, 0);
	if (!dev->prp_page_pool)
		return -ENOMEM;

2258
	/* Optimisation for I/Os between 4k and 128k */
2259
	dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2260 2261 2262 2263 2264
						256, 256, 0);
	if (!dev->prp_small_pool) {
		dma_pool_destroy(dev->prp_page_pool);
		return -ENOMEM;
	}
M
Matthew Wilcox 已提交
2265 2266 2267 2268 2269 2270
	return 0;
}

static void nvme_release_prp_pools(struct nvme_dev *dev)
{
	dma_pool_destroy(dev->prp_page_pool);
2271
	dma_pool_destroy(dev->prp_small_pool);
M
Matthew Wilcox 已提交
2272 2273
}

2274
static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
2275
{
2276
	struct nvme_dev *dev = to_nvme_dev(ctrl);
2277

2278
	nvme_dbbuf_dma_free(dev);
2279
	put_device(dev->dev);
2280 2281
	if (dev->tagset.tags)
		blk_mq_free_tag_set(&dev->tagset);
2282 2283
	if (dev->ctrl.admin_q)
		blk_put_queue(dev->ctrl.admin_q);
2284
	kfree(dev->queues);
2285
	free_opal_dev(dev->ctrl.opal_dev);
2286 2287 2288
	kfree(dev);
}

2289 2290
static void nvme_remove_dead_ctrl(struct nvme_dev *dev, int status)
{
2291
	dev_warn(dev->ctrl.device, "Removing after probe failure status: %d\n", status);
2292

2293
	nvme_get_ctrl(&dev->ctrl);
2294
	nvme_dev_disable(dev, false);
2295
	if (!queue_work(nvme_wq, &dev->remove_work))
2296 2297 2298
		nvme_put_ctrl(&dev->ctrl);
}

2299
static void nvme_reset_work(struct work_struct *work)
2300
{
2301 2302
	struct nvme_dev *dev =
		container_of(work, struct nvme_dev, ctrl.reset_work);
2303
	bool was_suspend = !!(dev->ctrl.ctrl_config & NVME_CC_SHN_NORMAL);
2304
	int result = -ENODEV;
2305
	enum nvme_ctrl_state new_state = NVME_CTRL_LIVE;
2306

2307
	if (WARN_ON(dev->ctrl.state != NVME_CTRL_RESETTING))
2308
		goto out;
2309

2310 2311 2312 2313
	/*
	 * If we're called to reset a live controller first shut it down before
	 * moving on.
	 */
2314
	if (dev->ctrl.ctrl_config & NVME_CC_ENABLE)
2315
		nvme_dev_disable(dev, false);
2316

2317
	/*
2318
	 * Introduce CONNECTING state from nvme-fc/rdma transports to mark the
2319 2320
	 * initializing procedure here.
	 */
2321
	if (!nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_CONNECTING)) {
2322
		dev_warn(dev->ctrl.device,
2323
			"failed to mark controller CONNECTING\n");
2324 2325 2326
		goto out;
	}

2327
	result = nvme_pci_enable(dev);
2328
	if (result)
2329
		goto out;
2330

2331
	result = nvme_pci_configure_admin_queue(dev);
2332
	if (result)
2333
		goto out;
2334

K
Keith Busch 已提交
2335 2336
	result = nvme_alloc_admin_tags(dev);
	if (result)
2337
		goto out;
2338

2339 2340
	result = nvme_init_identify(&dev->ctrl);
	if (result)
2341
		goto out;
2342

2343 2344 2345 2346 2347 2348 2349 2350 2351
	if (dev->ctrl.oacs & NVME_CTRL_OACS_SEC_SUPP) {
		if (!dev->ctrl.opal_dev)
			dev->ctrl.opal_dev =
				init_opal_dev(&dev->ctrl, &nvme_sec_submit);
		else if (was_suspend)
			opal_unlock_from_suspend(dev->ctrl.opal_dev);
	} else {
		free_opal_dev(dev->ctrl.opal_dev);
		dev->ctrl.opal_dev = NULL;
2352
	}
2353

2354 2355 2356 2357 2358 2359 2360
	if (dev->ctrl.oacs & NVME_CTRL_OACS_DBBUF_SUPP) {
		result = nvme_dbbuf_dma_alloc(dev);
		if (result)
			dev_warn(dev->dev,
				 "unable to allocate dma for dbbuf\n");
	}

2361 2362 2363 2364 2365
	if (dev->ctrl.hmpre) {
		result = nvme_setup_host_mem(dev);
		if (result < 0)
			goto out;
	}
2366

2367
	result = nvme_setup_io_queues(dev);
2368
	if (result)
2369
		goto out;
2370

2371 2372 2373 2374
	/*
	 * Keep the controller around but remove all namespaces if we don't have
	 * any working I/O queue.
	 */
2375
	if (dev->online_queues < 2) {
2376
		dev_warn(dev->ctrl.device, "IO queues not created\n");
2377
		nvme_kill_queues(&dev->ctrl);
2378
		nvme_remove_namespaces(&dev->ctrl);
2379
		new_state = NVME_CTRL_ADMIN_ONLY;
2380
	} else {
2381
		nvme_start_queues(&dev->ctrl);
K
Keith Busch 已提交
2382
		nvme_wait_freeze(&dev->ctrl);
2383 2384 2385
		/* hit this only when allocate tagset fails */
		if (nvme_dev_add(dev))
			new_state = NVME_CTRL_ADMIN_ONLY;
K
Keith Busch 已提交
2386
		nvme_unfreeze(&dev->ctrl);
2387 2388
	}

2389 2390 2391 2392 2393 2394 2395
	/*
	 * If only admin queue live, keep it to do further investigation or
	 * recovery.
	 */
	if (!nvme_change_ctrl_state(&dev->ctrl, new_state)) {
		dev_warn(dev->ctrl.device,
			"failed to mark controller state %d\n", new_state);
2396 2397
		goto out;
	}
2398

2399
	nvme_start_ctrl(&dev->ctrl);
2400
	return;
2401

2402
 out:
2403
	nvme_remove_dead_ctrl(dev, result);
2404 2405
}

2406
static void nvme_remove_dead_ctrl_work(struct work_struct *work)
K
Keith Busch 已提交
2407
{
2408
	struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
2409
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
2410

2411
	nvme_kill_queues(&dev->ctrl);
K
Keith Busch 已提交
2412
	if (pci_get_drvdata(pdev))
K
Keith Busch 已提交
2413
		device_release_driver(&pdev->dev);
2414
	nvme_put_ctrl(&dev->ctrl);
K
Keith Busch 已提交
2415 2416
}

2417
static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
T
Tejun Heo 已提交
2418
{
2419
	*val = readl(to_nvme_dev(ctrl)->bar + off);
2420
	return 0;
T
Tejun Heo 已提交
2421 2422
}

2423
static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
2424
{
2425 2426 2427
	writel(val, to_nvme_dev(ctrl)->bar + off);
	return 0;
}
2428

2429 2430 2431 2432
static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
{
	*val = readq(to_nvme_dev(ctrl)->bar + off);
	return 0;
2433 2434
}

2435 2436 2437 2438 2439 2440 2441
static int nvme_pci_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
{
	struct pci_dev *pdev = to_pci_dev(to_nvme_dev(ctrl)->dev);

	return snprintf(buf, size, "%s", dev_name(&pdev->dev));
}

2442
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
M
Ming Lin 已提交
2443
	.name			= "pcie",
2444
	.module			= THIS_MODULE,
2445
	.flags			= NVME_F_METADATA_SUPPORTED,
2446
	.reg_read32		= nvme_pci_reg_read32,
2447
	.reg_write32		= nvme_pci_reg_write32,
2448
	.reg_read64		= nvme_pci_reg_read64,
2449
	.free_ctrl		= nvme_pci_free_ctrl,
2450
	.submit_async_event	= nvme_pci_submit_async_event,
2451
	.get_address		= nvme_pci_get_address,
2452
};
2453

2454 2455 2456 2457
static int nvme_dev_map(struct nvme_dev *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev->dev);

2458
	if (pci_request_mem_regions(pdev, "nvme"))
2459 2460
		return -ENODEV;

2461
	if (nvme_remap_bar(dev, NVME_REG_DBS + 4096))
2462 2463
		goto release;

M
Max Gurtovoy 已提交
2464
	return 0;
2465
  release:
M
Max Gurtovoy 已提交
2466 2467
	pci_release_mem_regions(pdev);
	return -ENODEV;
2468 2469
}

2470
static unsigned long check_vendor_combination_bug(struct pci_dev *pdev)
2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484
{
	if (pdev->vendor == 0x144d && pdev->device == 0xa802) {
		/*
		 * Several Samsung devices seem to drop off the PCIe bus
		 * randomly when APST is on and uses the deepest sleep state.
		 * This has been observed on a Samsung "SM951 NVMe SAMSUNG
		 * 256GB", a "PM951 NVMe SAMSUNG 512GB", and a "Samsung SSD
		 * 950 PRO 256GB", but it seems to be restricted to two Dell
		 * laptops.
		 */
		if (dmi_match(DMI_SYS_VENDOR, "Dell Inc.") &&
		    (dmi_match(DMI_PRODUCT_NAME, "XPS 15 9550") ||
		     dmi_match(DMI_PRODUCT_NAME, "Precision 5510")))
			return NVME_QUIRK_NO_DEEPEST_PS;
2485 2486 2487
	} else if (pdev->vendor == 0x144d && pdev->device == 0xa804) {
		/*
		 * Samsung SSD 960 EVO drops off the PCIe bus after system
2488 2489 2490
		 * suspend on a Ryzen board, ASUS PRIME B350M-A, as well as
		 * within few minutes after bootup on a Coffee Lake board -
		 * ASUS PRIME Z370-A
2491 2492
		 */
		if (dmi_match(DMI_BOARD_VENDOR, "ASUSTeK COMPUTER INC.") &&
2493 2494
		    (dmi_match(DMI_BOARD_NAME, "PRIME B350M-A") ||
		     dmi_match(DMI_BOARD_NAME, "PRIME Z370-A")))
2495
			return NVME_QUIRK_NO_APST;
2496 2497 2498 2499 2500
	}

	return 0;
}

2501 2502 2503
static void nvme_async_probe(void *data, async_cookie_t cookie)
{
	struct nvme_dev *dev = data;
2504

2505 2506
	nvme_reset_ctrl_sync(&dev->ctrl);
	flush_work(&dev->ctrl.scan_work);
2507
	nvme_put_ctrl(&dev->ctrl);
2508 2509
}

2510
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
M
Matthew Wilcox 已提交
2511
{
M
Matias Bjørling 已提交
2512
	int node, result = -ENOMEM;
M
Matthew Wilcox 已提交
2513
	struct nvme_dev *dev;
2514
	unsigned long quirks = id->driver_data;
M
Matthew Wilcox 已提交
2515

M
Matias Bjørling 已提交
2516 2517
	node = dev_to_node(&pdev->dev);
	if (node == NUMA_NO_NODE)
2518
		set_dev_node(&pdev->dev, first_memory_node);
M
Matias Bjørling 已提交
2519 2520

	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2521 2522
	if (!dev)
		return -ENOMEM;
2523 2524 2525

	dev->queues = kcalloc_node(num_possible_cpus() + 1,
			sizeof(struct nvme_queue), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2526 2527 2528
	if (!dev->queues)
		goto free;

2529
	dev->dev = get_device(&pdev->dev);
K
Keith Busch 已提交
2530
	pci_set_drvdata(pdev, dev);
2531

2532 2533
	result = nvme_dev_map(dev);
	if (result)
2534
		goto put_pci;
2535

2536
	INIT_WORK(&dev->ctrl.reset_work, nvme_reset_work);
2537
	INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
2538
	mutex_init(&dev->shutdown_lock);
K
Keith Busch 已提交
2539
	init_completion(&dev->ioq_wait);
M
Matthew Wilcox 已提交
2540

M
Matthew Wilcox 已提交
2541 2542
	result = nvme_setup_prp_pools(dev);
	if (result)
2543
		goto unmap;
2544

2545
	quirks |= check_vendor_combination_bug(pdev);
2546

2547
	result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
2548
			quirks);
2549
	if (result)
2550
		goto release_pools;
2551

2552 2553
	dev_info(dev->ctrl.device, "pci function %s\n", dev_name(&pdev->dev));

2554
	nvme_get_ctrl(&dev->ctrl);
2555
	async_schedule(nvme_async_probe, dev);
2556

M
Matthew Wilcox 已提交
2557 2558
	return 0;

2559
 release_pools:
M
Matthew Wilcox 已提交
2560
	nvme_release_prp_pools(dev);
2561 2562
 unmap:
	nvme_dev_unmap(dev);
K
Keith Busch 已提交
2563
 put_pci:
2564
	put_device(dev->dev);
M
Matthew Wilcox 已提交
2565 2566 2567 2568 2569 2570
 free:
	kfree(dev->queues);
	kfree(dev);
	return result;
}

2571
static void nvme_reset_prepare(struct pci_dev *pdev)
2572
{
K
Keith Busch 已提交
2573
	struct nvme_dev *dev = pci_get_drvdata(pdev);
2574
	nvme_dev_disable(dev, false);
2575
}
2576

2577 2578
static void nvme_reset_done(struct pci_dev *pdev)
{
2579
	struct nvme_dev *dev = pci_get_drvdata(pdev);
S
Sagi Grimberg 已提交
2580
	nvme_reset_ctrl_sync(&dev->ctrl);
2581 2582
}

2583 2584 2585
static void nvme_shutdown(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
2586
	nvme_dev_disable(dev, true);
2587 2588
}

2589 2590 2591 2592 2593
/*
 * The driver's remove may be called on a device in a partially initialized
 * state. This function must not have any dependencies on the device state in
 * order to proceed.
 */
2594
static void nvme_remove(struct pci_dev *pdev)
M
Matthew Wilcox 已提交
2595 2596
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
K
Keith Busch 已提交
2597

2598 2599
	nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DELETING);

2600
	cancel_work_sync(&dev->ctrl.reset_work);
K
Keith Busch 已提交
2601
	pci_set_drvdata(pdev, NULL);
2602

2603
	if (!pci_device_is_present(pdev)) {
2604
		nvme_change_ctrl_state(&dev->ctrl, NVME_CTRL_DEAD);
2605 2606
		nvme_dev_disable(dev, false);
	}
2607

2608
	flush_work(&dev->ctrl.reset_work);
2609 2610
	nvme_stop_ctrl(&dev->ctrl);
	nvme_remove_namespaces(&dev->ctrl);
2611
	nvme_dev_disable(dev, true);
2612
	nvme_free_host_mem(dev);
M
Matias Bjørling 已提交
2613
	nvme_dev_remove_admin(dev);
2614
	nvme_free_queues(dev, 0);
2615
	nvme_uninit_ctrl(&dev->ctrl);
K
Keith Busch 已提交
2616
	nvme_release_prp_pools(dev);
2617
	nvme_dev_unmap(dev);
2618
	nvme_put_ctrl(&dev->ctrl);
M
Matthew Wilcox 已提交
2619 2620
}

K
Keith Busch 已提交
2621 2622 2623 2624 2625 2626 2627 2628 2629 2630 2631 2632 2633 2634 2635 2636 2637 2638
static int nvme_pci_sriov_configure(struct pci_dev *pdev, int numvfs)
{
	int ret = 0;

	if (numvfs == 0) {
		if (pci_vfs_assigned(pdev)) {
			dev_warn(&pdev->dev,
				"Cannot disable SR-IOV VFs while assigned\n");
			return -EPERM;
		}
		pci_disable_sriov(pdev);
		return 0;
	}

	ret = pci_enable_sriov(pdev, numvfs);
	return ret ? ret : numvfs;
}

2639
#ifdef CONFIG_PM_SLEEP
2640 2641 2642 2643 2644
static int nvme_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

2645
	nvme_dev_disable(ndev, true);
2646 2647 2648 2649 2650 2651 2652 2653
	return 0;
}

static int nvme_resume(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

2654
	nvme_reset_ctrl(&ndev->ctrl);
K
Keith Busch 已提交
2655
	return 0;
2656
}
2657
#endif
2658 2659

static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
M
Matthew Wilcox 已提交
2660

K
Keith Busch 已提交
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674
static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
						pci_channel_state_t state)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);

	/*
	 * A frozen channel requires a reset. When detected, this method will
	 * shutdown the controller to quiesce. The controller will be restarted
	 * after the slot reset through driver's slot_reset callback.
	 */
	switch (state) {
	case pci_channel_io_normal:
		return PCI_ERS_RESULT_CAN_RECOVER;
	case pci_channel_io_frozen:
K
Keith Busch 已提交
2675 2676
		dev_warn(dev->ctrl.device,
			"frozen state error detected, reset controller\n");
2677
		nvme_dev_disable(dev, false);
K
Keith Busch 已提交
2678 2679
		return PCI_ERS_RESULT_NEED_RESET;
	case pci_channel_io_perm_failure:
K
Keith Busch 已提交
2680 2681
		dev_warn(dev->ctrl.device,
			"failure state error detected, request disconnect\n");
K
Keith Busch 已提交
2682 2683 2684 2685 2686 2687 2688 2689 2690
		return PCI_ERS_RESULT_DISCONNECT;
	}
	return PCI_ERS_RESULT_NEED_RESET;
}

static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);

2691
	dev_info(dev->ctrl.device, "restart after slot reset\n");
K
Keith Busch 已提交
2692
	pci_restore_state(pdev);
2693 2694 2695 2696 2697 2698 2699 2700 2701
	nvme_reset_ctrl_sync(&dev->ctrl);

	switch (dev->ctrl.state) {
	case NVME_CTRL_LIVE:
	case NVME_CTRL_ADMIN_ONLY:
		return PCI_ERS_RESULT_RECOVERED;
	default:
		return PCI_ERS_RESULT_DISCONNECT;
	}
K
Keith Busch 已提交
2702 2703 2704 2705 2706 2707 2708
}

static void nvme_error_resume(struct pci_dev *pdev)
{
	pci_cleanup_aer_uncorrect_error_status(pdev);
}

2709
static const struct pci_error_handlers nvme_err_handler = {
M
Matthew Wilcox 已提交
2710 2711 2712
	.error_detected	= nvme_error_detected,
	.slot_reset	= nvme_slot_reset,
	.resume		= nvme_error_resume,
2713 2714
	.reset_prepare	= nvme_reset_prepare,
	.reset_done	= nvme_reset_done,
M
Matthew Wilcox 已提交
2715 2716
};

2717
static const struct pci_device_id nvme_id_table[] = {
2718
	{ PCI_VDEVICE(INTEL, 0x0953),
2719
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
2720
				NVME_QUIRK_DEALLOCATE_ZEROES, },
2721 2722
	{ PCI_VDEVICE(INTEL, 0x0a53),
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
2723
				NVME_QUIRK_DEALLOCATE_ZEROES, },
2724 2725
	{ PCI_VDEVICE(INTEL, 0x0a54),
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
2726
				NVME_QUIRK_DEALLOCATE_ZEROES, },
2727 2728 2729
	{ PCI_VDEVICE(INTEL, 0x0a55),
		.driver_data = NVME_QUIRK_STRIPE_SIZE |
				NVME_QUIRK_DEALLOCATE_ZEROES, },
2730 2731
	{ PCI_VDEVICE(INTEL, 0xf1a5),	/* Intel 600P/P3100 */
		.driver_data = NVME_QUIRK_NO_DEEPEST_PS },
2732 2733
	{ PCI_VDEVICE(INTEL, 0x5845),	/* Qemu emulated controller */
		.driver_data = NVME_QUIRK_IDENTIFY_CNS, },
2734 2735
	{ PCI_DEVICE(0x1bb1, 0x0100),   /* Seagate Nytro Flash Storage */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2736 2737
	{ PCI_DEVICE(0x1c58, 0x0003),	/* HGST adapter */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2738 2739
	{ PCI_DEVICE(0x1c58, 0x0023),	/* WDC SN200 adapter */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2740 2741
	{ PCI_DEVICE(0x1c5f, 0x0540),	/* Memblaze Pblaze4 adapter */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
2742 2743 2744 2745
	{ PCI_DEVICE(0x144d, 0xa821),   /* Samsung PM1725 */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
	{ PCI_DEVICE(0x144d, 0xa822),   /* Samsung PM1725a */
		.driver_data = NVME_QUIRK_DELAY_BEFORE_CHK_RDY, },
C
Christoph Hellwig 已提交
2746 2747 2748 2749
	{ PCI_DEVICE(0x1d1d, 0x1f1f),	/* LighNVM qemu device */
		.driver_data = NVME_QUIRK_LIGHTNVM, },
	{ PCI_DEVICE(0x1d1d, 0x2807),	/* CNEX WL */
		.driver_data = NVME_QUIRK_LIGHTNVM, },
W
Wei Xu 已提交
2750 2751
	{ PCI_DEVICE(0x1d1d, 0x2601),	/* CNEX Granby */
		.driver_data = NVME_QUIRK_LIGHTNVM, },
M
Matthew Wilcox 已提交
2752
	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2753
	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
2754
	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2003) },
M
Matthew Wilcox 已提交
2755 2756 2757 2758 2759 2760 2761 2762
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, nvme_id_table);

static struct pci_driver nvme_driver = {
	.name		= "nvme",
	.id_table	= nvme_id_table,
	.probe		= nvme_probe,
2763
	.remove		= nvme_remove,
2764
	.shutdown	= nvme_shutdown,
2765 2766 2767
	.driver		= {
		.pm	= &nvme_dev_pm_ops,
	},
K
Keith Busch 已提交
2768
	.sriov_configure = nvme_pci_sriov_configure,
M
Matthew Wilcox 已提交
2769 2770 2771 2772 2773
	.err_handler	= &nvme_err_handler,
};

static int __init nvme_init(void)
{
2774
	return pci_register_driver(&nvme_driver);
M
Matthew Wilcox 已提交
2775 2776 2777 2778 2779
}

static void __exit nvme_exit(void)
{
	pci_unregister_driver(&nvme_driver);
2780
	flush_workqueue(nvme_wq);
2781
	_nvme_check_size();
M
Matthew Wilcox 已提交
2782 2783 2784 2785
}

MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
MODULE_LICENSE("GPL");
2786
MODULE_VERSION("1.0");
M
Matthew Wilcox 已提交
2787 2788
module_init(nvme_init);
module_exit(nvme_exit);