pci.c 55.7 KB
Newer Older
M
Matthew Wilcox 已提交
1 2
/*
 * NVM Express device driver
3
 * Copyright (c) 2011-2014, Intel Corporation.
M
Matthew Wilcox 已提交
4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

K
Keith Busch 已提交
15
#include <linux/aer.h>
16
#include <linux/bitops.h>
M
Matthew Wilcox 已提交
17
#include <linux/blkdev.h>
M
Matias Bjørling 已提交
18
#include <linux/blk-mq.h>
K
Keith Busch 已提交
19
#include <linux/cpu.h>
20
#include <linux/delay.h>
M
Matthew Wilcox 已提交
21 22 23
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/genhd.h>
K
Keith Busch 已提交
24
#include <linux/hdreg.h>
25
#include <linux/idr.h>
M
Matthew Wilcox 已提交
26 27 28 29
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kdev_t.h>
30
#include <linux/kthread.h>
M
Matthew Wilcox 已提交
31 32 33 34
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
35
#include <linux/mutex.h>
M
Matthew Wilcox 已提交
36
#include <linux/pci.h>
37
#include <linux/poison.h>
38
#include <linux/ptrace.h>
M
Matthew Wilcox 已提交
39 40
#include <linux/sched.h>
#include <linux/slab.h>
K
Keith Busch 已提交
41
#include <linux/t10-pi.h>
M
Matthew Wilcox 已提交
42
#include <linux/types.h>
43
#include <linux/io-64-nonatomic-lo-hi.h>
K
Keith Busch 已提交
44
#include <asm/unaligned.h>
45

46 47
#include "nvme.h"

48
#define NVME_Q_DEPTH		1024
J
Jens Axboe 已提交
49
#define NVME_AQ_DEPTH		256
M
Matthew Wilcox 已提交
50 51
#define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))
#define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))
52 53 54 55 56 57 58
		
/*
 * We handle AEN commands ourselves and don't even let the
 * block layer know about them.
 */
#define NVME_NR_AEN_COMMANDS	1
#define NVME_AQ_BLKMQ_DEPTH	(NVME_AQ_DEPTH - NVME_NR_AEN_COMMANDS)
59

60
unsigned char admin_timeout = 60;
61 62
module_param(admin_timeout, byte, 0644);
MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
M
Matthew Wilcox 已提交
63

64 65
unsigned char nvme_io_timeout = 30;
module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
66
MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
M
Matthew Wilcox 已提交
67

68
unsigned char shutdown_timeout = 5;
69 70 71
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");

72 73 74
static int use_threaded_interrupts;
module_param(use_threaded_interrupts, int, 0);

75 76 77 78
static bool use_cmb_sqes = true;
module_param(use_cmb_sqes, bool, 0644);
MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");

79 80
static LIST_HEAD(dev_list);
static struct task_struct *nvme_thread;
K
Keith Busch 已提交
81
static struct workqueue_struct *nvme_workq;
82
static wait_queue_head_t nvme_kthread_wait;
83

84 85
struct nvme_dev;
struct nvme_queue;
86

87
static int nvme_reset(struct nvme_dev *dev);
J
Jens Axboe 已提交
88
static void nvme_process_cq(struct nvme_queue *nvmeq);
89
static void nvme_remove_dead_ctrl(struct nvme_dev *dev);
90
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown);
91

92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112
/*
 * Represents an NVM Express device.  Each nvme_dev is a PCI function.
 */
struct nvme_dev {
	struct list_head node;
	struct nvme_queue **queues;
	struct blk_mq_tag_set tagset;
	struct blk_mq_tag_set admin_tagset;
	u32 __iomem *dbs;
	struct device *dev;
	struct dma_pool *prp_page_pool;
	struct dma_pool *prp_small_pool;
	unsigned queue_count;
	unsigned online_queues;
	unsigned max_qid;
	int q_depth;
	u32 db_stride;
	struct msix_entry *entry;
	void __iomem *bar;
	struct work_struct reset_work;
	struct work_struct scan_work;
113
	struct work_struct remove_work;
114
	struct mutex shutdown_lock;
115 116 117 118 119
	bool subsystem;
	void __iomem *cmb;
	dma_addr_t cmb_dma_addr;
	u64 cmb_size;
	u32 cmbsz;
120
	unsigned long flags;
K
Keith Busch 已提交
121

122
#define NVME_CTRL_RESETTING    0
123 124

	struct nvme_ctrl ctrl;
K
Keith Busch 已提交
125
	struct completion ioq_wait;
K
Keith Busch 已提交
126
};
127

128 129 130 131 132
static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
{
	return container_of(ctrl, struct nvme_dev, ctrl);
}

M
Matthew Wilcox 已提交
133 134 135 136 137 138
/*
 * An NVM Express queue.  Each device has at least two (one for admin
 * commands and one for I/O commands).
 */
struct nvme_queue {
	struct device *q_dmadev;
M
Matthew Wilcox 已提交
139
	struct nvme_dev *dev;
140
	char irqname[24];	/* nvme4294967295-65535\0 */
M
Matthew Wilcox 已提交
141 142
	spinlock_t q_lock;
	struct nvme_command *sq_cmds;
143
	struct nvme_command __iomem *sq_cmds_io;
M
Matthew Wilcox 已提交
144
	volatile struct nvme_completion *cqes;
145
	struct blk_mq_tags **tags;
M
Matthew Wilcox 已提交
146 147 148 149
	dma_addr_t sq_dma_addr;
	dma_addr_t cq_dma_addr;
	u32 __iomem *q_db;
	u16 q_depth;
J
Jens Axboe 已提交
150
	s16 cq_vector;
M
Matthew Wilcox 已提交
151 152 153
	u16 sq_head;
	u16 sq_tail;
	u16 cq_head;
K
Keith Busch 已提交
154
	u16 qid;
155 156
	u8 cq_phase;
	u8 cqe_seen;
M
Matthew Wilcox 已提交
157 158
};

159 160 161
/*
 * The nvme_iod describes the data in an I/O, including the list of PRP
 * entries.  You can't see it in this data structure because C doesn't let
C
Christoph Hellwig 已提交
162
 * me express that.  Use nvme_init_iod to ensure there's enough space
163 164 165
 * allocated to store the PRP list.
 */
struct nvme_iod {
C
Christoph Hellwig 已提交
166 167
	struct nvme_queue *nvmeq;
	int aborted;
168 169 170 171
	int npages;		/* In the PRP list. 0 means small pool in use */
	int nents;		/* Used in scatterlist */
	int length;		/* Of data, in bytes */
	dma_addr_t first_dma;
172
	struct scatterlist meta_sg; /* metadata requires single contiguous buffer */
C
Christoph Hellwig 已提交
173 174
	struct scatterlist *sg;
	struct scatterlist inline_sg[0];
M
Matthew Wilcox 已提交
175 176 177 178 179 180 181 182 183 184 185 186
};

/*
 * Check we didin't inadvertently grow the command struct
 */
static inline void _nvme_check_size(void)
{
	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
187
	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
K
Keith Busch 已提交
188
	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
M
Matthew Wilcox 已提交
189 190 191 192
	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
K
Keith Busch 已提交
193
	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
M
Matthew Wilcox 已提交
194 195
}

196 197 198 199
/*
 * Max size of iod being embedded in the request payload
 */
#define NVME_INT_PAGES		2
200
#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->ctrl.page_size)
201 202 203 204 205 206 207 208

/*
 * Will slightly overestimate the number of pages needed.  This is OK
 * as it only leads to a small amount of wasted memory for the lifetime of
 * the I/O.
 */
static int nvme_npages(unsigned size, struct nvme_dev *dev)
{
209 210
	unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
				      dev->ctrl.page_size);
211 212 213
	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}

C
Christoph Hellwig 已提交
214 215
static unsigned int nvme_iod_alloc_size(struct nvme_dev *dev,
		unsigned int size, unsigned int nseg)
216
{
C
Christoph Hellwig 已提交
217 218 219
	return sizeof(__le64 *) * nvme_npages(size, dev) +
			sizeof(struct scatterlist) * nseg;
}
220

C
Christoph Hellwig 已提交
221 222 223 224
static unsigned int nvme_cmd_size(struct nvme_dev *dev)
{
	return sizeof(struct nvme_iod) +
		nvme_iod_alloc_size(dev, NVME_INT_BYTES(dev), NVME_INT_PAGES);
225 226
}

M
Matias Bjørling 已提交
227 228
static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
				unsigned int hctx_idx)
229
{
M
Matias Bjørling 已提交
230 231 232
	struct nvme_dev *dev = data;
	struct nvme_queue *nvmeq = dev->queues[0];

233 234 235 236
	WARN_ON(hctx_idx != 0);
	WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
	WARN_ON(nvmeq->tags);

M
Matias Bjørling 已提交
237
	hctx->driver_data = nvmeq;
238
	nvmeq->tags = &dev->admin_tagset.tags[0];
M
Matias Bjørling 已提交
239
	return 0;
240 241
}

242 243 244 245 246 247 248
static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	nvmeq->tags = NULL;
}

M
Matias Bjørling 已提交
249 250 251
static int nvme_admin_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
252
{
M
Matias Bjørling 已提交
253
	struct nvme_dev *dev = data;
C
Christoph Hellwig 已提交
254
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
M
Matias Bjørling 已提交
255 256 257
	struct nvme_queue *nvmeq = dev->queues[0];

	BUG_ON(!nvmeq);
C
Christoph Hellwig 已提交
258
	iod->nvmeq = nvmeq;
M
Matias Bjørling 已提交
259
	return 0;
260 261
}

M
Matias Bjørling 已提交
262 263
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			  unsigned int hctx_idx)
M
Matthew Wilcox 已提交
264
{
M
Matias Bjørling 已提交
265
	struct nvme_dev *dev = data;
266
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
M
Matias Bjørling 已提交
267

268 269
	if (!nvmeq->tags)
		nvmeq->tags = &dev->tagset.tags[hctx_idx];
M
Matthew Wilcox 已提交
270

271
	WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
M
Matias Bjørling 已提交
272 273
	hctx->driver_data = nvmeq;
	return 0;
M
Matthew Wilcox 已提交
274 275
}

M
Matias Bjørling 已提交
276 277 278
static int nvme_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
M
Matthew Wilcox 已提交
279
{
M
Matias Bjørling 已提交
280
	struct nvme_dev *dev = data;
C
Christoph Hellwig 已提交
281
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
M
Matias Bjørling 已提交
282 283 284
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];

	BUG_ON(!nvmeq);
C
Christoph Hellwig 已提交
285
	iod->nvmeq = nvmeq;
M
Matias Bjørling 已提交
286 287 288
	return 0;
}

289 290
static void nvme_complete_async_event(struct nvme_dev *dev,
		struct nvme_completion *cqe)
M
Matias Bjørling 已提交
291
{
292 293
	u16 status = le16_to_cpu(cqe->status) >> 1;
	u32 result = le32_to_cpu(cqe->result);
M
Matias Bjørling 已提交
294 295

	if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
296
		++dev->ctrl.event_limit;
K
Keith Busch 已提交
297 298 299 300 301
	if (status != NVME_SC_SUCCESS)
		return;

	switch (result & 0xff07) {
	case NVME_AER_NOTICE_NS_CHANGED:
302 303
		dev_info(dev->dev, "rescanning\n");
		queue_work(nvme_workq, &dev->scan_work);
K
Keith Busch 已提交
304
	default:
305
		dev_warn(dev->dev, "async event result %08x\n", result);
M
Matias Bjørling 已提交
306
	}
M
Matthew Wilcox 已提交
307 308 309
}

/**
310
 * __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
M
Matthew Wilcox 已提交
311 312 313 314 315
 * @nvmeq: The queue to use
 * @cmd: The command to send
 *
 * Safe to use from interrupt context
 */
316 317
static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
						struct nvme_command *cmd)
M
Matthew Wilcox 已提交
318
{
M
Matias Bjørling 已提交
319 320
	u16 tail = nvmeq->sq_tail;

321 322 323 324 325
	if (nvmeq->sq_cmds_io)
		memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd));
	else
		memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));

M
Matthew Wilcox 已提交
326 327
	if (++tail == nvmeq->q_depth)
		tail = 0;
328
	writel(tail, nvmeq->q_db);
M
Matthew Wilcox 已提交
329 330 331
	nvmeq->sq_tail = tail;
}

C
Christoph Hellwig 已提交
332
static __le64 **iod_list(struct request *req)
M
Matthew Wilcox 已提交
333
{
C
Christoph Hellwig 已提交
334 335
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	return (__le64 **)(iod->sg + req->nr_phys_segments);
M
Matthew Wilcox 已提交
336 337
}

C
Christoph Hellwig 已提交
338
static int nvme_init_iod(struct request *rq, struct nvme_dev *dev)
339
{
C
Christoph Hellwig 已提交
340 341 342
	struct nvme_iod *iod = blk_mq_rq_to_pdu(rq);
	int nseg = rq->nr_phys_segments;
	unsigned size;
343

C
Christoph Hellwig 已提交
344 345 346 347
	if (rq->cmd_flags & REQ_DISCARD)
		size = sizeof(struct nvme_dsm_range);
	else
		size = blk_rq_bytes(rq);
348

C
Christoph Hellwig 已提交
349 350 351 352 353 354
	if (nseg > NVME_INT_PAGES || size > NVME_INT_BYTES(dev)) {
		iod->sg = kmalloc(nvme_iod_alloc_size(dev, size, nseg), GFP_ATOMIC);
		if (!iod->sg)
			return BLK_MQ_RQ_QUEUE_BUSY;
	} else {
		iod->sg = iod->inline_sg;
355 356
	}

C
Christoph Hellwig 已提交
357 358 359 360 361
	iod->aborted = 0;
	iod->npages = -1;
	iod->nents = 0;
	iod->length = size;
	return 0;
362 363
}

C
Christoph Hellwig 已提交
364
static void nvme_free_iod(struct nvme_dev *dev, struct request *req)
M
Matthew Wilcox 已提交
365
{
C
Christoph Hellwig 已提交
366
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
367
	const int last_prp = dev->ctrl.page_size / 8 - 1;
368
	int i;
C
Christoph Hellwig 已提交
369
	__le64 **list = iod_list(req);
370 371 372 373 374 375 376 377 378 379
	dma_addr_t prp_dma = iod->first_dma;

	if (iod->npages == 0)
		dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
	for (i = 0; i < iod->npages; i++) {
		__le64 *prp_list = list[i];
		dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
		dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
		prp_dma = next_prp_dma;
	}
380

C
Christoph Hellwig 已提交
381 382
	if (iod->sg != iod->inline_sg)
		kfree(iod->sg);
K
Keith Busch 已提交
383 384
}

385
#ifdef CONFIG_BLK_DEV_INTEGRITY
K
Keith Busch 已提交
386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == v)
		pi->ref_tag = cpu_to_be32(p);
}

static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == p)
		pi->ref_tag = cpu_to_be32(v);
}

/**
 * nvme_dif_remap - remaps ref tags to bip seed and physical lba
 *
 * The virtual start sector is the one that was originally submitted by the
 * block layer.	Due to partitioning, MD/DM cloning, etc. the actual physical
 * start sector may be different. Remap protection information to match the
 * physical LBA on writes, and back to the original seed on reads.
 *
 * Type 0 and 3 do not have a ref tag, so no remapping required.
 */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
	struct nvme_ns *ns = req->rq_disk->private_data;
	struct bio_integrity_payload *bip;
	struct t10_pi_tuple *pi;
	void *p, *pmap;
	u32 i, nlb, ts, phys, virt;

	if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
		return;

	bip = bio_integrity(req->bio);
	if (!bip)
		return;

	pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;

	p = pmap;
	virt = bip_get_seed(bip);
	phys = nvme_block_nr(ns, blk_rq_pos(req));
	nlb = (blk_rq_bytes(req) >> ns->lba_shift);
430
	ts = ns->disk->queue->integrity.tuple_size;
K
Keith Busch 已提交
431 432 433 434 435 436 437 438

	for (i = 0; i < nlb; i++, virt++, phys++) {
		pi = (struct t10_pi_tuple *)p;
		dif_swap(phys, virt, pi);
		p += ts;
	}
	kunmap_atomic(pmap);
}
439 440 441 442 443 444 445 446 447 448 449 450 451
#else /* CONFIG_BLK_DEV_INTEGRITY */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
}
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
#endif

C
Christoph Hellwig 已提交
452
static bool nvme_setup_prps(struct nvme_dev *dev, struct request *req,
453
		int total_len)
M
Matthew Wilcox 已提交
454
{
C
Christoph Hellwig 已提交
455
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
456
	struct dma_pool *pool;
457 458
	int length = total_len;
	struct scatterlist *sg = iod->sg;
M
Matthew Wilcox 已提交
459 460
	int dma_len = sg_dma_len(sg);
	u64 dma_addr = sg_dma_address(sg);
461
	u32 page_size = dev->ctrl.page_size;
462
	int offset = dma_addr & (page_size - 1);
463
	__le64 *prp_list;
C
Christoph Hellwig 已提交
464
	__le64 **list = iod_list(req);
465
	dma_addr_t prp_dma;
466
	int nprps, i;
M
Matthew Wilcox 已提交
467

468
	length -= (page_size - offset);
M
Matthew Wilcox 已提交
469
	if (length <= 0)
470
		return true;
M
Matthew Wilcox 已提交
471

472
	dma_len -= (page_size - offset);
M
Matthew Wilcox 已提交
473
	if (dma_len) {
474
		dma_addr += (page_size - offset);
M
Matthew Wilcox 已提交
475 476 477 478 479 480
	} else {
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
	}

481
	if (length <= page_size) {
482
		iod->first_dma = dma_addr;
483
		return true;
484 485
	}

486
	nprps = DIV_ROUND_UP(length, page_size);
487 488
	if (nprps <= (256 / 8)) {
		pool = dev->prp_small_pool;
489
		iod->npages = 0;
490 491
	} else {
		pool = dev->prp_page_pool;
492
		iod->npages = 1;
493 494
	}

495
	prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
496
	if (!prp_list) {
497
		iod->first_dma = dma_addr;
498
		iod->npages = -1;
499
		return false;
500
	}
501 502
	list[0] = prp_list;
	iod->first_dma = prp_dma;
503 504
	i = 0;
	for (;;) {
505
		if (i == page_size >> 3) {
506
			__le64 *old_prp_list = prp_list;
507
			prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
508
			if (!prp_list)
509
				return false;
510
			list[iod->npages++] = prp_list;
511 512 513
			prp_list[0] = old_prp_list[i - 1];
			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
			i = 1;
514 515
		}
		prp_list[i++] = cpu_to_le64(dma_addr);
516 517 518
		dma_len -= page_size;
		dma_addr += page_size;
		length -= page_size;
519 520 521 522 523 524 525 526
		if (length <= 0)
			break;
		if (dma_len > 0)
			continue;
		BUG_ON(dma_len < 0);
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
M
Matthew Wilcox 已提交
527 528
	}

529
	return true;
M
Matthew Wilcox 已提交
530 531
}

C
Christoph Hellwig 已提交
532
static int nvme_map_data(struct nvme_dev *dev, struct request *req,
C
Christoph Hellwig 已提交
533
		struct nvme_command *cmnd)
534
{
C
Christoph Hellwig 已提交
535
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
C
Christoph Hellwig 已提交
536 537 538 539
	struct request_queue *q = req->q;
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;
	int ret = BLK_MQ_RQ_QUEUE_ERROR;
540

C
Christoph Hellwig 已提交
541 542 543 544
	sg_init_table(iod->sg, req->nr_phys_segments);
	iod->nents = blk_rq_map_sg(q, req, iod->sg);
	if (!iod->nents)
		goto out;
545

C
Christoph Hellwig 已提交
546 547 548
	ret = BLK_MQ_RQ_QUEUE_BUSY;
	if (!dma_map_sg(dev->dev, iod->sg, iod->nents, dma_dir))
		goto out;
549

C
Christoph Hellwig 已提交
550
	if (!nvme_setup_prps(dev, req, blk_rq_bytes(req)))
C
Christoph Hellwig 已提交
551
		goto out_unmap;
552

C
Christoph Hellwig 已提交
553 554 555 556
	ret = BLK_MQ_RQ_QUEUE_ERROR;
	if (blk_integrity_rq(req)) {
		if (blk_rq_count_integrity_sg(q, req->bio) != 1)
			goto out_unmap;
557

558 559
		sg_init_table(&iod->meta_sg, 1);
		if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1)
C
Christoph Hellwig 已提交
560
			goto out_unmap;
561

C
Christoph Hellwig 已提交
562 563
		if (rq_data_dir(req))
			nvme_dif_remap(req, nvme_dif_prep);
564

565
		if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir))
C
Christoph Hellwig 已提交
566
			goto out_unmap;
567
	}
M
Matthew Wilcox 已提交
568

C
Christoph Hellwig 已提交
569 570 571
	cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
	cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);
	if (blk_integrity_rq(req))
572
		cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg));
C
Christoph Hellwig 已提交
573
	return BLK_MQ_RQ_QUEUE_OK;
M
Matthew Wilcox 已提交
574

C
Christoph Hellwig 已提交
575 576 577 578
out_unmap:
	dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
out:
	return ret;
M
Matthew Wilcox 已提交
579 580
}

C
Christoph Hellwig 已提交
581
static void nvme_unmap_data(struct nvme_dev *dev, struct request *req)
M
Matthew Wilcox 已提交
582
{
C
Christoph Hellwig 已提交
583
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
584 585 586 587 588 589 590 591
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;

	if (iod->nents) {
		dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
		if (blk_integrity_rq(req)) {
			if (!rq_data_dir(req))
				nvme_dif_remap(req, nvme_dif_complete);
592
			dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir);
K
Keith Busch 已提交
593
		}
594
	}
K
Keith Busch 已提交
595

C
Christoph Hellwig 已提交
596
	nvme_free_iod(dev, req);
597
}
M
Matthew Wilcox 已提交
598

M
Matias Bjørling 已提交
599 600 601 602 603
/*
 * We reuse the small pool to allocate the 16-byte range here as it is not
 * worth having a special pool for these or additional cases to handle freeing
 * the iod.
 */
C
Christoph Hellwig 已提交
604
static int nvme_setup_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
C
Christoph Hellwig 已提交
605
		struct request *req, struct nvme_command *cmnd)
606
{
C
Christoph Hellwig 已提交
607
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
C
Christoph Hellwig 已提交
608
	struct nvme_dsm_range *range;
M
Matthew Wilcox 已提交
609

C
Christoph Hellwig 已提交
610 611 612 613
	range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
						&iod->first_dma);
	if (!range)
		return BLK_MQ_RQ_QUEUE_BUSY;
C
Christoph Hellwig 已提交
614
	iod_list(req)[0] = (__le64 *)range;
C
Christoph Hellwig 已提交
615
	iod->npages = 0;
616 617

	range->cattr = cpu_to_le32(0);
M
Matias Bjørling 已提交
618 619
	range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
	range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
620

C
Christoph Hellwig 已提交
621 622 623 624 625 626 627
	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->dsm.opcode = nvme_cmd_dsm;
	cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
	cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
	cmnd->dsm.nr = 0;
	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
	return BLK_MQ_RQ_QUEUE_OK;
628 629
}

630 631 632
/*
 * NOTE: ns is NULL when called on the admin queue.
 */
M
Matias Bjørling 已提交
633 634
static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
			 const struct blk_mq_queue_data *bd)
635
{
M
Matias Bjørling 已提交
636 637
	struct nvme_ns *ns = hctx->queue->queuedata;
	struct nvme_queue *nvmeq = hctx->driver_data;
638
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
639
	struct request *req = bd->rq;
C
Christoph Hellwig 已提交
640 641
	struct nvme_command cmnd;
	int ret = BLK_MQ_RQ_QUEUE_OK;
642

K
Keith Busch 已提交
643 644 645 646 647
	/*
	 * If formated with metadata, require the block layer provide a buffer
	 * unless this namespace is formated such that the metadata can be
	 * stripped/generated by the controller with PRACT=1.
	 */
648
	if (ns && ns->ms && !blk_integrity_rq(req)) {
649 650
		if (!(ns->pi_type && ns->ms == 8) &&
					req->cmd_type != REQ_TYPE_DRV_PRIV) {
651
			blk_mq_end_request(req, -EFAULT);
K
Keith Busch 已提交
652 653 654 655
			return BLK_MQ_RQ_QUEUE_OK;
		}
	}

C
Christoph Hellwig 已提交
656 657 658
	ret = nvme_init_iod(req, dev);
	if (ret)
		return ret;
M
Matias Bjørling 已提交
659 660

	if (req->cmd_flags & REQ_DISCARD) {
C
Christoph Hellwig 已提交
661
		ret = nvme_setup_discard(nvmeq, ns, req, &cmnd);
C
Christoph Hellwig 已提交
662 663 664 665 666 667 668
	} else {
		if (req->cmd_type == REQ_TYPE_DRV_PRIV)
			memcpy(&cmnd, req->cmd, sizeof(cmnd));
		else if (req->cmd_flags & REQ_FLUSH)
			nvme_setup_flush(ns, &cmnd);
		else
			nvme_setup_rw(ns, req, &cmnd);
M
Matias Bjørling 已提交
669

C
Christoph Hellwig 已提交
670
		if (req->nr_phys_segments)
C
Christoph Hellwig 已提交
671
			ret = nvme_map_data(dev, req, &cmnd);
672
	}
M
Matias Bjørling 已提交
673

C
Christoph Hellwig 已提交
674 675
	if (ret)
		goto out;
M
Matias Bjørling 已提交
676

C
Christoph Hellwig 已提交
677
	cmnd.common.command_id = req->tag;
678
	blk_mq_start_request(req);
M
Matias Bjørling 已提交
679

C
Christoph Hellwig 已提交
680 681
	spin_lock_irq(&nvmeq->q_lock);
	__nvme_submit_cmd(nvmeq, &cmnd);
M
Matias Bjørling 已提交
682 683 684
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
	return BLK_MQ_RQ_QUEUE_OK;
C
Christoph Hellwig 已提交
685
out:
C
Christoph Hellwig 已提交
686
	nvme_free_iod(dev, req);
C
Christoph Hellwig 已提交
687
	return ret;
M
Matthew Wilcox 已提交
688
}
K
Keith Busch 已提交
689

690 691
static void nvme_complete_rq(struct request *req)
{
C
Christoph Hellwig 已提交
692 693
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct nvme_dev *dev = iod->nvmeq->dev;
694
	int error = 0;
K
Keith Busch 已提交
695

C
Christoph Hellwig 已提交
696
	nvme_unmap_data(dev, req);
K
Keith Busch 已提交
697

698 699 700 701
	if (unlikely(req->errors)) {
		if (nvme_req_needs_retry(req, req->errors)) {
			nvme_requeue_req(req);
			return;
K
Keith Busch 已提交
702
		}
703

704 705 706 707 708
		if (req->cmd_type == REQ_TYPE_DRV_PRIV)
			error = req->errors;
		else
			error = nvme_error_status(req->errors);
	}
M
Matias Bjørling 已提交
709

C
Christoph Hellwig 已提交
710
	if (unlikely(iod->aborted)) {
711 712 713 714
		dev_warn(dev->dev,
			"completing aborted command with status: %04x\n",
			req->errors);
	}
M
Matias Bjørling 已提交
715

716
	blk_mq_end_request(req, error);
M
Matthew Wilcox 已提交
717 718
}

J
Jens Axboe 已提交
719
static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
M
Matthew Wilcox 已提交
720
{
M
Matthew Wilcox 已提交
721
	u16 head, phase;
M
Matthew Wilcox 已提交
722 723

	head = nvmeq->cq_head;
M
Matthew Wilcox 已提交
724
	phase = nvmeq->cq_phase;
M
Matthew Wilcox 已提交
725 726 727

	for (;;) {
		struct nvme_completion cqe = nvmeq->cqes[head];
728
		u16 status = le16_to_cpu(cqe.status);
729
		struct request *req;
730 731

		if ((status & 1) != phase)
M
Matthew Wilcox 已提交
732 733 734 735
			break;
		nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
		if (++head == nvmeq->q_depth) {
			head = 0;
M
Matthew Wilcox 已提交
736
			phase = !phase;
M
Matthew Wilcox 已提交
737
		}
738

J
Jens Axboe 已提交
739 740
		if (tag && *tag == cqe.command_id)
			*tag = -1;
741

742 743 744 745 746 747 748
		if (unlikely(cqe.command_id >= nvmeq->q_depth)) {
			dev_warn(nvmeq->q_dmadev,
				"invalid id %d completed on queue %d\n",
				cqe.command_id, le16_to_cpu(cqe.sq_id));
			continue;
		}

749 750 751 752 753 754 755 756 757 758 759 760
		/*
		 * AEN requests are special as they don't time out and can
		 * survive any kind of queue freeze and often don't respond to
		 * aborts.  We don't even bother to allocate a struct request
		 * for them but rather special case them here.
		 */
		if (unlikely(nvmeq->qid == 0 &&
				cqe.command_id >= NVME_AQ_BLKMQ_DEPTH)) {
			nvme_complete_async_event(nvmeq->dev, &cqe);
			continue;
		}

761 762 763 764 765 766 767
		req = blk_mq_tag_to_rq(*nvmeq->tags, cqe.command_id);
		if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
			u32 result = le32_to_cpu(cqe.result);
			req->special = (void *)(uintptr_t)result;
		}
		blk_mq_complete_request(req, status >> 1);

M
Matthew Wilcox 已提交
768 769 770 771 772 773 774 775
	}

	/* If the controller ignores the cq head doorbell and continuously
	 * writes to the queue, it is theoretically possible to wrap around
	 * the queue twice and mistakenly return IRQ_NONE.  Linux only
	 * requires that 0.1% of your interrupts are handled, so this isn't
	 * a big problem.
	 */
M
Matthew Wilcox 已提交
776
	if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
J
Jens Axboe 已提交
777
		return;
M
Matthew Wilcox 已提交
778

779 780
	if (likely(nvmeq->cq_vector >= 0))
		writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
M
Matthew Wilcox 已提交
781
	nvmeq->cq_head = head;
M
Matthew Wilcox 已提交
782
	nvmeq->cq_phase = phase;
M
Matthew Wilcox 已提交
783

784
	nvmeq->cqe_seen = 1;
J
Jens Axboe 已提交
785 786 787 788 789
}

static void nvme_process_cq(struct nvme_queue *nvmeq)
{
	__nvme_process_cq(nvmeq, NULL);
M
Matthew Wilcox 已提交
790 791 792
}

static irqreturn_t nvme_irq(int irq, void *data)
793 794 795 796
{
	irqreturn_t result;
	struct nvme_queue *nvmeq = data;
	spin_lock(&nvmeq->q_lock);
797 798 799
	nvme_process_cq(nvmeq);
	result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
	nvmeq->cqe_seen = 0;
800 801 802 803 804 805 806 807 808 809 810 811 812
	spin_unlock(&nvmeq->q_lock);
	return result;
}

static irqreturn_t nvme_irq_check(int irq, void *data)
{
	struct nvme_queue *nvmeq = data;
	struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
	if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
		return IRQ_NONE;
	return IRQ_WAKE_THREAD;
}

J
Jens Axboe 已提交
813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829
static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	if ((le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
	    nvmeq->cq_phase) {
		spin_lock_irq(&nvmeq->q_lock);
		__nvme_process_cq(nvmeq, &tag);
		spin_unlock_irq(&nvmeq->q_lock);

		if (tag == -1)
			return 1;
	}

	return 0;
}

830
static void nvme_submit_async_event(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
831
{
M
Matias Bjørling 已提交
832
	struct nvme_command c;
M
Matthew Wilcox 已提交
833

M
Matias Bjørling 已提交
834 835
	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_async_event;
836
	c.common.command_id = NVME_AQ_BLKMQ_DEPTH + --dev->ctrl.event_limit;
837

838
	__nvme_submit_cmd(dev->queues[0], &c);
839 840
}

M
Matthew Wilcox 已提交
841
static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
842
{
M
Matthew Wilcox 已提交
843 844 845 846 847 848
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(id);

849
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
850 851 852 853 854 855 856 857
}

static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;

858 859 860 861
	/*
	 * Note: we (ab)use the fact the the prp fields survive if no data
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
862 863 864 865 866 867 868 869
	memset(&c, 0, sizeof(c));
	c.create_cq.opcode = nvme_admin_create_cq;
	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
	c.create_cq.cqid = cpu_to_le16(qid);
	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_cq.cq_flags = cpu_to_le16(flags);
	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);

870
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
871 872 873 874 875 876 877 878
}

static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;

879 880 881 882
	/*
	 * Note: we (ab)use the fact the the prp fields survive if no data
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
883 884 885 886 887 888 889 890
	memset(&c, 0, sizeof(c));
	c.create_sq.opcode = nvme_admin_create_sq;
	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
	c.create_sq.sqid = cpu_to_le16(qid);
	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_sq.sq_flags = cpu_to_le16(flags);
	c.create_sq.cqid = cpu_to_le16(qid);

891
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
892 893 894 895 896 897 898 899 900 901 902 903
}

static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
}

static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
}

904
static void abort_endio(struct request *req, int error)
905
{
C
Christoph Hellwig 已提交
906 907
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = iod->nvmeq;
908 909
	u32 result = (u32)(uintptr_t)req->special;
	u16 status = req->errors;
910

911 912
	dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
	atomic_inc(&nvmeq->dev->ctrl.abort_limit);
913

914
	blk_mq_free_request(req);
915 916
}

917
static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
K
Keith Busch 已提交
918
{
C
Christoph Hellwig 已提交
919 920
	struct nvme_iod *iod = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = iod->nvmeq;
K
Keith Busch 已提交
921
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
922 923
	struct request *abort_req;
	struct nvme_command cmd;
K
Keith Busch 已提交
924

925
	/*
926 927 928 929 930 931 932 933 934
	 * Shutdown immediately if controller times out while starting. The
	 * reset work will see the pci device disabled when it gets the forced
	 * cancellation error. All outstanding requests are completed on
	 * shutdown, so we return BLK_EH_HANDLED.
	 */
	if (test_bit(NVME_CTRL_RESETTING, &dev->flags)) {
		dev_warn(dev->dev,
			 "I/O %d QID %d timeout, disable controller\n",
			 req->tag, nvmeq->qid);
935
		nvme_dev_disable(dev, false);
936 937
		req->errors = NVME_SC_CANCELLED;
		return BLK_EH_HANDLED;
K
Keith Busch 已提交
938 939
	}

940 941 942 943
	/*
 	 * Shutdown the controller immediately and schedule a reset if the
 	 * command was already aborted once before and still hasn't been
 	 * returned to the driver, or if this is the admin queue.
944
	 */
C
Christoph Hellwig 已提交
945
	if (!nvmeq->qid || iod->aborted) {
946 947 948
		dev_warn(dev->dev,
			 "I/O %d QID %d timeout, reset controller\n",
			 req->tag, nvmeq->qid);
949
		nvme_dev_disable(dev, false);
950
		queue_work(nvme_workq, &dev->reset_work);
K
Keith Busch 已提交
951

952 953 954 955 956 957
		/*
		 * Mark the request as handled, since the inline shutdown
		 * forces all outstanding requests to complete.
		 */
		req->errors = NVME_SC_CANCELLED;
		return BLK_EH_HANDLED;
K
Keith Busch 已提交
958 959
	}

C
Christoph Hellwig 已提交
960
	iod->aborted = 1;
K
Keith Busch 已提交
961

962
	if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
963
		atomic_inc(&dev->ctrl.abort_limit);
964
		return BLK_EH_RESET_TIMER;
965
	}
M
Matias Bjørling 已提交
966

K
Keith Busch 已提交
967 968
	memset(&cmd, 0, sizeof(cmd));
	cmd.abort.opcode = nvme_admin_abort_cmd;
M
Matias Bjørling 已提交
969
	cmd.abort.cid = req->tag;
K
Keith Busch 已提交
970 971
	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);

972 973
	dev_warn(nvmeq->q_dmadev, "I/O %d QID %d timeout, aborting\n",
				 req->tag, nvmeq->qid);
974 975 976 977 978 979 980 981 982 983 984

	abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
			BLK_MQ_REQ_NOWAIT);
	if (IS_ERR(abort_req)) {
		atomic_inc(&dev->ctrl.abort_limit);
		return BLK_EH_RESET_TIMER;
	}

	abort_req->timeout = ADMIN_TIMEOUT;
	abort_req->end_io_data = NULL;
	blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
K
Keith Busch 已提交
985

986 987 988 989 990 991
	/*
	 * The aborted req will be completed on receiving the abort req.
	 * We enable the timer again. If hit twice, it'll cause a device reset,
	 * as the device then is in a faulty state.
	 */
	return BLK_EH_RESET_TIMER;
K
Keith Busch 已提交
992 993
}

994
static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved)
995
{
M
Matias Bjørling 已提交
996
	struct nvme_queue *nvmeq = data;
997
	int status;
K
Keith Busch 已提交
998 999 1000

	if (!blk_mq_request_started(req))
		return;
1001

1002 1003
	dev_warn(nvmeq->q_dmadev,
		 "Cancelling I/O %d QID %d\n", req->tag, nvmeq->qid);
M
Matias Bjørling 已提交
1004

1005
	status = NVME_SC_ABORT_REQ;
K
Keith Busch 已提交
1006
	if (blk_queue_dying(req->q))
1007 1008
		status |= NVME_SC_DNR;
	blk_mq_complete_request(req, status);
M
Matias Bjørling 已提交
1009
}
1010

M
Matias Bjørling 已提交
1011 1012
static void nvme_free_queue(struct nvme_queue *nvmeq)
{
1013 1014
	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1015 1016
	if (nvmeq->sq_cmds)
		dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1017 1018 1019 1020
					nvmeq->sq_cmds, nvmeq->sq_dma_addr);
	kfree(nvmeq);
}

1021
static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1022 1023 1024
{
	int i;

1025
	for (i = dev->queue_count - 1; i >= lowest; i--) {
M
Matias Bjørling 已提交
1026
		struct nvme_queue *nvmeq = dev->queues[i];
1027
		dev->queue_count--;
M
Matias Bjørling 已提交
1028
		dev->queues[i] = NULL;
K
Keith Busch 已提交
1029
		nvme_free_queue(nvmeq);
1030
	}
1031 1032
}

K
Keith Busch 已提交
1033 1034 1035 1036 1037
/**
 * nvme_suspend_queue - put queue into suspended state
 * @nvmeq - queue to suspend
 */
static int nvme_suspend_queue(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
1038
{
K
Keith Busch 已提交
1039
	int vector;
M
Matthew Wilcox 已提交
1040

1041
	spin_lock_irq(&nvmeq->q_lock);
K
Keith Busch 已提交
1042 1043 1044 1045 1046
	if (nvmeq->cq_vector == -1) {
		spin_unlock_irq(&nvmeq->q_lock);
		return 1;
	}
	vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
K
Keith Busch 已提交
1047
	nvmeq->dev->online_queues--;
K
Keith Busch 已提交
1048
	nvmeq->cq_vector = -1;
1049 1050
	spin_unlock_irq(&nvmeq->q_lock);

1051
	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
1052
		blk_mq_stop_hw_queues(nvmeq->dev->ctrl.admin_q);
1053

M
Matthew Wilcox 已提交
1054 1055
	irq_set_affinity_hint(vector, NULL);
	free_irq(vector, nvmeq);
M
Matthew Wilcox 已提交
1056

K
Keith Busch 已提交
1057 1058
	return 0;
}
M
Matthew Wilcox 已提交
1059

K
Keith Busch 已提交
1060 1061
static void nvme_clear_queue(struct nvme_queue *nvmeq)
{
1062
	spin_lock_irq(&nvmeq->q_lock);
1063 1064
	if (nvmeq->tags && *nvmeq->tags)
		blk_mq_all_tag_busy_iter(*nvmeq->tags, nvme_cancel_queue_ios, nvmeq);
1065
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1066 1067
}

1068
static void nvme_disable_admin_queue(struct nvme_dev *dev, bool shutdown)
K
Keith Busch 已提交
1069
{
1070
	struct nvme_queue *nvmeq = dev->queues[0];
K
Keith Busch 已提交
1071 1072 1073 1074 1075 1076

	if (!nvmeq)
		return;
	if (nvme_suspend_queue(nvmeq))
		return;

1077 1078 1079 1080 1081
	if (shutdown)
		nvme_shutdown_ctrl(&dev->ctrl);
	else
		nvme_disable_ctrl(&dev->ctrl, lo_hi_readq(
						dev->bar + NVME_REG_CAP));
1082 1083 1084 1085

	spin_lock_irq(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1086 1087
}

1088 1089 1090 1091
static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
				int entry_size)
{
	int q_depth = dev->q_depth;
1092 1093
	unsigned q_size_aligned = roundup(q_depth * entry_size,
					  dev->ctrl.page_size);
1094 1095

	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1096
		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1097
		mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
1098
		q_depth = div_u64(mem_per_q, entry_size);
1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115

		/*
		 * Ensure the reduced q_depth is above some threshold where it
		 * would be better to map queues in system memory with the
		 * original depth
		 */
		if (q_depth < 64)
			return -ENOMEM;
	}

	return q_depth;
}

static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
				int qid, int depth)
{
	if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
1116 1117
		unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth),
						      dev->ctrl.page_size);
1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
		nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
		nvmeq->sq_cmds_io = dev->cmb + offset;
	} else {
		nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
					&nvmeq->sq_dma_addr, GFP_KERNEL);
		if (!nvmeq->sq_cmds)
			return -ENOMEM;
	}

	return 0;
}

M
Matthew Wilcox 已提交
1130
static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
K
Keith Busch 已提交
1131
							int depth)
M
Matthew Wilcox 已提交
1132
{
M
Matias Bjørling 已提交
1133
	struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
M
Matthew Wilcox 已提交
1134 1135 1136
	if (!nvmeq)
		return NULL;

1137
	nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
J
Joe Perches 已提交
1138
					  &nvmeq->cq_dma_addr, GFP_KERNEL);
M
Matthew Wilcox 已提交
1139 1140 1141
	if (!nvmeq->cqes)
		goto free_nvmeq;

1142
	if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
M
Matthew Wilcox 已提交
1143 1144
		goto free_cqdma;

1145
	nvmeq->q_dmadev = dev->dev;
M
Matthew Wilcox 已提交
1146
	nvmeq->dev = dev;
1147
	snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
1148
			dev->ctrl.instance, qid);
M
Matthew Wilcox 已提交
1149 1150
	spin_lock_init(&nvmeq->q_lock);
	nvmeq->cq_head = 0;
M
Matthew Wilcox 已提交
1151
	nvmeq->cq_phase = 1;
1152
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
M
Matthew Wilcox 已提交
1153
	nvmeq->q_depth = depth;
K
Keith Busch 已提交
1154
	nvmeq->qid = qid;
1155
	nvmeq->cq_vector = -1;
M
Matias Bjørling 已提交
1156
	dev->queues[qid] = nvmeq;
M
Matthew Wilcox 已提交
1157

1158 1159 1160 1161
	/* make sure queue descriptor is set before queue count, for kthread */
	mb();
	dev->queue_count++;

M
Matthew Wilcox 已提交
1162 1163 1164
	return nvmeq;

 free_cqdma:
1165
	dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
M
Matthew Wilcox 已提交
1166 1167 1168 1169 1170 1171
							nvmeq->cq_dma_addr);
 free_nvmeq:
	kfree(nvmeq);
	return NULL;
}

1172 1173 1174
static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
							const char *name)
{
1175 1176
	if (use_threaded_interrupts)
		return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1177
					nvme_irq_check, nvme_irq, IRQF_SHARED,
1178
					name, nvmeq);
1179
	return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1180
				IRQF_SHARED, name, nvmeq);
1181 1182
}

1183
static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
M
Matthew Wilcox 已提交
1184
{
1185
	struct nvme_dev *dev = nvmeq->dev;
M
Matthew Wilcox 已提交
1186

1187
	spin_lock_irq(&nvmeq->q_lock);
1188 1189 1190
	nvmeq->sq_tail = 0;
	nvmeq->cq_head = 0;
	nvmeq->cq_phase = 1;
1191
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1192
	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
K
Keith Busch 已提交
1193
	dev->online_queues++;
1194
	spin_unlock_irq(&nvmeq->q_lock);
1195 1196 1197 1198 1199 1200
}

static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
{
	struct nvme_dev *dev = nvmeq->dev;
	int result;
1201

K
Keith Busch 已提交
1202
	nvmeq->cq_vector = qid - 1;
M
Matthew Wilcox 已提交
1203 1204
	result = adapter_alloc_cq(dev, qid, nvmeq);
	if (result < 0)
1205
		return result;
M
Matthew Wilcox 已提交
1206 1207 1208 1209 1210

	result = adapter_alloc_sq(dev, qid, nvmeq);
	if (result < 0)
		goto release_cq;

1211
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
M
Matthew Wilcox 已提交
1212 1213 1214
	if (result < 0)
		goto release_sq;

1215 1216
	nvme_init_queue(nvmeq, qid);
	return result;
M
Matthew Wilcox 已提交
1217 1218 1219 1220 1221

 release_sq:
	adapter_delete_sq(dev, qid);
 release_cq:
	adapter_delete_cq(dev, qid);
1222
	return result;
M
Matthew Wilcox 已提交
1223 1224
}

M
Matias Bjørling 已提交
1225
static struct blk_mq_ops nvme_mq_admin_ops = {
1226
	.queue_rq	= nvme_queue_rq,
1227
	.complete	= nvme_complete_rq,
M
Matias Bjørling 已提交
1228 1229
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_admin_init_hctx,
1230
	.exit_hctx      = nvme_admin_exit_hctx,
M
Matias Bjørling 已提交
1231 1232 1233 1234 1235 1236
	.init_request	= nvme_admin_init_request,
	.timeout	= nvme_timeout,
};

static struct blk_mq_ops nvme_mq_ops = {
	.queue_rq	= nvme_queue_rq,
1237
	.complete	= nvme_complete_rq,
M
Matias Bjørling 已提交
1238 1239 1240 1241
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_init_hctx,
	.init_request	= nvme_init_request,
	.timeout	= nvme_timeout,
J
Jens Axboe 已提交
1242
	.poll		= nvme_poll,
M
Matias Bjørling 已提交
1243 1244
};

1245 1246
static void nvme_dev_remove_admin(struct nvme_dev *dev)
{
1247 1248
	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
		blk_cleanup_queue(dev->ctrl.admin_q);
1249 1250 1251 1252
		blk_mq_free_tag_set(&dev->admin_tagset);
	}
}

M
Matias Bjørling 已提交
1253 1254
static int nvme_alloc_admin_tags(struct nvme_dev *dev)
{
1255
	if (!dev->ctrl.admin_q) {
M
Matias Bjørling 已提交
1256 1257
		dev->admin_tagset.ops = &nvme_mq_admin_ops;
		dev->admin_tagset.nr_hw_queues = 1;
K
Keith Busch 已提交
1258 1259 1260 1261 1262 1263

		/*
		 * Subtract one to leave an empty queue entry for 'Full Queue'
		 * condition. See NVM-Express 1.2 specification, section 4.1.2.
		 */
		dev->admin_tagset.queue_depth = NVME_AQ_BLKMQ_DEPTH - 1;
M
Matias Bjørling 已提交
1264
		dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1265
		dev->admin_tagset.numa_node = dev_to_node(dev->dev);
1266
		dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
M
Matias Bjørling 已提交
1267 1268 1269 1270 1271
		dev->admin_tagset.driver_data = dev;

		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
			return -ENOMEM;

1272 1273
		dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
		if (IS_ERR(dev->ctrl.admin_q)) {
M
Matias Bjørling 已提交
1274 1275 1276
			blk_mq_free_tag_set(&dev->admin_tagset);
			return -ENOMEM;
		}
1277
		if (!blk_get_queue(dev->ctrl.admin_q)) {
1278
			nvme_dev_remove_admin(dev);
1279
			dev->ctrl.admin_q = NULL;
1280 1281
			return -ENODEV;
		}
K
Keith Busch 已提交
1282
	} else
1283
		blk_mq_start_stopped_hw_queues(dev->ctrl.admin_q, true);
M
Matias Bjørling 已提交
1284 1285 1286 1287

	return 0;
}

1288
static int nvme_configure_admin_queue(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1289
{
1290
	int result;
M
Matthew Wilcox 已提交
1291
	u32 aqa;
1292
	u64 cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
M
Matthew Wilcox 已提交
1293 1294
	struct nvme_queue *nvmeq;

1295
	dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1) ?
1296 1297
						NVME_CAP_NSSRC(cap) : 0;

1298 1299 1300
	if (dev->subsystem &&
	    (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
		writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1301

1302
	result = nvme_disable_ctrl(&dev->ctrl, cap);
1303 1304
	if (result < 0)
		return result;
M
Matthew Wilcox 已提交
1305

M
Matias Bjørling 已提交
1306
	nvmeq = dev->queues[0];
1307
	if (!nvmeq) {
K
Keith Busch 已提交
1308
		nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1309 1310 1311
		if (!nvmeq)
			return -ENOMEM;
	}
M
Matthew Wilcox 已提交
1312 1313 1314 1315

	aqa = nvmeq->q_depth - 1;
	aqa |= aqa << 16;

1316 1317 1318
	writel(aqa, dev->bar + NVME_REG_AQA);
	lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
	lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
M
Matthew Wilcox 已提交
1319

1320
	result = nvme_enable_ctrl(&dev->ctrl, cap);
1321
	if (result)
M
Matias Bjørling 已提交
1322 1323
		goto free_nvmeq;

K
Keith Busch 已提交
1324
	nvmeq->cq_vector = 0;
1325
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
1326 1327
	if (result) {
		nvmeq->cq_vector = -1;
K
Keith Busch 已提交
1328
		goto free_nvmeq;
1329
	}
1330

M
Matthew Wilcox 已提交
1331
	return result;
M
Matias Bjørling 已提交
1332 1333 1334 1335

 free_nvmeq:
	nvme_free_queues(dev, 0);
	return result;
M
Matthew Wilcox 已提交
1336 1337
}

1338 1339
static int nvme_kthread(void *data)
{
1340
	struct nvme_dev *dev, *next;
1341 1342

	while (!kthread_should_stop()) {
1343
		set_current_state(TASK_INTERRUPTIBLE);
1344
		spin_lock(&dev_list_lock);
1345
		list_for_each_entry_safe(dev, next, &dev_list, node) {
1346
			int i;
1347
			u32 csts = readl(dev->bar + NVME_REG_CSTS);
1348

C
Christoph Hellwig 已提交
1349 1350 1351 1352 1353
			/*
			 * Skip controllers currently under reset.
			 */
			if (work_pending(&dev->reset_work) || work_busy(&dev->reset_work))
				continue;
1354 1355 1356

			if ((dev->subsystem && (csts & NVME_CSTS_NSSRO)) ||
							csts & NVME_CSTS_CFS) {
C
Christoph Hellwig 已提交
1357
				if (queue_work(nvme_workq, &dev->reset_work)) {
1358 1359
					dev_warn(dev->dev,
						"Failed status: %x, reset controller\n",
1360
						readl(dev->bar + NVME_REG_CSTS));
1361
				}
1362 1363
				continue;
			}
1364
			for (i = 0; i < dev->queue_count; i++) {
M
Matias Bjørling 已提交
1365
				struct nvme_queue *nvmeq = dev->queues[i];
1366 1367
				if (!nvmeq)
					continue;
1368
				spin_lock_irq(&nvmeq->q_lock);
1369
				nvme_process_cq(nvmeq);
K
Keith Busch 已提交
1370

1371 1372
				while (i == 0 && dev->ctrl.event_limit > 0)
					nvme_submit_async_event(dev);
1373 1374 1375 1376
				spin_unlock_irq(&nvmeq->q_lock);
			}
		}
		spin_unlock(&dev_list_lock);
1377
		schedule_timeout(round_jiffies_relative(HZ));
1378 1379 1380 1381
	}
	return 0;
}

1382
static int nvme_create_io_queues(struct nvme_dev *dev)
K
Keith Busch 已提交
1383
{
M
Matias Bjørling 已提交
1384
	unsigned i;
1385
	int ret = 0;
K
Keith Busch 已提交
1386

1387 1388 1389
	for (i = dev->queue_count; i <= dev->max_qid; i++) {
		if (!nvme_alloc_queue(dev, i, dev->q_depth)) {
			ret = -ENOMEM;
K
Keith Busch 已提交
1390
			break;
1391 1392
		}
	}
K
Keith Busch 已提交
1393

1394 1395 1396
	for (i = dev->online_queues; i <= dev->queue_count - 1; i++) {
		ret = nvme_create_queue(dev->queues[i], i);
		if (ret) {
1397
			nvme_free_queues(dev, i);
K
Keith Busch 已提交
1398
			break;
1399
		}
M
Matthew Wilcox 已提交
1400
	}
1401 1402 1403 1404 1405 1406 1407 1408

	/*
	 * Ignore failing Create SQ/CQ commands, we can continue with less
	 * than the desired aount of queues, and even a controller without
	 * I/O queues an still be used to issue admin commands.  This might
	 * be useful to upgrade a buggy firmware for example.
	 */
	return ret >= 0 ? 0 : ret;
M
Matthew Wilcox 已提交
1409 1410
}

1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421 1422
static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
{
	u64 szu, size, offset;
	u32 cmbloc;
	resource_size_t bar_size;
	struct pci_dev *pdev = to_pci_dev(dev->dev);
	void __iomem *cmb;
	dma_addr_t dma_addr;

	if (!use_cmb_sqes)
		return NULL;

1423
	dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1424 1425 1426
	if (!(NVME_CMB_SZ(dev->cmbsz)))
		return NULL;

1427
	cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459 1460 1461 1462

	szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
	size = szu * NVME_CMB_SZ(dev->cmbsz);
	offset = szu * NVME_CMB_OFST(cmbloc);
	bar_size = pci_resource_len(pdev, NVME_CMB_BIR(cmbloc));

	if (offset > bar_size)
		return NULL;

	/*
	 * Controllers may support a CMB size larger than their BAR,
	 * for example, due to being behind a bridge. Reduce the CMB to
	 * the reported size of the BAR
	 */
	if (size > bar_size - offset)
		size = bar_size - offset;

	dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(cmbloc)) + offset;
	cmb = ioremap_wc(dma_addr, size);
	if (!cmb)
		return NULL;

	dev->cmb_dma_addr = dma_addr;
	dev->cmb_size = size;
	return cmb;
}

static inline void nvme_release_cmb(struct nvme_dev *dev)
{
	if (dev->cmb) {
		iounmap(dev->cmb);
		dev->cmb = NULL;
	}
}

K
Keith Busch 已提交
1463 1464
static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
{
1465
	return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
K
Keith Busch 已提交
1466 1467
}

1468
static int nvme_setup_io_queues(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1469
{
M
Matias Bjørling 已提交
1470
	struct nvme_queue *adminq = dev->queues[0];
1471
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
1472
	int result, i, vecs, nr_io_queues, size;
M
Matthew Wilcox 已提交
1473

K
Keith Busch 已提交
1474
	nr_io_queues = num_possible_cpus();
C
Christoph Hellwig 已提交
1475 1476
	result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
	if (result < 0)
M
Matthew Wilcox 已提交
1477
		return result;
C
Christoph Hellwig 已提交
1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488

	/*
	 * Degraded controllers might return an error when setting the queue
	 * count.  We still want to be able to bring them online and offer
	 * access to the admin queue, as that might be only way to fix them up.
	 */
	if (result > 0) {
		dev_err(dev->dev, "Could not set queue count (%d)\n", result);
		nr_io_queues = 0;
		result = 0;
	}
M
Matthew Wilcox 已提交
1489

1490 1491 1492 1493 1494 1495 1496 1497 1498
	if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
		result = nvme_cmb_qdepth(dev, nr_io_queues,
				sizeof(struct nvme_command));
		if (result > 0)
			dev->q_depth = result;
		else
			nvme_release_cmb(dev);
	}

K
Keith Busch 已提交
1499 1500
	size = db_bar_size(dev, nr_io_queues);
	if (size > 8192) {
1501
		iounmap(dev->bar);
K
Keith Busch 已提交
1502 1503 1504 1505 1506 1507 1508 1509
		do {
			dev->bar = ioremap(pci_resource_start(pdev, 0), size);
			if (dev->bar)
				break;
			if (!--nr_io_queues)
				return -ENOMEM;
			size = db_bar_size(dev, nr_io_queues);
		} while (1);
1510
		dev->dbs = dev->bar + 4096;
1511
		adminq->q_db = dev->dbs;
1512 1513
	}

K
Keith Busch 已提交
1514
	/* Deregister the admin queue's interrupt */
1515
	free_irq(dev->entry[0].vector, adminq);
K
Keith Busch 已提交
1516

1517 1518 1519 1520 1521 1522 1523
	/*
	 * If we enable msix early due to not intx, disable it again before
	 * setting up the full range we need.
	 */
	if (!pdev->irq)
		pci_disable_msix(pdev);

1524
	for (i = 0; i < nr_io_queues; i++)
M
Matthew Wilcox 已提交
1525
		dev->entry[i].entry = i;
1526 1527 1528 1529 1530 1531 1532 1533
	vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
	if (vecs < 0) {
		vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
		if (vecs < 0) {
			vecs = 1;
		} else {
			for (i = 0; i < vecs; i++)
				dev->entry[i].vector = i + pdev->irq;
R
Ramachandra Rao Gajula 已提交
1534 1535 1536
		}
	}

1537 1538 1539 1540 1541 1542 1543
	/*
	 * Should investigate if there's a performance win from allocating
	 * more queues than interrupt vectors; it might allow the submission
	 * path to scale better, even if the receive path is limited by the
	 * number of interrupts.
	 */
	nr_io_queues = vecs;
K
Keith Busch 已提交
1544
	dev->max_qid = nr_io_queues;
1545

1546
	result = queue_request_irq(dev, adminq, adminq->irqname);
1547 1548
	if (result) {
		adminq->cq_vector = -1;
1549
		goto free_queues;
1550
	}
M
Matthew Wilcox 已提交
1551

1552
	/* Free previously allocated queues that are no longer usable */
K
Keith Busch 已提交
1553
	nvme_free_queues(dev, nr_io_queues + 1);
1554
	return nvme_create_io_queues(dev);
M
Matthew Wilcox 已提交
1555

1556
 free_queues:
1557
	nvme_free_queues(dev, 1);
1558
	return result;
M
Matthew Wilcox 已提交
1559 1560
}

1561
static void nvme_set_irq_hints(struct nvme_dev *dev)
K
Keith Busch 已提交
1562
{
1563 1564
	struct nvme_queue *nvmeq;
	int i;
K
Keith Busch 已提交
1565

1566 1567
	for (i = 0; i < dev->online_queues; i++) {
		nvmeq = dev->queues[i];
K
Keith Busch 已提交
1568

1569 1570
		if (!nvmeq->tags || !(*nvmeq->tags))
			continue;
K
Keith Busch 已提交
1571

1572 1573
		irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
					blk_mq_tags_cpumask(*nvmeq->tags));
K
Keith Busch 已提交
1574 1575 1576 1577 1578 1579 1580 1581 1582
	}
}

static void nvme_dev_scan(struct work_struct *work)
{
	struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);

	if (!dev->tagset.tags)
		return;
1583
	nvme_scan_namespaces(&dev->ctrl);
1584
	nvme_set_irq_hints(dev);
K
Keith Busch 已提交
1585 1586
}

K
Keith Busch 已提交
1587
static void nvme_del_queue_end(struct request *req, int error)
K
Keith Busch 已提交
1588
{
K
Keith Busch 已提交
1589
	struct nvme_queue *nvmeq = req->end_io_data;
1590

K
Keith Busch 已提交
1591 1592
	blk_mq_free_request(req);
	complete(&nvmeq->dev->ioq_wait);
K
Keith Busch 已提交
1593 1594
}

K
Keith Busch 已提交
1595
static void nvme_del_cq_end(struct request *req, int error)
K
Keith Busch 已提交
1596
{
K
Keith Busch 已提交
1597
	struct nvme_queue *nvmeq = req->end_io_data;
K
Keith Busch 已提交
1598

K
Keith Busch 已提交
1599 1600 1601 1602 1603 1604
	if (!error) {
		unsigned long flags;

		spin_lock_irqsave(&nvmeq->q_lock, flags);
		nvme_process_cq(nvmeq);
		spin_unlock_irqrestore(&nvmeq->q_lock, flags);
K
Keith Busch 已提交
1605
	}
K
Keith Busch 已提交
1606 1607

	nvme_del_queue_end(req, error);
K
Keith Busch 已提交
1608 1609
}

K
Keith Busch 已提交
1610
static int nvme_delete_queue(struct nvme_queue *nvmeq, u8 opcode)
1611
{
K
Keith Busch 已提交
1612 1613 1614
	struct request_queue *q = nvmeq->dev->ctrl.admin_q;
	struct request *req;
	struct nvme_command cmd;
1615

K
Keith Busch 已提交
1616 1617 1618
	memset(&cmd, 0, sizeof(cmd));
	cmd.delete_queue.opcode = opcode;
	cmd.delete_queue.qid = cpu_to_le16(nvmeq->qid);
1619

K
Keith Busch 已提交
1620 1621 1622
	req = nvme_alloc_request(q, &cmd, BLK_MQ_REQ_NOWAIT);
	if (IS_ERR(req))
		return PTR_ERR(req);
1623

K
Keith Busch 已提交
1624 1625 1626 1627 1628 1629 1630
	req->timeout = ADMIN_TIMEOUT;
	req->end_io_data = nvmeq;

	blk_execute_rq_nowait(q, NULL, req, false,
			opcode == nvme_admin_delete_cq ?
				nvme_del_cq_end : nvme_del_queue_end);
	return 0;
1631 1632
}

K
Keith Busch 已提交
1633
static void nvme_disable_io_queues(struct nvme_dev *dev)
K
Keith Busch 已提交
1634
{
K
Keith Busch 已提交
1635 1636 1637
	int pass;
	unsigned long timeout;
	u8 opcode = nvme_admin_delete_sq;
K
Keith Busch 已提交
1638

K
Keith Busch 已提交
1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662
	for (pass = 0; pass < 2; pass++) {
		int sent = 0, i = dev->queue_count - 1;

		reinit_completion(&dev->ioq_wait);
 retry:
		timeout = ADMIN_TIMEOUT;
		for (; i > 0; i--) {
			struct nvme_queue *nvmeq = dev->queues[i];

			if (!pass)
				nvme_suspend_queue(nvmeq);
			if (nvme_delete_queue(nvmeq, opcode))
				break;
			++sent;
		}
		while (sent--) {
			timeout = wait_for_completion_io_timeout(&dev->ioq_wait, timeout);
			if (timeout == 0)
				return;
			if (i)
				goto retry;
		}
		opcode = nvme_admin_delete_cq;
	}
K
Keith Busch 已提交
1663 1664
}

1665 1666 1667 1668 1669 1670
/*
 * Return: error value if an error occurred setting up the queues or calling
 * Identify Device.  0 if these succeeded, even if adding some of the
 * namespaces failed.  At the moment, these failures are silent.  TBD which
 * failures should be reported.
 */
1671
static int nvme_dev_add(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1672
{
1673
	if (!dev->ctrl.tagset) {
1674 1675 1676 1677 1678
		dev->tagset.ops = &nvme_mq_ops;
		dev->tagset.nr_hw_queues = dev->online_queues - 1;
		dev->tagset.timeout = NVME_IO_TIMEOUT;
		dev->tagset.numa_node = dev_to_node(dev->dev);
		dev->tagset.queue_depth =
M
Matias Bjørling 已提交
1679
				min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
1680 1681 1682
		dev->tagset.cmd_size = nvme_cmd_size(dev);
		dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
		dev->tagset.driver_data = dev;
M
Matthew Wilcox 已提交
1683

1684 1685
		if (blk_mq_alloc_tag_set(&dev->tagset))
			return 0;
1686
		dev->ctrl.tagset = &dev->tagset;
1687
	}
1688
	queue_work(nvme_workq, &dev->scan_work);
K
Keith Busch 已提交
1689
	return 0;
M
Matthew Wilcox 已提交
1690 1691
}

1692 1693
static int nvme_dev_map(struct nvme_dev *dev)
{
K
Keith Busch 已提交
1694
	u64 cap;
1695
	int bars, result = -ENOMEM;
1696
	struct pci_dev *pdev = to_pci_dev(dev->dev);
1697 1698 1699 1700 1701 1702 1703

	if (pci_enable_device_mem(pdev))
		return result;

	dev->entry[0].vector = pdev->irq;
	pci_set_master(pdev);
	bars = pci_select_bars(pdev, IORESOURCE_MEM);
1704 1705 1706
	if (!bars)
		goto disable_pci;

1707 1708 1709
	if (pci_request_selected_regions(pdev, bars, "nvme"))
		goto disable_pci;

1710 1711
	if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
	    dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
1712
		goto disable;
1713 1714 1715 1716

	dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
	if (!dev->bar)
		goto disable;
1717

1718
	if (readl(dev->bar + NVME_REG_CSTS) == -1) {
K
Keith Busch 已提交
1719 1720 1721
		result = -ENODEV;
		goto unmap;
	}
1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732

	/*
	 * Some devices don't advertse INTx interrupts, pre-enable a single
	 * MSIX vec for setup. We'll adjust this later.
	 */
	if (!pdev->irq) {
		result = pci_enable_msix(pdev, dev->entry, 1);
		if (result < 0)
			goto unmap;
	}

1733 1734
	cap = lo_hi_readq(dev->bar + NVME_REG_CAP);

K
Keith Busch 已提交
1735 1736
	dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
	dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
1737
	dev->dbs = dev->bar + 4096;
1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749

	/*
	 * Temporary fix for the Apple controller found in the MacBook8,1 and
	 * some MacBook7,1 to avoid controller resets and data loss.
	 */
	if (pdev->vendor == PCI_VENDOR_ID_APPLE && pdev->device == 0x2001) {
		dev->q_depth = 2;
		dev_warn(dev->dev, "detected Apple NVMe controller, set "
			"queue depth=%u to work around controller resets\n",
			dev->q_depth);
	}

1750
	if (readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 2))
1751
		dev->cmb = nvme_map_cmb(dev);
1752

K
Keith Busch 已提交
1753 1754
	pci_enable_pcie_error_reporting(pdev);
	pci_save_state(pdev);
1755 1756
	return 0;

K
Keith Busch 已提交
1757 1758 1759
 unmap:
	iounmap(dev->bar);
	dev->bar = NULL;
1760 1761 1762 1763 1764 1765 1766 1767 1768
 disable:
	pci_release_regions(pdev);
 disable_pci:
	pci_disable_device(pdev);
	return result;
}

static void nvme_dev_unmap(struct nvme_dev *dev)
{
1769 1770 1771 1772 1773 1774
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	if (pdev->msi_enabled)
		pci_disable_msi(pdev);
	else if (pdev->msix_enabled)
		pci_disable_msix(pdev);
1775 1776 1777 1778

	if (dev->bar) {
		iounmap(dev->bar);
		dev->bar = NULL;
1779
		pci_release_regions(pdev);
1780 1781
	}

K
Keith Busch 已提交
1782 1783
	if (pci_is_enabled(pdev)) {
		pci_disable_pcie_error_reporting(pdev);
1784
		pci_disable_device(pdev);
K
Keith Busch 已提交
1785 1786 1787
	}
}

1788
static int nvme_dev_list_add(struct nvme_dev *dev)
K
Keith Busch 已提交
1789
{
1790
	bool start_thread = false;
K
Keith Busch 已提交
1791

1792 1793 1794 1795
	spin_lock(&dev_list_lock);
	if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
		start_thread = true;
		nvme_thread = NULL;
K
Keith Busch 已提交
1796
	}
1797 1798
	list_add(&dev->node, &dev_list);
	spin_unlock(&dev_list_lock);
K
Keith Busch 已提交
1799

1800 1801 1802 1803 1804
	if (start_thread) {
		nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
		wake_up_all(&nvme_kthread_wait);
	} else
		wait_event_killable(nvme_kthread_wait, nvme_thread);
K
Keith Busch 已提交
1805

1806 1807 1808 1809
	if (IS_ERR_OR_NULL(nvme_thread))
		return nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;

	return 0;
K
Keith Busch 已提交
1810 1811
}

1812 1813 1814 1815 1816 1817 1818 1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831
/*
* Remove the node from the device list and check
* for whether or not we need to stop the nvme_thread.
*/
static void nvme_dev_list_remove(struct nvme_dev *dev)
{
	struct task_struct *tmp = NULL;

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) {
		tmp = nvme_thread;
		nvme_thread = NULL;
	}
	spin_unlock(&dev_list_lock);

	if (tmp)
		kthread_stop(tmp);
}

1832
static void nvme_dev_disable(struct nvme_dev *dev, bool shutdown)
M
Matthew Wilcox 已提交
1833
{
1834
	int i;
1835
	u32 csts = -1;
1836

1837
	nvme_dev_list_remove(dev);
1838

1839
	mutex_lock(&dev->shutdown_lock);
1840
	if (dev->bar) {
1841
		nvme_stop_queues(&dev->ctrl);
1842
		csts = readl(dev->bar + NVME_REG_CSTS);
1843
	}
1844
	if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
K
Keith Busch 已提交
1845
		for (i = dev->queue_count - 1; i >= 0; i--) {
M
Matias Bjørling 已提交
1846
			struct nvme_queue *nvmeq = dev->queues[i];
K
Keith Busch 已提交
1847 1848 1849 1850
			nvme_suspend_queue(nvmeq);
		}
	} else {
		nvme_disable_io_queues(dev);
1851
		nvme_disable_admin_queue(dev, shutdown);
K
Keith Busch 已提交
1852
	}
1853
	nvme_dev_unmap(dev);
1854 1855 1856

	for (i = dev->queue_count - 1; i >= 0; i--)
		nvme_clear_queue(dev->queues[i]);
1857
	mutex_unlock(&dev->shutdown_lock);
M
Matthew Wilcox 已提交
1858 1859
}

M
Matthew Wilcox 已提交
1860 1861
static int nvme_setup_prp_pools(struct nvme_dev *dev)
{
1862
	dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
M
Matthew Wilcox 已提交
1863 1864 1865 1866
						PAGE_SIZE, PAGE_SIZE, 0);
	if (!dev->prp_page_pool)
		return -ENOMEM;

1867
	/* Optimisation for I/Os between 4k and 128k */
1868
	dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
1869 1870 1871 1872 1873
						256, 256, 0);
	if (!dev->prp_small_pool) {
		dma_pool_destroy(dev->prp_page_pool);
		return -ENOMEM;
	}
M
Matthew Wilcox 已提交
1874 1875 1876 1877 1878 1879
	return 0;
}

static void nvme_release_prp_pools(struct nvme_dev *dev)
{
	dma_pool_destroy(dev->prp_page_pool);
1880
	dma_pool_destroy(dev->prp_small_pool);
M
Matthew Wilcox 已提交
1881 1882
}

1883
static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
1884
{
1885
	struct nvme_dev *dev = to_nvme_dev(ctrl);
1886

1887
	put_device(dev->dev);
1888 1889
	if (dev->tagset.tags)
		blk_mq_free_tag_set(&dev->tagset);
1890 1891
	if (dev->ctrl.admin_q)
		blk_put_queue(dev->ctrl.admin_q);
1892 1893 1894 1895 1896
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
}

1897
static void nvme_reset_work(struct work_struct *work)
1898
{
1899
	struct nvme_dev *dev = container_of(work, struct nvme_dev, reset_work);
1900
	int result;
1901

1902 1903
	if (WARN_ON(test_bit(NVME_CTRL_RESETTING, &dev->flags)))
		goto out;
1904

1905 1906 1907 1908 1909
	/*
	 * If we're called to reset a live controller first shut it down before
	 * moving on.
	 */
	if (dev->bar)
1910
		nvme_dev_disable(dev, false);
1911

1912
	set_bit(NVME_CTRL_RESETTING, &dev->flags);
1913 1914 1915

	result = nvme_dev_map(dev);
	if (result)
1916
		goto out;
1917 1918 1919 1920 1921

	result = nvme_configure_admin_queue(dev);
	if (result)
		goto unmap;

M
Matias Bjørling 已提交
1922
	nvme_init_queue(dev->queues[0], 0);
K
Keith Busch 已提交
1923 1924 1925
	result = nvme_alloc_admin_tags(dev);
	if (result)
		goto disable;
1926

1927 1928 1929 1930
	result = nvme_init_identify(&dev->ctrl);
	if (result)
		goto free_tags;

1931
	result = nvme_setup_io_queues(dev);
1932
	if (result)
K
Keith Busch 已提交
1933
		goto free_tags;
1934

1935
	dev->ctrl.event_limit = NVME_NR_AEN_COMMANDS;
1936

1937 1938 1939
	result = nvme_dev_list_add(dev);
	if (result)
		goto remove;
1940

1941 1942 1943 1944
	/*
	 * Keep the controller around but remove all namespaces if we don't have
	 * any working I/O queue.
	 */
1945 1946
	if (dev->online_queues < 2) {
		dev_warn(dev->dev, "IO queues not created\n");
1947
		nvme_remove_namespaces(&dev->ctrl);
1948
	} else {
1949
		nvme_start_queues(&dev->ctrl);
1950 1951 1952
		nvme_dev_add(dev);
	}

1953
	clear_bit(NVME_CTRL_RESETTING, &dev->flags);
1954
	return;
1955

1956 1957
 remove:
	nvme_dev_list_remove(dev);
K
Keith Busch 已提交
1958 1959
 free_tags:
	nvme_dev_remove_admin(dev);
1960 1961
	blk_put_queue(dev->ctrl.admin_q);
	dev->ctrl.admin_q = NULL;
1962
	dev->queues[0]->tags = NULL;
1963
 disable:
1964
	nvme_disable_admin_queue(dev, false);
1965 1966
 unmap:
	nvme_dev_unmap(dev);
1967
 out:
1968
	nvme_remove_dead_ctrl(dev);
1969 1970
}

1971
static void nvme_remove_dead_ctrl_work(struct work_struct *work)
K
Keith Busch 已提交
1972
{
1973
	struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
1974
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
1975 1976

	if (pci_get_drvdata(pdev))
1977
		pci_stop_and_remove_bus_device_locked(pdev);
1978
	nvme_put_ctrl(&dev->ctrl);
K
Keith Busch 已提交
1979 1980
}

1981
static void nvme_remove_dead_ctrl(struct nvme_dev *dev)
1982
{
1983
	dev_warn(dev->dev, "Removing after probe failure\n");
1984
	kref_get(&dev->ctrl.kref);
1985
	if (!schedule_work(&dev->remove_work))
1986
		nvme_put_ctrl(&dev->ctrl);
1987 1988
}

1989
static int nvme_reset(struct nvme_dev *dev)
K
Keith Busch 已提交
1990
{
1991
	if (!dev->ctrl.admin_q || blk_queue_dying(dev->ctrl.admin_q))
1992
		return -ENODEV;
1993

C
Christoph Hellwig 已提交
1994 1995
	if (!queue_work(nvme_workq, &dev->reset_work))
		return -EBUSY;
1996

C
Christoph Hellwig 已提交
1997 1998
	flush_work(&dev->reset_work);
	return 0;
K
Keith Busch 已提交
1999 2000
}

2001
static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
T
Tejun Heo 已提交
2002
{
2003
	*val = readl(to_nvme_dev(ctrl)->bar + off);
2004
	return 0;
T
Tejun Heo 已提交
2005 2006
}

2007
static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
2008
{
2009 2010 2011
	writel(val, to_nvme_dev(ctrl)->bar + off);
	return 0;
}
2012

2013 2014 2015 2016
static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
{
	*val = readq(to_nvme_dev(ctrl)->bar + off);
	return 0;
2017 2018
}

2019
static bool nvme_pci_io_incapable(struct nvme_ctrl *ctrl)
2020
{
2021
	struct nvme_dev *dev = to_nvme_dev(ctrl);
2022

2023 2024
	return !dev->bar || dev->online_queues < 2;
}
2025

2026 2027 2028
static int nvme_pci_reset_ctrl(struct nvme_ctrl *ctrl)
{
	return nvme_reset(to_nvme_dev(ctrl));
2029
}
2030

2031 2032
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
	.reg_read32		= nvme_pci_reg_read32,
2033
	.reg_write32		= nvme_pci_reg_write32,
2034
	.reg_read64		= nvme_pci_reg_read64,
2035
	.io_incapable		= nvme_pci_io_incapable,
2036
	.reset_ctrl		= nvme_pci_reset_ctrl,
2037
	.free_ctrl		= nvme_pci_free_ctrl,
2038
};
2039

2040
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
M
Matthew Wilcox 已提交
2041
{
M
Matias Bjørling 已提交
2042
	int node, result = -ENOMEM;
M
Matthew Wilcox 已提交
2043 2044
	struct nvme_dev *dev;

M
Matias Bjørling 已提交
2045 2046 2047 2048 2049
	node = dev_to_node(&pdev->dev);
	if (node == NUMA_NO_NODE)
		set_dev_node(&pdev->dev, 0);

	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2050 2051
	if (!dev)
		return -ENOMEM;
M
Matias Bjørling 已提交
2052 2053
	dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2054 2055
	if (!dev->entry)
		goto free;
M
Matias Bjørling 已提交
2056 2057
	dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2058 2059 2060
	if (!dev->queues)
		goto free;

2061
	dev->dev = get_device(&pdev->dev);
K
Keith Busch 已提交
2062
	pci_set_drvdata(pdev, dev);
2063

2064 2065 2066
	INIT_LIST_HEAD(&dev->node);
	INIT_WORK(&dev->scan_work, nvme_dev_scan);
	INIT_WORK(&dev->reset_work, nvme_reset_work);
2067
	INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
2068
	mutex_init(&dev->shutdown_lock);
K
Keith Busch 已提交
2069
	init_completion(&dev->ioq_wait);
M
Matthew Wilcox 已提交
2070

M
Matthew Wilcox 已提交
2071 2072
	result = nvme_setup_prp_pools(dev);
	if (result)
K
Keith Busch 已提交
2073
		goto put_pci;
2074

2075 2076
	result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
			id->driver_data);
2077
	if (result)
2078
		goto release_pools;
2079

2080
	queue_work(nvme_workq, &dev->reset_work);
M
Matthew Wilcox 已提交
2081 2082
	return 0;

2083
 release_pools:
M
Matthew Wilcox 已提交
2084
	nvme_release_prp_pools(dev);
K
Keith Busch 已提交
2085
 put_pci:
2086
	put_device(dev->dev);
M
Matthew Wilcox 已提交
2087 2088 2089 2090 2091 2092 2093
 free:
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
	return result;
}

2094 2095
static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
{
K
Keith Busch 已提交
2096
	struct nvme_dev *dev = pci_get_drvdata(pdev);
2097

K
Keith Busch 已提交
2098
	if (prepare)
2099
		nvme_dev_disable(dev, false);
K
Keith Busch 已提交
2100
	else
2101
		queue_work(nvme_workq, &dev->reset_work);
2102 2103
}

2104 2105 2106
static void nvme_shutdown(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
2107
	nvme_dev_disable(dev, true);
2108 2109
}

2110
static void nvme_remove(struct pci_dev *pdev)
M
Matthew Wilcox 已提交
2111 2112
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
K
Keith Busch 已提交
2113 2114 2115 2116 2117 2118 2119

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	spin_unlock(&dev_list_lock);

	pci_set_drvdata(pdev, NULL);
	flush_work(&dev->reset_work);
K
Keith Busch 已提交
2120
	flush_work(&dev->scan_work);
2121
	nvme_remove_namespaces(&dev->ctrl);
2122
	nvme_uninit_ctrl(&dev->ctrl);
2123
	nvme_dev_disable(dev, true);
M
Matias Bjørling 已提交
2124
	nvme_dev_remove_admin(dev);
2125
	nvme_free_queues(dev, 0);
2126
	nvme_release_cmb(dev);
K
Keith Busch 已提交
2127
	nvme_release_prp_pools(dev);
2128
	nvme_put_ctrl(&dev->ctrl);
M
Matthew Wilcox 已提交
2129 2130
}

2131
#ifdef CONFIG_PM_SLEEP
2132 2133 2134 2135 2136
static int nvme_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

2137
	nvme_dev_disable(ndev, true);
2138 2139 2140 2141 2142 2143 2144 2145
	return 0;
}

static int nvme_resume(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

2146
	queue_work(nvme_workq, &ndev->reset_work);
K
Keith Busch 已提交
2147
	return 0;
2148
}
2149
#endif
2150 2151

static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
M
Matthew Wilcox 已提交
2152

K
Keith Busch 已提交
2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165 2166 2167
static pci_ers_result_t nvme_error_detected(struct pci_dev *pdev,
						pci_channel_state_t state)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);

	/*
	 * A frozen channel requires a reset. When detected, this method will
	 * shutdown the controller to quiesce. The controller will be restarted
	 * after the slot reset through driver's slot_reset callback.
	 */
	dev_warn(&pdev->dev, "error detected: state:%d\n", state);
	switch (state) {
	case pci_channel_io_normal:
		return PCI_ERS_RESULT_CAN_RECOVER;
	case pci_channel_io_frozen:
2168
		nvme_dev_disable(dev, false);
K
Keith Busch 已提交
2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190
		return PCI_ERS_RESULT_NEED_RESET;
	case pci_channel_io_perm_failure:
		return PCI_ERS_RESULT_DISCONNECT;
	}
	return PCI_ERS_RESULT_NEED_RESET;
}

static pci_ers_result_t nvme_slot_reset(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);

	dev_info(&pdev->dev, "restart after slot reset\n");
	pci_restore_state(pdev);
	queue_work(nvme_workq, &dev->reset_work);
	return PCI_ERS_RESULT_RECOVERED;
}

static void nvme_error_resume(struct pci_dev *pdev)
{
	pci_cleanup_aer_uncorrect_error_status(pdev);
}

2191
static const struct pci_error_handlers nvme_err_handler = {
M
Matthew Wilcox 已提交
2192 2193 2194
	.error_detected	= nvme_error_detected,
	.slot_reset	= nvme_slot_reset,
	.resume		= nvme_error_resume,
2195
	.reset_notify	= nvme_reset_notify,
M
Matthew Wilcox 已提交
2196 2197 2198 2199 2200
};

/* Move to pci_ids.h later */
#define PCI_CLASS_STORAGE_EXPRESS	0x010802

2201
static const struct pci_device_id nvme_id_table[] = {
2202 2203
	{ PCI_VDEVICE(INTEL, 0x0953),
		.driver_data = NVME_QUIRK_STRIPE_SIZE, },
2204 2205
	{ PCI_VDEVICE(INTEL, 0x5845),	/* Qemu emulated controller */
		.driver_data = NVME_QUIRK_IDENTIFY_CNS, },
M
Matthew Wilcox 已提交
2206
	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2207
	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
M
Matthew Wilcox 已提交
2208 2209 2210 2211 2212 2213 2214 2215
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, nvme_id_table);

static struct pci_driver nvme_driver = {
	.name		= "nvme",
	.id_table	= nvme_id_table,
	.probe		= nvme_probe,
2216
	.remove		= nvme_remove,
2217
	.shutdown	= nvme_shutdown,
2218 2219 2220
	.driver		= {
		.pm	= &nvme_dev_pm_ops,
	},
M
Matthew Wilcox 已提交
2221 2222 2223 2224 2225
	.err_handler	= &nvme_err_handler,
};

static int __init nvme_init(void)
{
2226
	int result;
2227

2228
	init_waitqueue_head(&nvme_kthread_wait);
M
Matthew Wilcox 已提交
2229

2230
	nvme_workq = alloc_workqueue("nvme", WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
K
Keith Busch 已提交
2231
	if (!nvme_workq)
2232
		return -ENOMEM;
K
Keith Busch 已提交
2233

2234
	result = nvme_core_init();
2235
	if (result < 0)
K
Keith Busch 已提交
2236
		goto kill_workq;
2237

2238 2239
	result = pci_register_driver(&nvme_driver);
	if (result)
2240
		goto core_exit;
2241
	return 0;
M
Matthew Wilcox 已提交
2242

2243
 core_exit:
2244
	nvme_core_exit();
K
Keith Busch 已提交
2245 2246
 kill_workq:
	destroy_workqueue(nvme_workq);
M
Matthew Wilcox 已提交
2247 2248 2249 2250 2251 2252
	return result;
}

static void __exit nvme_exit(void)
{
	pci_unregister_driver(&nvme_driver);
2253
	nvme_core_exit();
K
Keith Busch 已提交
2254
	destroy_workqueue(nvme_workq);
2255
	BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
2256
	_nvme_check_size();
M
Matthew Wilcox 已提交
2257 2258 2259 2260
}

MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
MODULE_LICENSE("GPL");
2261
MODULE_VERSION("1.0");
M
Matthew Wilcox 已提交
2262 2263
module_init(nvme_init);
module_exit(nvme_exit);