pci.c 58.4 KB
Newer Older
M
Matthew Wilcox 已提交
1 2
/*
 * NVM Express device driver
3
 * Copyright (c) 2011-2014, Intel Corporation.
M
Matthew Wilcox 已提交
4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

15
#include <linux/bitops.h>
M
Matthew Wilcox 已提交
16
#include <linux/blkdev.h>
M
Matias Bjørling 已提交
17
#include <linux/blk-mq.h>
K
Keith Busch 已提交
18
#include <linux/cpu.h>
19
#include <linux/delay.h>
M
Matthew Wilcox 已提交
20 21 22
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/genhd.h>
K
Keith Busch 已提交
23
#include <linux/hdreg.h>
24
#include <linux/idr.h>
M
Matthew Wilcox 已提交
25 26 27 28
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kdev_t.h>
29
#include <linux/kthread.h>
M
Matthew Wilcox 已提交
30 31 32 33
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
34
#include <linux/mutex.h>
M
Matthew Wilcox 已提交
35
#include <linux/pci.h>
36
#include <linux/poison.h>
37
#include <linux/ptrace.h>
M
Matthew Wilcox 已提交
38 39
#include <linux/sched.h>
#include <linux/slab.h>
K
Keith Busch 已提交
40
#include <linux/t10-pi.h>
M
Matthew Wilcox 已提交
41
#include <linux/types.h>
42
#include <linux/io-64-nonatomic-lo-hi.h>
K
Keith Busch 已提交
43
#include <asm/unaligned.h>
44

45 46
#include "nvme.h"

47
#define NVME_Q_DEPTH		1024
J
Jens Axboe 已提交
48
#define NVME_AQ_DEPTH		256
M
Matthew Wilcox 已提交
49 50
#define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))
#define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))
51 52 53 54 55 56 57
		
/*
 * We handle AEN commands ourselves and don't even let the
 * block layer know about them.
 */
#define NVME_NR_AEN_COMMANDS	1
#define NVME_AQ_BLKMQ_DEPTH	(NVME_AQ_DEPTH - NVME_NR_AEN_COMMANDS)
58

59
unsigned char admin_timeout = 60;
60 61
module_param(admin_timeout, byte, 0644);
MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
M
Matthew Wilcox 已提交
62

63 64
unsigned char nvme_io_timeout = 30;
module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
65
MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
M
Matthew Wilcox 已提交
66

67
unsigned char shutdown_timeout = 5;
68 69 70
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");

71 72 73
static int use_threaded_interrupts;
module_param(use_threaded_interrupts, int, 0);

74 75 76 77
static bool use_cmb_sqes = true;
module_param(use_cmb_sqes, bool, 0644);
MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");

78 79
static LIST_HEAD(dev_list);
static struct task_struct *nvme_thread;
K
Keith Busch 已提交
80
static struct workqueue_struct *nvme_workq;
81
static wait_queue_head_t nvme_kthread_wait;
82

83 84 85
struct nvme_dev;
struct nvme_queue;

86
static int nvme_reset(struct nvme_dev *dev);
J
Jens Axboe 已提交
87
static void nvme_process_cq(struct nvme_queue *nvmeq);
88
static void nvme_remove_dead_ctrl(struct nvme_dev *dev);
89
static void nvme_dev_shutdown(struct nvme_dev *dev);
90

K
Keith Busch 已提交
91 92 93 94 95 96
struct async_cmd_info {
	struct kthread_work work;
	struct kthread_worker *worker;
	int status;
	void *ctx;
};
97

98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118
/*
 * Represents an NVM Express device.  Each nvme_dev is a PCI function.
 */
struct nvme_dev {
	struct list_head node;
	struct nvme_queue **queues;
	struct blk_mq_tag_set tagset;
	struct blk_mq_tag_set admin_tagset;
	u32 __iomem *dbs;
	struct device *dev;
	struct dma_pool *prp_page_pool;
	struct dma_pool *prp_small_pool;
	unsigned queue_count;
	unsigned online_queues;
	unsigned max_qid;
	int q_depth;
	u32 db_stride;
	struct msix_entry *entry;
	void __iomem *bar;
	struct work_struct reset_work;
	struct work_struct scan_work;
119
	struct work_struct remove_work;
120
	struct mutex shutdown_lock;
121 122 123 124 125
	bool subsystem;
	void __iomem *cmb;
	dma_addr_t cmb_dma_addr;
	u64 cmb_size;
	u32 cmbsz;
126 127
	unsigned long flags;
#define NVME_CTRL_RESETTING    0
128 129 130 131 132 133 134 135 136

	struct nvme_ctrl ctrl;
};

static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
{
	return container_of(ctrl, struct nvme_dev, ctrl);
}

M
Matthew Wilcox 已提交
137 138 139 140 141 142
/*
 * An NVM Express queue.  Each device has at least two (one for admin
 * commands and one for I/O commands).
 */
struct nvme_queue {
	struct device *q_dmadev;
M
Matthew Wilcox 已提交
143
	struct nvme_dev *dev;
144
	char irqname[24];	/* nvme4294967295-65535\0 */
M
Matthew Wilcox 已提交
145 146
	spinlock_t q_lock;
	struct nvme_command *sq_cmds;
147
	struct nvme_command __iomem *sq_cmds_io;
M
Matthew Wilcox 已提交
148
	volatile struct nvme_completion *cqes;
149
	struct blk_mq_tags **tags;
M
Matthew Wilcox 已提交
150 151 152 153
	dma_addr_t sq_dma_addr;
	dma_addr_t cq_dma_addr;
	u32 __iomem *q_db;
	u16 q_depth;
J
Jens Axboe 已提交
154
	s16 cq_vector;
M
Matthew Wilcox 已提交
155 156 157
	u16 sq_head;
	u16 sq_tail;
	u16 cq_head;
K
Keith Busch 已提交
158
	u16 qid;
159 160
	u8 cq_phase;
	u8 cqe_seen;
K
Keith Busch 已提交
161
	struct async_cmd_info cmdinfo;
M
Matthew Wilcox 已提交
162 163
};

164 165 166 167 168 169 170 171 172 173 174 175 176
/*
 * The nvme_iod describes the data in an I/O, including the list of PRP
 * entries.  You can't see it in this data structure because C doesn't let
 * me express that.  Use nvme_alloc_iod to ensure there's enough space
 * allocated to store the PRP list.
 */
struct nvme_iod {
	unsigned long private;	/* For the use of the submitter of the I/O */
	int npages;		/* In the PRP list. 0 means small pool in use */
	int offset;		/* Of PRP list */
	int nents;		/* Used in scatterlist */
	int length;		/* Of data, in bytes */
	dma_addr_t first_dma;
177
	struct scatterlist meta_sg; /* metadata requires single contiguous buffer */
178 179 180
	struct scatterlist sg[0];
};

M
Matthew Wilcox 已提交
181 182 183 184 185 186 187 188 189 190
/*
 * Check we didin't inadvertently grow the command struct
 */
static inline void _nvme_check_size(void)
{
	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
191
	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
K
Keith Busch 已提交
192
	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
M
Matthew Wilcox 已提交
193 194 195 196
	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
K
Keith Busch 已提交
197
	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
M
Matthew Wilcox 已提交
198 199
}

200
struct nvme_cmd_info {
K
Keith Busch 已提交
201
	int aborted;
M
Matias Bjørling 已提交
202
	struct nvme_queue *nvmeq;
203 204
	struct nvme_iod *iod;
	struct nvme_iod __iod;
205 206
};

207 208 209 210
/*
 * Max size of iod being embedded in the request payload
 */
#define NVME_INT_PAGES		2
211
#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->ctrl.page_size)
C
Chong Yuan 已提交
212
#define NVME_INT_MASK		0x01
213 214 215 216 217 218 219 220

/*
 * Will slightly overestimate the number of pages needed.  This is OK
 * as it only leads to a small amount of wasted memory for the lifetime of
 * the I/O.
 */
static int nvme_npages(unsigned size, struct nvme_dev *dev)
{
221 222
	unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size,
				      dev->ctrl.page_size);
223 224 225 226 227 228 229 230 231 232 233 234 235 236
	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}

static unsigned int nvme_cmd_size(struct nvme_dev *dev)
{
	unsigned int ret = sizeof(struct nvme_cmd_info);

	ret += sizeof(struct nvme_iod);
	ret += sizeof(__le64 *) * nvme_npages(NVME_INT_BYTES(dev), dev);
	ret += sizeof(struct scatterlist) * NVME_INT_PAGES;

	return ret;
}

M
Matias Bjørling 已提交
237 238
static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
				unsigned int hctx_idx)
239
{
M
Matias Bjørling 已提交
240 241 242
	struct nvme_dev *dev = data;
	struct nvme_queue *nvmeq = dev->queues[0];

243 244 245 246
	WARN_ON(hctx_idx != 0);
	WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
	WARN_ON(nvmeq->tags);

M
Matias Bjørling 已提交
247
	hctx->driver_data = nvmeq;
248
	nvmeq->tags = &dev->admin_tagset.tags[0];
M
Matias Bjørling 已提交
249
	return 0;
250 251
}

252 253 254 255 256 257 258
static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	nvmeq->tags = NULL;
}

M
Matias Bjørling 已提交
259 260 261
static int nvme_admin_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
262
{
M
Matias Bjørling 已提交
263 264 265 266 267 268 269
	struct nvme_dev *dev = data;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = dev->queues[0];

	BUG_ON(!nvmeq);
	cmd->nvmeq = nvmeq;
	return 0;
270 271
}

M
Matias Bjørling 已提交
272 273
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			  unsigned int hctx_idx)
M
Matthew Wilcox 已提交
274
{
M
Matias Bjørling 已提交
275
	struct nvme_dev *dev = data;
276
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
M
Matias Bjørling 已提交
277

278 279
	if (!nvmeq->tags)
		nvmeq->tags = &dev->tagset.tags[hctx_idx];
M
Matthew Wilcox 已提交
280

281
	WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
M
Matias Bjørling 已提交
282 283
	hctx->driver_data = nvmeq;
	return 0;
M
Matthew Wilcox 已提交
284 285
}

M
Matias Bjørling 已提交
286 287 288
static int nvme_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
M
Matthew Wilcox 已提交
289
{
M
Matias Bjørling 已提交
290 291 292 293 294 295 296 297 298
	struct nvme_dev *dev = data;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];

	BUG_ON(!nvmeq);
	cmd->nvmeq = nvmeq;
	return 0;
}

299 300 301 302 303 304 305 306 307 308
static void *iod_get_private(struct nvme_iod *iod)
{
	return (void *) (iod->private & ~0x1UL);
}

/*
 * If bit 0 is set, the iod is embedded in the request payload.
 */
static bool iod_should_kfree(struct nvme_iod *iod)
{
C
Chong Yuan 已提交
309
	return (iod->private & NVME_INT_MASK) == 0;
310 311
}

312 313
static void nvme_complete_async_event(struct nvme_dev *dev,
		struct nvme_completion *cqe)
314
{
315 316
	u16 status = le16_to_cpu(cqe->status) >> 1;
	u32 result = le32_to_cpu(cqe->result);
M
Matias Bjørling 已提交
317 318

	if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
319
		++dev->ctrl.event_limit;
K
Keith Busch 已提交
320 321 322 323 324
	if (status != NVME_SC_SUCCESS)
		return;

	switch (result & 0xff07) {
	case NVME_AER_NOTICE_NS_CHANGED:
325 326
		dev_info(dev->dev, "rescanning\n");
		queue_work(nvme_workq, &dev->scan_work);
K
Keith Busch 已提交
327
	default:
328
		dev_warn(dev->dev, "async event result %08x\n", result);
K
Keith Busch 已提交
329
	}
M
Matthew Wilcox 已提交
330 331 332
}

/**
333
 * __nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
M
Matthew Wilcox 已提交
334 335 336 337 338
 * @nvmeq: The queue to use
 * @cmd: The command to send
 *
 * Safe to use from interrupt context
 */
339 340
static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
						struct nvme_command *cmd)
M
Matthew Wilcox 已提交
341
{
M
Matias Bjørling 已提交
342 343
	u16 tail = nvmeq->sq_tail;

344 345 346 347 348
	if (nvmeq->sq_cmds_io)
		memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd));
	else
		memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));

M
Matthew Wilcox 已提交
349 350
	if (++tail == nvmeq->q_depth)
		tail = 0;
351
	writel(tail, nvmeq->q_db);
M
Matthew Wilcox 已提交
352 353 354
	nvmeq->sq_tail = tail;
}

355
static __le64 **iod_list(struct nvme_iod *iod)
356
{
357
	return ((void *)iod) + iod->offset;
358 359
}

360 361
static inline void iod_init(struct nvme_iod *iod, unsigned nbytes,
			    unsigned nseg, unsigned long private)
362
{
363 364 365 366 367
	iod->private = private;
	iod->offset = offsetof(struct nvme_iod, sg[nseg]);
	iod->npages = -1;
	iod->length = nbytes;
	iod->nents = 0;
368
}
M
Matthew Wilcox 已提交
369

370
static struct nvme_iod *
371 372
__nvme_alloc_iod(unsigned nseg, unsigned bytes, struct nvme_dev *dev,
		 unsigned long priv, gfp_t gfp)
M
Matthew Wilcox 已提交
373
{
374
	struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
375
				sizeof(__le64 *) * nvme_npages(bytes, dev) +
376 377
				sizeof(struct scatterlist) * nseg, gfp);

378 379
	if (iod)
		iod_init(iod, bytes, nseg, priv);
380 381

	return iod;
M
Matthew Wilcox 已提交
382 383
}

384 385 386 387 388 389 390 391 392 393 394
static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
			               gfp_t gfp)
{
	unsigned size = !(rq->cmd_flags & REQ_DISCARD) ? blk_rq_bytes(rq) :
                                                sizeof(struct nvme_dsm_range);
	struct nvme_iod *iod;

	if (rq->nr_phys_segments <= NVME_INT_PAGES &&
	    size <= NVME_INT_BYTES(dev)) {
		struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);

395
		iod = &cmd->__iod;
396
		iod_init(iod, size, rq->nr_phys_segments,
C
Chong Yuan 已提交
397
				(unsigned long) rq | NVME_INT_MASK);
398 399 400 401 402 403 404
		return iod;
	}

	return __nvme_alloc_iod(rq->nr_phys_segments, size, dev,
				(unsigned long) rq, gfp);
}

405
static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
M
Matthew Wilcox 已提交
406
{
407
	const int last_prp = dev->ctrl.page_size / 8 - 1;
408 409 410 411 412 413 414 415 416 417 418 419
	int i;
	__le64 **list = iod_list(iod);
	dma_addr_t prp_dma = iod->first_dma;

	if (iod->npages == 0)
		dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
	for (i = 0; i < iod->npages; i++) {
		__le64 *prp_list = list[i];
		dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
		dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
		prp_dma = next_prp_dma;
	}
420 421 422

	if (iod_should_kfree(iod))
		kfree(iod);
M
Matthew Wilcox 已提交
423 424
}

425
#ifdef CONFIG_BLK_DEV_INTEGRITY
K
Keith Busch 已提交
426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == v)
		pi->ref_tag = cpu_to_be32(p);
}

static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == p)
		pi->ref_tag = cpu_to_be32(v);
}

/**
 * nvme_dif_remap - remaps ref tags to bip seed and physical lba
 *
 * The virtual start sector is the one that was originally submitted by the
 * block layer.	Due to partitioning, MD/DM cloning, etc. the actual physical
 * start sector may be different. Remap protection information to match the
 * physical LBA on writes, and back to the original seed on reads.
 *
 * Type 0 and 3 do not have a ref tag, so no remapping required.
 */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
	struct nvme_ns *ns = req->rq_disk->private_data;
	struct bio_integrity_payload *bip;
	struct t10_pi_tuple *pi;
	void *p, *pmap;
	u32 i, nlb, ts, phys, virt;

	if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
		return;

	bip = bio_integrity(req->bio);
	if (!bip)
		return;

	pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;

	p = pmap;
	virt = bip_get_seed(bip);
	phys = nvme_block_nr(ns, blk_rq_pos(req));
	nlb = (blk_rq_bytes(req) >> ns->lba_shift);
470
	ts = ns->disk->queue->integrity.tuple_size;
K
Keith Busch 已提交
471 472 473 474 475 476 477 478

	for (i = 0; i < nlb; i++, virt++, phys++) {
		pi = (struct t10_pi_tuple *)p;
		dif_swap(phys, virt, pi);
		p += ts;
	}
	kunmap_atomic(pmap);
}
479 480 481 482 483 484 485 486 487 488 489 490 491
#else /* CONFIG_BLK_DEV_INTEGRITY */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
}
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
#endif

492 493
static bool nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
		int total_len)
M
Matthew Wilcox 已提交
494
{
495
	struct dma_pool *pool;
496 497
	int length = total_len;
	struct scatterlist *sg = iod->sg;
M
Matthew Wilcox 已提交
498 499
	int dma_len = sg_dma_len(sg);
	u64 dma_addr = sg_dma_address(sg);
500
	u32 page_size = dev->ctrl.page_size;
501
	int offset = dma_addr & (page_size - 1);
502
	__le64 *prp_list;
503
	__le64 **list = iod_list(iod);
504
	dma_addr_t prp_dma;
505
	int nprps, i;
M
Matthew Wilcox 已提交
506

507
	length -= (page_size - offset);
M
Matthew Wilcox 已提交
508
	if (length <= 0)
509
		return true;
M
Matthew Wilcox 已提交
510

511
	dma_len -= (page_size - offset);
M
Matthew Wilcox 已提交
512
	if (dma_len) {
513
		dma_addr += (page_size - offset);
M
Matthew Wilcox 已提交
514 515 516 517 518 519
	} else {
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
	}

520
	if (length <= page_size) {
521
		iod->first_dma = dma_addr;
522
		return true;
523 524
	}

525
	nprps = DIV_ROUND_UP(length, page_size);
526 527
	if (nprps <= (256 / 8)) {
		pool = dev->prp_small_pool;
528
		iod->npages = 0;
529 530
	} else {
		pool = dev->prp_page_pool;
531
		iod->npages = 1;
532 533
	}

534
	prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
535
	if (!prp_list) {
536
		iod->first_dma = dma_addr;
537
		iod->npages = -1;
538
		return false;
539
	}
540 541
	list[0] = prp_list;
	iod->first_dma = prp_dma;
542 543
	i = 0;
	for (;;) {
544
		if (i == page_size >> 3) {
545
			__le64 *old_prp_list = prp_list;
546
			prp_list = dma_pool_alloc(pool, GFP_ATOMIC, &prp_dma);
547
			if (!prp_list)
548
				return false;
549
			list[iod->npages++] = prp_list;
550 551 552
			prp_list[0] = old_prp_list[i - 1];
			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
			i = 1;
553 554
		}
		prp_list[i++] = cpu_to_le64(dma_addr);
555 556 557
		dma_len -= page_size;
		dma_addr += page_size;
		length -= page_size;
558 559 560 561 562 563 564 565
		if (length <= 0)
			break;
		if (dma_len > 0)
			continue;
		BUG_ON(dma_len < 0);
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
M
Matthew Wilcox 已提交
566 567
	}

568
	return true;
M
Matthew Wilcox 已提交
569 570
}

C
Christoph Hellwig 已提交
571 572
static int nvme_map_data(struct nvme_dev *dev, struct nvme_iod *iod,
		struct nvme_command *cmnd)
573
{
C
Christoph Hellwig 已提交
574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596
	struct request *req = iod_get_private(iod);
	struct request_queue *q = req->q;
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;
	int ret = BLK_MQ_RQ_QUEUE_ERROR;

	sg_init_table(iod->sg, req->nr_phys_segments);
	iod->nents = blk_rq_map_sg(q, req, iod->sg);
	if (!iod->nents)
		goto out;

	ret = BLK_MQ_RQ_QUEUE_BUSY;
	if (!dma_map_sg(dev->dev, iod->sg, iod->nents, dma_dir))
		goto out;

	if (!nvme_setup_prps(dev, iod, blk_rq_bytes(req)))
		goto out_unmap;

	ret = BLK_MQ_RQ_QUEUE_ERROR;
	if (blk_integrity_rq(req)) {
		if (blk_rq_count_integrity_sg(q, req->bio) != 1)
			goto out_unmap;

597 598
		sg_init_table(&iod->meta_sg, 1);
		if (blk_rq_map_integrity_sg(q, req->bio, &iod->meta_sg) != 1)
C
Christoph Hellwig 已提交
599
			goto out_unmap;
600

C
Christoph Hellwig 已提交
601 602 603
		if (rq_data_dir(req))
			nvme_dif_remap(req, nvme_dif_prep);

604
		if (!dma_map_sg(dev->dev, &iod->meta_sg, 1, dma_dir))
C
Christoph Hellwig 已提交
605
			goto out_unmap;
606 607
	}

C
Christoph Hellwig 已提交
608 609 610
	cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
	cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);
	if (blk_integrity_rq(req))
611
		cmnd->rw.metadata = cpu_to_le64(sg_dma_address(&iod->meta_sg));
C
Christoph Hellwig 已提交
612 613 614 615 616 617
	return BLK_MQ_RQ_QUEUE_OK;

out_unmap:
	dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
out:
	return ret;
618 619
}

620 621 622 623 624 625 626 627 628 629 630
static void nvme_unmap_data(struct nvme_dev *dev, struct nvme_iod *iod)
{
	struct request *req = iod_get_private(iod);
	enum dma_data_direction dma_dir = rq_data_dir(req) ?
			DMA_TO_DEVICE : DMA_FROM_DEVICE;

	if (iod->nents) {
		dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
		if (blk_integrity_rq(req)) {
			if (!rq_data_dir(req))
				nvme_dif_remap(req, nvme_dif_complete);
631
			dma_unmap_sg(dev->dev, &iod->meta_sg, 1, dma_dir);
632 633 634 635 636 637
		}
	}

	nvme_free_iod(dev, iod);
}

M
Matias Bjørling 已提交
638 639 640 641 642
/*
 * We reuse the small pool to allocate the 16-byte range here as it is not
 * worth having a special pool for these or additional cases to handle freeing
 * the iod.
 */
C
Christoph Hellwig 已提交
643 644
static int nvme_setup_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
		struct nvme_iod *iod, struct nvme_command *cmnd)
645
{
C
Christoph Hellwig 已提交
646 647 648 649 650 651 652 653 654
	struct request *req = iod_get_private(iod);
	struct nvme_dsm_range *range;

	range = dma_pool_alloc(nvmeq->dev->prp_small_pool, GFP_ATOMIC,
						&iod->first_dma);
	if (!range)
		return BLK_MQ_RQ_QUEUE_BUSY;
	iod_list(iod)[0] = (__le64 *)range;
	iod->npages = 0;
655 656

	range->cattr = cpu_to_le32(0);
M
Matias Bjørling 已提交
657 658
	range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
	range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
659

C
Christoph Hellwig 已提交
660 661 662 663 664 665 666
	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->dsm.opcode = nvme_cmd_dsm;
	cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
	cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
	cmnd->dsm.nr = 0;
	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
	return BLK_MQ_RQ_QUEUE_OK;
667 668
}

669 670 671
/*
 * NOTE: ns is NULL when called on the admin queue.
 */
M
Matias Bjørling 已提交
672 673
static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
			 const struct blk_mq_queue_data *bd)
674
{
M
Matias Bjørling 已提交
675 676
	struct nvme_ns *ns = hctx->queue->queuedata;
	struct nvme_queue *nvmeq = hctx->driver_data;
677
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
678 679
	struct request *req = bd->rq;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
680
	struct nvme_iod *iod;
C
Christoph Hellwig 已提交
681 682
	struct nvme_command cmnd;
	int ret = BLK_MQ_RQ_QUEUE_OK;
683

K
Keith Busch 已提交
684 685 686 687 688
	/*
	 * If formated with metadata, require the block layer provide a buffer
	 * unless this namespace is formated such that the metadata can be
	 * stripped/generated by the controller with PRACT=1.
	 */
689
	if (ns && ns->ms && !blk_integrity_rq(req)) {
690 691
		if (!(ns->pi_type && ns->ms == 8) &&
					req->cmd_type != REQ_TYPE_DRV_PRIV) {
692
			blk_mq_end_request(req, -EFAULT);
K
Keith Busch 已提交
693 694 695 696
			return BLK_MQ_RQ_QUEUE_OK;
		}
	}

697
	iod = nvme_alloc_iod(req, dev, GFP_ATOMIC);
698
	if (!iod)
699
		return BLK_MQ_RQ_QUEUE_BUSY;
M
Matias Bjørling 已提交
700 701

	if (req->cmd_flags & REQ_DISCARD) {
C
Christoph Hellwig 已提交
702 703 704 705 706 707 708 709
		ret = nvme_setup_discard(nvmeq, ns, iod, &cmnd);
	} else {
		if (req->cmd_type == REQ_TYPE_DRV_PRIV)
			memcpy(&cmnd, req->cmd, sizeof(cmnd));
		else if (req->cmd_flags & REQ_FLUSH)
			nvme_setup_flush(ns, &cmnd);
		else
			nvme_setup_rw(ns, req, &cmnd);
M
Matias Bjørling 已提交
710

C
Christoph Hellwig 已提交
711 712
		if (req->nr_phys_segments)
			ret = nvme_map_data(dev, iod, &cmnd);
713
	}
714

C
Christoph Hellwig 已提交
715 716 717
	if (ret)
		goto out;

718 719
	cmd->iod = iod;
	cmd->aborted = 0;
C
Christoph Hellwig 已提交
720
	cmnd.common.command_id = req->tag;
721
	blk_mq_start_request(req);
M
Matias Bjørling 已提交
722

C
Christoph Hellwig 已提交
723 724
	spin_lock_irq(&nvmeq->q_lock);
	__nvme_submit_cmd(nvmeq, &cmnd);
M
Matias Bjørling 已提交
725 726 727
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
	return BLK_MQ_RQ_QUEUE_OK;
C
Christoph Hellwig 已提交
728
out:
729
	nvme_free_iod(dev, iod);
C
Christoph Hellwig 已提交
730
	return ret;
M
Matthew Wilcox 已提交
731 732
}

733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761
static void nvme_complete_rq(struct request *req)
{
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_dev *dev = cmd->nvmeq->dev;
	int error = 0;

	nvme_unmap_data(dev, cmd->iod);

	if (unlikely(req->errors)) {
		if (nvme_req_needs_retry(req, req->errors)) {
			nvme_requeue_req(req);
			return;
		}

		if (req->cmd_type == REQ_TYPE_DRV_PRIV)
			error = req->errors;
		else
			error = nvme_error_status(req->errors);
	}

	if (unlikely(cmd->aborted)) {
		dev_warn(dev->dev,
			"completing aborted command with status: %04x\n",
			req->errors);
	}

	blk_mq_end_request(req, error);
}

J
Jens Axboe 已提交
762
static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
M
Matthew Wilcox 已提交
763
{
M
Matthew Wilcox 已提交
764
	u16 head, phase;
M
Matthew Wilcox 已提交
765 766

	head = nvmeq->cq_head;
M
Matthew Wilcox 已提交
767
	phase = nvmeq->cq_phase;
M
Matthew Wilcox 已提交
768 769 770

	for (;;) {
		struct nvme_completion cqe = nvmeq->cqes[head];
771
		u16 status = le16_to_cpu(cqe.status);
772
		struct request *req;
773 774

		if ((status & 1) != phase)
M
Matthew Wilcox 已提交
775 776 777 778
			break;
		nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
		if (++head == nvmeq->q_depth) {
			head = 0;
M
Matthew Wilcox 已提交
779
			phase = !phase;
M
Matthew Wilcox 已提交
780
		}
781

J
Jens Axboe 已提交
782 783
		if (tag && *tag == cqe.command_id)
			*tag = -1;
784

785 786 787 788 789 790 791
		if (unlikely(cqe.command_id >= nvmeq->q_depth)) {
			dev_warn(nvmeq->q_dmadev,
				"invalid id %d completed on queue %d\n",
				cqe.command_id, le16_to_cpu(cqe.sq_id));
			continue;
		}

792 793 794 795 796 797 798 799 800 801 802 803
		/*
		 * AEN requests are special as they don't time out and can
		 * survive any kind of queue freeze and often don't respond to
		 * aborts.  We don't even bother to allocate a struct request
		 * for them but rather special case them here.
		 */
		if (unlikely(nvmeq->qid == 0 &&
				cqe.command_id >= NVME_AQ_BLKMQ_DEPTH)) {
			nvme_complete_async_event(nvmeq->dev, &cqe);
			continue;
		}

804 805 806 807 808 809 810
		req = blk_mq_tag_to_rq(*nvmeq->tags, cqe.command_id);
		if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
			u32 result = le32_to_cpu(cqe.result);
			req->special = (void *)(uintptr_t)result;
		}
		blk_mq_complete_request(req, status >> 1);

M
Matthew Wilcox 已提交
811 812 813 814 815 816 817 818
	}

	/* If the controller ignores the cq head doorbell and continuously
	 * writes to the queue, it is theoretically possible to wrap around
	 * the queue twice and mistakenly return IRQ_NONE.  Linux only
	 * requires that 0.1% of your interrupts are handled, so this isn't
	 * a big problem.
	 */
M
Matthew Wilcox 已提交
819
	if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
J
Jens Axboe 已提交
820
		return;
M
Matthew Wilcox 已提交
821

822 823
	if (likely(nvmeq->cq_vector >= 0))
		writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
M
Matthew Wilcox 已提交
824
	nvmeq->cq_head = head;
M
Matthew Wilcox 已提交
825
	nvmeq->cq_phase = phase;
M
Matthew Wilcox 已提交
826

827
	nvmeq->cqe_seen = 1;
J
Jens Axboe 已提交
828 829 830 831 832
}

static void nvme_process_cq(struct nvme_queue *nvmeq)
{
	__nvme_process_cq(nvmeq, NULL);
M
Matthew Wilcox 已提交
833 834 835
}

static irqreturn_t nvme_irq(int irq, void *data)
836 837 838 839
{
	irqreturn_t result;
	struct nvme_queue *nvmeq = data;
	spin_lock(&nvmeq->q_lock);
840 841 842
	nvme_process_cq(nvmeq);
	result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
	nvmeq->cqe_seen = 0;
843 844 845 846 847 848 849 850 851 852 853 854 855
	spin_unlock(&nvmeq->q_lock);
	return result;
}

static irqreturn_t nvme_irq_check(int irq, void *data)
{
	struct nvme_queue *nvmeq = data;
	struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
	if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
		return IRQ_NONE;
	return IRQ_WAKE_THREAD;
}

J
Jens Axboe 已提交
856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872
static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	if ((le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
	    nvmeq->cq_phase) {
		spin_lock_irq(&nvmeq->q_lock);
		__nvme_process_cq(nvmeq, &tag);
		spin_unlock_irq(&nvmeq->q_lock);

		if (tag == -1)
			return 1;
	}

	return 0;
}

873
static void nvme_submit_async_event(struct nvme_dev *dev)
M
Matias Bjørling 已提交
874 875 876 877 878
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_async_event;
879
	c.common.command_id = NVME_AQ_BLKMQ_DEPTH + --dev->ctrl.event_limit;
M
Matias Bjørling 已提交
880

881
	__nvme_submit_cmd(dev->queues[0], &c);
M
Matias Bjørling 已提交
882 883
}

884
static void async_cmd_info_endio(struct request *req, int error)
K
Keith Busch 已提交
885
{
886
	struct async_cmd_info *cmdinfo = req->end_io_data;
M
Matias Bjørling 已提交
887

888 889 890
	cmdinfo->status = req->errors;
	queue_kthread_work(cmdinfo->worker, &cmdinfo->work);
	blk_mq_free_request(req);
K
Keith Busch 已提交
891 892
}

M
Matthew Wilcox 已提交
893 894 895 896 897 898 899 900
static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(id);

901
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
902 903 904 905 906 907 908 909
}

static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;

910 911 912 913
	/*
	 * Note: we (ab)use the fact the the prp fields survive if no data
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
914 915 916 917 918 919 920 921
	memset(&c, 0, sizeof(c));
	c.create_cq.opcode = nvme_admin_create_cq;
	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
	c.create_cq.cqid = cpu_to_le16(qid);
	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_cq.cq_flags = cpu_to_le16(flags);
	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);

922
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
923 924 925 926 927 928 929 930
}

static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;

931 932 933 934
	/*
	 * Note: we (ab)use the fact the the prp fields survive if no data
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
935 936 937 938 939 940 941 942
	memset(&c, 0, sizeof(c));
	c.create_sq.opcode = nvme_admin_create_sq;
	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
	c.create_sq.sqid = cpu_to_le16(qid);
	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_sq.sq_flags = cpu_to_le16(flags);
	c.create_sq.cqid = cpu_to_le16(qid);

943
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
944 945 946 947 948 949 950 951 952 953 954 955
}

static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
}

static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
}

956 957 958 959 960 961 962 963 964 965 966 967 968
static void abort_endio(struct request *req, int error)
{
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd->nvmeq;
	u32 result = (u32)(uintptr_t)req->special;
	u16 status = req->errors;

	dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
	atomic_inc(&nvmeq->dev->ctrl.abort_limit);

	blk_mq_free_request(req);
}

969
static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
K
Keith Busch 已提交
970
{
M
Matias Bjørling 已提交
971 972
	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd_rq->nvmeq;
K
Keith Busch 已提交
973
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
974 975
	struct request *abort_req;
	struct nvme_command cmd;
K
Keith Busch 已提交
976

977
	/*
978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995
	 * Shutdown immediately if controller times out while starting. The
	 * reset work will see the pci device disabled when it gets the forced
	 * cancellation error. All outstanding requests are completed on
	 * shutdown, so we return BLK_EH_HANDLED.
	 */
	if (test_bit(NVME_CTRL_RESETTING, &dev->flags)) {
		dev_warn(dev->dev,
			 "I/O %d QID %d timeout, disable controller\n",
			 req->tag, nvmeq->qid);
		nvme_dev_shutdown(dev);
		req->errors = NVME_SC_CANCELLED;
		return BLK_EH_HANDLED;
	}

	/*
 	 * Shutdown the controller immediately and schedule a reset if the
 	 * command was already aborted once before and still hasn't been
 	 * returned to the driver, or if this is the admin queue.
996
	 */
M
Matias Bjørling 已提交
997
	if (!nvmeq->qid || cmd_rq->aborted) {
998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009
		dev_warn(dev->dev,
			 "I/O %d QID %d timeout, reset controller\n",
			 req->tag, nvmeq->qid);
		nvme_dev_shutdown(dev);
		queue_work(nvme_workq, &dev->reset_work);

		/*
		 * Mark the request as handled, since the inline shutdown
		 * forces all outstanding requests to complete.
		 */
		req->errors = NVME_SC_CANCELLED;
		return BLK_EH_HANDLED;
K
Keith Busch 已提交
1010 1011
	}

1012
	cmd_rq->aborted = 1;
K
Keith Busch 已提交
1013

1014
	if (atomic_dec_return(&dev->ctrl.abort_limit) < 0) {
1015
		atomic_inc(&dev->ctrl.abort_limit);
1016
		return BLK_EH_RESET_TIMER;
1017
	}
K
Keith Busch 已提交
1018 1019 1020

	memset(&cmd, 0, sizeof(cmd));
	cmd.abort.opcode = nvme_admin_abort_cmd;
M
Matias Bjørling 已提交
1021
	cmd.abort.cid = req->tag;
K
Keith Busch 已提交
1022 1023
	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);

1024 1025
	dev_warn(nvmeq->q_dmadev, "I/O %d QID %d timeout, aborting\n",
				 req->tag, nvmeq->qid);
1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036

	abort_req = nvme_alloc_request(dev->ctrl.admin_q, &cmd,
			BLK_MQ_REQ_NOWAIT);
	if (IS_ERR(abort_req)) {
		atomic_inc(&dev->ctrl.abort_limit);
		return BLK_EH_RESET_TIMER;
	}

	abort_req->timeout = ADMIN_TIMEOUT;
	abort_req->end_io_data = NULL;
	blk_execute_rq_nowait(abort_req->q, NULL, abort_req, 0, abort_endio);
1037 1038 1039 1040 1041 1042 1043

	/*
	 * The aborted req will be completed on receiving the abort req.
	 * We enable the timer again. If hit twice, it'll cause a device reset,
	 * as the device then is in a faulty state.
	 */
	return BLK_EH_RESET_TIMER;
K
Keith Busch 已提交
1044 1045
}

1046
static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved)
1047
{
M
Matias Bjørling 已提交
1048
	struct nvme_queue *nvmeq = data;
1049
	int status;
K
Keith Busch 已提交
1050 1051 1052

	if (!blk_mq_request_started(req))
		return;
1053

1054 1055
	dev_warn(nvmeq->q_dmadev,
		 "Cancelling I/O %d QID %d\n", req->tag, nvmeq->qid);
M
Matias Bjørling 已提交
1056

1057
	status = NVME_SC_CANCELLED;
K
Keith Busch 已提交
1058
	if (blk_queue_dying(req->q))
1059 1060
		status |= NVME_SC_DNR;
	blk_mq_complete_request(req, status);
1061 1062
}

M
Matias Bjørling 已提交
1063 1064
static void nvme_free_queue(struct nvme_queue *nvmeq)
{
1065 1066
	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1067 1068
	if (nvmeq->sq_cmds)
		dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1069 1070 1071 1072
					nvmeq->sq_cmds, nvmeq->sq_dma_addr);
	kfree(nvmeq);
}

1073
static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1074 1075 1076
{
	int i;

1077
	for (i = dev->queue_count - 1; i >= lowest; i--) {
M
Matias Bjørling 已提交
1078
		struct nvme_queue *nvmeq = dev->queues[i];
1079
		dev->queue_count--;
M
Matias Bjørling 已提交
1080
		dev->queues[i] = NULL;
K
Keith Busch 已提交
1081
		nvme_free_queue(nvmeq);
1082
	}
1083 1084
}

K
Keith Busch 已提交
1085 1086 1087 1088 1089
/**
 * nvme_suspend_queue - put queue into suspended state
 * @nvmeq - queue to suspend
 */
static int nvme_suspend_queue(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
1090
{
K
Keith Busch 已提交
1091
	int vector;
M
Matthew Wilcox 已提交
1092

1093
	spin_lock_irq(&nvmeq->q_lock);
K
Keith Busch 已提交
1094 1095 1096 1097 1098
	if (nvmeq->cq_vector == -1) {
		spin_unlock_irq(&nvmeq->q_lock);
		return 1;
	}
	vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
K
Keith Busch 已提交
1099
	nvmeq->dev->online_queues--;
K
Keith Busch 已提交
1100
	nvmeq->cq_vector = -1;
1101 1102
	spin_unlock_irq(&nvmeq->q_lock);

1103 1104
	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
		blk_mq_freeze_queue_start(nvmeq->dev->ctrl.admin_q);
1105

M
Matthew Wilcox 已提交
1106 1107
	irq_set_affinity_hint(vector, NULL);
	free_irq(vector, nvmeq);
M
Matthew Wilcox 已提交
1108

K
Keith Busch 已提交
1109 1110
	return 0;
}
M
Matthew Wilcox 已提交
1111

K
Keith Busch 已提交
1112 1113
static void nvme_clear_queue(struct nvme_queue *nvmeq)
{
1114
	spin_lock_irq(&nvmeq->q_lock);
1115 1116
	if (nvmeq->tags && *nvmeq->tags)
		blk_mq_all_tag_busy_iter(*nvmeq->tags, nvme_cancel_queue_ios, nvmeq);
1117
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1118 1119
}

K
Keith Busch 已提交
1120 1121
static void nvme_disable_queue(struct nvme_dev *dev, int qid)
{
M
Matias Bjørling 已提交
1122
	struct nvme_queue *nvmeq = dev->queues[qid];
K
Keith Busch 已提交
1123 1124 1125 1126 1127 1128

	if (!nvmeq)
		return;
	if (nvme_suspend_queue(nvmeq))
		return;

K
Keith Busch 已提交
1129 1130
	/* Don't tell the adapter to delete the admin queue.
	 * Don't tell a removed adapter to delete IO queues. */
1131
	if (qid && readl(dev->bar + NVME_REG_CSTS) != -1) {
M
Matthew Wilcox 已提交
1132 1133 1134
		adapter_delete_sq(dev, qid);
		adapter_delete_cq(dev, qid);
	}
1135 1136 1137 1138

	spin_lock_irq(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1139 1140
}

1141 1142 1143 1144
static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
				int entry_size)
{
	int q_depth = dev->q_depth;
1145 1146
	unsigned q_size_aligned = roundup(q_depth * entry_size,
					  dev->ctrl.page_size);
1147 1148

	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1149
		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
1150
		mem_per_q = round_down(mem_per_q, dev->ctrl.page_size);
1151
		q_depth = div_u64(mem_per_q, entry_size);
1152 1153 1154 1155 1156 1157 1158 1159 1160 1161 1162 1163 1164 1165 1166 1167 1168

		/*
		 * Ensure the reduced q_depth is above some threshold where it
		 * would be better to map queues in system memory with the
		 * original depth
		 */
		if (q_depth < 64)
			return -ENOMEM;
	}

	return q_depth;
}

static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
				int qid, int depth)
{
	if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
1169 1170
		unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth),
						      dev->ctrl.page_size);
1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181 1182
		nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
		nvmeq->sq_cmds_io = dev->cmb + offset;
	} else {
		nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
					&nvmeq->sq_dma_addr, GFP_KERNEL);
		if (!nvmeq->sq_cmds)
			return -ENOMEM;
	}

	return 0;
}

M
Matthew Wilcox 已提交
1183
static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
K
Keith Busch 已提交
1184
							int depth)
M
Matthew Wilcox 已提交
1185
{
M
Matias Bjørling 已提交
1186
	struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
M
Matthew Wilcox 已提交
1187 1188 1189
	if (!nvmeq)
		return NULL;

1190
	nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
J
Joe Perches 已提交
1191
					  &nvmeq->cq_dma_addr, GFP_KERNEL);
M
Matthew Wilcox 已提交
1192 1193 1194
	if (!nvmeq->cqes)
		goto free_nvmeq;

1195
	if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
M
Matthew Wilcox 已提交
1196 1197
		goto free_cqdma;

1198
	nvmeq->q_dmadev = dev->dev;
M
Matthew Wilcox 已提交
1199
	nvmeq->dev = dev;
1200
	snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
1201
			dev->ctrl.instance, qid);
M
Matthew Wilcox 已提交
1202 1203
	spin_lock_init(&nvmeq->q_lock);
	nvmeq->cq_head = 0;
M
Matthew Wilcox 已提交
1204
	nvmeq->cq_phase = 1;
1205
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
M
Matthew Wilcox 已提交
1206
	nvmeq->q_depth = depth;
K
Keith Busch 已提交
1207
	nvmeq->qid = qid;
1208
	nvmeq->cq_vector = -1;
M
Matias Bjørling 已提交
1209
	dev->queues[qid] = nvmeq;
M
Matthew Wilcox 已提交
1210

1211 1212 1213 1214
	/* make sure queue descriptor is set before queue count, for kthread */
	mb();
	dev->queue_count++;

M
Matthew Wilcox 已提交
1215 1216 1217
	return nvmeq;

 free_cqdma:
1218
	dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
M
Matthew Wilcox 已提交
1219 1220 1221 1222 1223 1224
							nvmeq->cq_dma_addr);
 free_nvmeq:
	kfree(nvmeq);
	return NULL;
}

1225 1226 1227
static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
							const char *name)
{
1228 1229
	if (use_threaded_interrupts)
		return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1230
					nvme_irq_check, nvme_irq, IRQF_SHARED,
1231
					name, nvmeq);
1232
	return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1233
				IRQF_SHARED, name, nvmeq);
1234 1235
}

1236
static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
M
Matthew Wilcox 已提交
1237
{
1238
	struct nvme_dev *dev = nvmeq->dev;
M
Matthew Wilcox 已提交
1239

1240
	spin_lock_irq(&nvmeq->q_lock);
1241 1242 1243
	nvmeq->sq_tail = 0;
	nvmeq->cq_head = 0;
	nvmeq->cq_phase = 1;
1244
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1245
	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
K
Keith Busch 已提交
1246
	dev->online_queues++;
1247
	spin_unlock_irq(&nvmeq->q_lock);
1248 1249 1250 1251 1252 1253
}

static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
{
	struct nvme_dev *dev = nvmeq->dev;
	int result;
1254

K
Keith Busch 已提交
1255
	nvmeq->cq_vector = qid - 1;
M
Matthew Wilcox 已提交
1256 1257
	result = adapter_alloc_cq(dev, qid, nvmeq);
	if (result < 0)
1258
		return result;
M
Matthew Wilcox 已提交
1259 1260 1261 1262 1263

	result = adapter_alloc_sq(dev, qid, nvmeq);
	if (result < 0)
		goto release_cq;

1264
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
M
Matthew Wilcox 已提交
1265 1266 1267
	if (result < 0)
		goto release_sq;

1268 1269
	nvme_init_queue(nvmeq, qid);
	return result;
M
Matthew Wilcox 已提交
1270 1271 1272 1273 1274

 release_sq:
	adapter_delete_sq(dev, qid);
 release_cq:
	adapter_delete_cq(dev, qid);
1275
	return result;
M
Matthew Wilcox 已提交
1276 1277
}

M
Matias Bjørling 已提交
1278
static struct blk_mq_ops nvme_mq_admin_ops = {
1279
	.queue_rq	= nvme_queue_rq,
1280
	.complete	= nvme_complete_rq,
M
Matias Bjørling 已提交
1281 1282
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_admin_init_hctx,
1283
	.exit_hctx      = nvme_admin_exit_hctx,
M
Matias Bjørling 已提交
1284 1285 1286 1287 1288 1289
	.init_request	= nvme_admin_init_request,
	.timeout	= nvme_timeout,
};

static struct blk_mq_ops nvme_mq_ops = {
	.queue_rq	= nvme_queue_rq,
1290
	.complete	= nvme_complete_rq,
M
Matias Bjørling 已提交
1291 1292 1293 1294
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_init_hctx,
	.init_request	= nvme_init_request,
	.timeout	= nvme_timeout,
J
Jens Axboe 已提交
1295
	.poll		= nvme_poll,
M
Matias Bjørling 已提交
1296 1297
};

1298 1299
static void nvme_dev_remove_admin(struct nvme_dev *dev)
{
1300 1301
	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
		blk_cleanup_queue(dev->ctrl.admin_q);
1302 1303 1304 1305
		blk_mq_free_tag_set(&dev->admin_tagset);
	}
}

M
Matias Bjørling 已提交
1306 1307
static int nvme_alloc_admin_tags(struct nvme_dev *dev)
{
1308
	if (!dev->ctrl.admin_q) {
M
Matias Bjørling 已提交
1309 1310
		dev->admin_tagset.ops = &nvme_mq_admin_ops;
		dev->admin_tagset.nr_hw_queues = 1;
1311
		dev->admin_tagset.queue_depth = NVME_AQ_BLKMQ_DEPTH;
M
Matias Bjørling 已提交
1312
		dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1313
		dev->admin_tagset.numa_node = dev_to_node(dev->dev);
1314
		dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
M
Matias Bjørling 已提交
1315 1316 1317 1318 1319
		dev->admin_tagset.driver_data = dev;

		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
			return -ENOMEM;

1320 1321
		dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
		if (IS_ERR(dev->ctrl.admin_q)) {
M
Matias Bjørling 已提交
1322 1323 1324
			blk_mq_free_tag_set(&dev->admin_tagset);
			return -ENOMEM;
		}
1325
		if (!blk_get_queue(dev->ctrl.admin_q)) {
1326
			nvme_dev_remove_admin(dev);
1327
			dev->ctrl.admin_q = NULL;
1328 1329
			return -ENODEV;
		}
K
Keith Busch 已提交
1330
	} else
1331
		blk_mq_unfreeze_queue(dev->ctrl.admin_q);
M
Matias Bjørling 已提交
1332 1333 1334 1335

	return 0;
}

1336
static int nvme_configure_admin_queue(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1337
{
1338
	int result;
M
Matthew Wilcox 已提交
1339
	u32 aqa;
1340
	u64 cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
M
Matthew Wilcox 已提交
1341 1342
	struct nvme_queue *nvmeq;

1343
	dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1) ?
1344 1345
						NVME_CAP_NSSRC(cap) : 0;

1346 1347 1348
	if (dev->subsystem &&
	    (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
		writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1349

1350
	result = nvme_disable_ctrl(&dev->ctrl, cap);
1351 1352
	if (result < 0)
		return result;
M
Matthew Wilcox 已提交
1353

M
Matias Bjørling 已提交
1354
	nvmeq = dev->queues[0];
1355
	if (!nvmeq) {
K
Keith Busch 已提交
1356
		nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1357 1358 1359
		if (!nvmeq)
			return -ENOMEM;
	}
M
Matthew Wilcox 已提交
1360 1361 1362 1363

	aqa = nvmeq->q_depth - 1;
	aqa |= aqa << 16;

1364 1365 1366
	writel(aqa, dev->bar + NVME_REG_AQA);
	lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
	lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
M
Matthew Wilcox 已提交
1367

1368
	result = nvme_enable_ctrl(&dev->ctrl, cap);
1369
	if (result)
M
Matias Bjørling 已提交
1370 1371
		goto free_nvmeq;

K
Keith Busch 已提交
1372
	nvmeq->cq_vector = 0;
1373
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
1374 1375
	if (result) {
		nvmeq->cq_vector = -1;
K
Keith Busch 已提交
1376
		goto free_nvmeq;
1377
	}
1378

M
Matthew Wilcox 已提交
1379
	return result;
M
Matias Bjørling 已提交
1380 1381 1382 1383

 free_nvmeq:
	nvme_free_queues(dev, 0);
	return result;
M
Matthew Wilcox 已提交
1384 1385
}

1386 1387
static int nvme_kthread(void *data)
{
1388
	struct nvme_dev *dev, *next;
1389 1390

	while (!kthread_should_stop()) {
1391
		set_current_state(TASK_INTERRUPTIBLE);
1392
		spin_lock(&dev_list_lock);
1393
		list_for_each_entry_safe(dev, next, &dev_list, node) {
1394
			int i;
1395
			u32 csts = readl(dev->bar + NVME_REG_CSTS);
1396

C
Christoph Hellwig 已提交
1397 1398 1399 1400 1401 1402
			/*
			 * Skip controllers currently under reset.
			 */
			if (work_pending(&dev->reset_work) || work_busy(&dev->reset_work))
				continue;

1403 1404
			if ((dev->subsystem && (csts & NVME_CSTS_NSSRO)) ||
							csts & NVME_CSTS_CFS) {
C
Christoph Hellwig 已提交
1405
				if (queue_work(nvme_workq, &dev->reset_work)) {
1406 1407
					dev_warn(dev->dev,
						"Failed status: %x, reset controller\n",
1408
						readl(dev->bar + NVME_REG_CSTS));
1409
				}
1410 1411
				continue;
			}
1412
			for (i = 0; i < dev->queue_count; i++) {
M
Matias Bjørling 已提交
1413
				struct nvme_queue *nvmeq = dev->queues[i];
1414 1415
				if (!nvmeq)
					continue;
1416
				spin_lock_irq(&nvmeq->q_lock);
1417
				nvme_process_cq(nvmeq);
K
Keith Busch 已提交
1418

1419 1420
				while (i == 0 && dev->ctrl.event_limit > 0)
					nvme_submit_async_event(dev);
1421 1422 1423 1424
				spin_unlock_irq(&nvmeq->q_lock);
			}
		}
		spin_unlock(&dev_list_lock);
1425
		schedule_timeout(round_jiffies_relative(HZ));
1426 1427 1428 1429
	}
	return 0;
}

1430
static int nvme_create_io_queues(struct nvme_dev *dev)
K
Keith Busch 已提交
1431
{
M
Matias Bjørling 已提交
1432
	unsigned i;
1433
	int ret = 0;
K
Keith Busch 已提交
1434

1435 1436 1437
	for (i = dev->queue_count; i <= dev->max_qid; i++) {
		if (!nvme_alloc_queue(dev, i, dev->q_depth)) {
			ret = -ENOMEM;
K
Keith Busch 已提交
1438
			break;
1439 1440
		}
	}
K
Keith Busch 已提交
1441

1442 1443 1444
	for (i = dev->online_queues; i <= dev->queue_count - 1; i++) {
		ret = nvme_create_queue(dev->queues[i], i);
		if (ret) {
1445
			nvme_free_queues(dev, i);
K
Keith Busch 已提交
1446
			break;
1447
		}
1448 1449 1450 1451 1452 1453 1454 1455 1456
	}

	/*
	 * Ignore failing Create SQ/CQ commands, we can continue with less
	 * than the desired aount of queues, and even a controller without
	 * I/O queues an still be used to issue admin commands.  This might
	 * be useful to upgrade a buggy firmware for example.
	 */
	return ret >= 0 ? 0 : ret;
K
Keith Busch 已提交
1457 1458
}

1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470
static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
{
	u64 szu, size, offset;
	u32 cmbloc;
	resource_size_t bar_size;
	struct pci_dev *pdev = to_pci_dev(dev->dev);
	void __iomem *cmb;
	dma_addr_t dma_addr;

	if (!use_cmb_sqes)
		return NULL;

1471
	dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
1472 1473 1474
	if (!(NVME_CMB_SZ(dev->cmbsz)))
		return NULL;

1475
	cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
1476 1477 1478 1479 1480 1481 1482 1483 1484 1485 1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510

	szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
	size = szu * NVME_CMB_SZ(dev->cmbsz);
	offset = szu * NVME_CMB_OFST(cmbloc);
	bar_size = pci_resource_len(pdev, NVME_CMB_BIR(cmbloc));

	if (offset > bar_size)
		return NULL;

	/*
	 * Controllers may support a CMB size larger than their BAR,
	 * for example, due to being behind a bridge. Reduce the CMB to
	 * the reported size of the BAR
	 */
	if (size > bar_size - offset)
		size = bar_size - offset;

	dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(cmbloc)) + offset;
	cmb = ioremap_wc(dma_addr, size);
	if (!cmb)
		return NULL;

	dev->cmb_dma_addr = dma_addr;
	dev->cmb_size = size;
	return cmb;
}

static inline void nvme_release_cmb(struct nvme_dev *dev)
{
	if (dev->cmb) {
		iounmap(dev->cmb);
		dev->cmb = NULL;
	}
}

K
Keith Busch 已提交
1511 1512
static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
{
1513
	return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
K
Keith Busch 已提交
1514 1515
}

1516
static int nvme_setup_io_queues(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1517
{
M
Matias Bjørling 已提交
1518
	struct nvme_queue *adminq = dev->queues[0];
1519
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
1520
	int result, i, vecs, nr_io_queues, size;
M
Matthew Wilcox 已提交
1521

K
Keith Busch 已提交
1522
	nr_io_queues = num_possible_cpus();
C
Christoph Hellwig 已提交
1523 1524
	result = nvme_set_queue_count(&dev->ctrl, &nr_io_queues);
	if (result < 0)
M
Matthew Wilcox 已提交
1525
		return result;
C
Christoph Hellwig 已提交
1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536

	/*
	 * Degraded controllers might return an error when setting the queue
	 * count.  We still want to be able to bring them online and offer
	 * access to the admin queue, as that might be only way to fix them up.
	 */
	if (result > 0) {
		dev_err(dev->dev, "Could not set queue count (%d)\n", result);
		nr_io_queues = 0;
		result = 0;
	}
M
Matthew Wilcox 已提交
1537

1538 1539 1540 1541 1542 1543 1544 1545 1546
	if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
		result = nvme_cmb_qdepth(dev, nr_io_queues,
				sizeof(struct nvme_command));
		if (result > 0)
			dev->q_depth = result;
		else
			nvme_release_cmb(dev);
	}

K
Keith Busch 已提交
1547 1548
	size = db_bar_size(dev, nr_io_queues);
	if (size > 8192) {
1549
		iounmap(dev->bar);
K
Keith Busch 已提交
1550 1551 1552 1553 1554 1555 1556 1557
		do {
			dev->bar = ioremap(pci_resource_start(pdev, 0), size);
			if (dev->bar)
				break;
			if (!--nr_io_queues)
				return -ENOMEM;
			size = db_bar_size(dev, nr_io_queues);
		} while (1);
1558
		dev->dbs = dev->bar + 4096;
1559
		adminq->q_db = dev->dbs;
1560 1561
	}

K
Keith Busch 已提交
1562
	/* Deregister the admin queue's interrupt */
1563
	free_irq(dev->entry[0].vector, adminq);
K
Keith Busch 已提交
1564

1565 1566 1567 1568 1569 1570 1571
	/*
	 * If we enable msix early due to not intx, disable it again before
	 * setting up the full range we need.
	 */
	if (!pdev->irq)
		pci_disable_msix(pdev);

1572
	for (i = 0; i < nr_io_queues; i++)
M
Matthew Wilcox 已提交
1573
		dev->entry[i].entry = i;
1574 1575 1576 1577 1578 1579 1580 1581
	vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
	if (vecs < 0) {
		vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
		if (vecs < 0) {
			vecs = 1;
		} else {
			for (i = 0; i < vecs; i++)
				dev->entry[i].vector = i + pdev->irq;
R
Ramachandra Rao Gajula 已提交
1582 1583 1584
		}
	}

1585 1586 1587 1588 1589 1590 1591
	/*
	 * Should investigate if there's a performance win from allocating
	 * more queues than interrupt vectors; it might allow the submission
	 * path to scale better, even if the receive path is limited by the
	 * number of interrupts.
	 */
	nr_io_queues = vecs;
K
Keith Busch 已提交
1592
	dev->max_qid = nr_io_queues;
1593

1594
	result = queue_request_irq(dev, adminq, adminq->irqname);
1595 1596
	if (result) {
		adminq->cq_vector = -1;
1597
		goto free_queues;
1598
	}
M
Matthew Wilcox 已提交
1599

1600
	/* Free previously allocated queues that are no longer usable */
K
Keith Busch 已提交
1601
	nvme_free_queues(dev, nr_io_queues + 1);
1602
	return nvme_create_io_queues(dev);
M
Matthew Wilcox 已提交
1603

1604
 free_queues:
1605
	nvme_free_queues(dev, 1);
1606
	return result;
M
Matthew Wilcox 已提交
1607 1608
}

1609 1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624
static void nvme_set_irq_hints(struct nvme_dev *dev)
{
	struct nvme_queue *nvmeq;
	int i;

	for (i = 0; i < dev->online_queues; i++) {
		nvmeq = dev->queues[i];

		if (!nvmeq->tags || !(*nvmeq->tags))
			continue;

		irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
					blk_mq_tags_cpumask(*nvmeq->tags));
	}
}

K
Keith Busch 已提交
1625 1626 1627 1628 1629 1630
static void nvme_dev_scan(struct work_struct *work)
{
	struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);

	if (!dev->tagset.tags)
		return;
1631
	nvme_scan_namespaces(&dev->ctrl);
1632
	nvme_set_irq_hints(dev);
K
Keith Busch 已提交
1633 1634
}

1635 1636 1637 1638 1639 1640
/*
 * Return: error value if an error occurred setting up the queues or calling
 * Identify Device.  0 if these succeeded, even if adding some of the
 * namespaces failed.  At the moment, these failures are silent.  TBD which
 * failures should be reported.
 */
1641
static int nvme_dev_add(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1642
{
1643
	if (!dev->ctrl.tagset) {
1644 1645 1646 1647 1648
		dev->tagset.ops = &nvme_mq_ops;
		dev->tagset.nr_hw_queues = dev->online_queues - 1;
		dev->tagset.timeout = NVME_IO_TIMEOUT;
		dev->tagset.numa_node = dev_to_node(dev->dev);
		dev->tagset.queue_depth =
M
Matias Bjørling 已提交
1649
				min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
1650 1651 1652
		dev->tagset.cmd_size = nvme_cmd_size(dev);
		dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
		dev->tagset.driver_data = dev;
M
Matthew Wilcox 已提交
1653

1654 1655
		if (blk_mq_alloc_tag_set(&dev->tagset))
			return 0;
1656
		dev->ctrl.tagset = &dev->tagset;
1657
	}
1658
	queue_work(nvme_workq, &dev->scan_work);
K
Keith Busch 已提交
1659
	return 0;
M
Matthew Wilcox 已提交
1660 1661
}

1662 1663
static int nvme_dev_map(struct nvme_dev *dev)
{
K
Keith Busch 已提交
1664
	u64 cap;
1665
	int bars, result = -ENOMEM;
1666
	struct pci_dev *pdev = to_pci_dev(dev->dev);
1667 1668 1669 1670 1671 1672 1673

	if (pci_enable_device_mem(pdev))
		return result;

	dev->entry[0].vector = pdev->irq;
	pci_set_master(pdev);
	bars = pci_select_bars(pdev, IORESOURCE_MEM);
1674 1675 1676
	if (!bars)
		goto disable_pci;

1677 1678 1679
	if (pci_request_selected_regions(pdev, bars, "nvme"))
		goto disable_pci;

1680 1681
	if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
	    dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
1682
		goto disable;
1683 1684 1685 1686

	dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
	if (!dev->bar)
		goto disable;
1687

1688
	if (readl(dev->bar + NVME_REG_CSTS) == -1) {
K
Keith Busch 已提交
1689 1690 1691
		result = -ENODEV;
		goto unmap;
	}
1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702

	/*
	 * Some devices don't advertse INTx interrupts, pre-enable a single
	 * MSIX vec for setup. We'll adjust this later.
	 */
	if (!pdev->irq) {
		result = pci_enable_msix(pdev, dev->entry, 1);
		if (result < 0)
			goto unmap;
	}

1703 1704
	cap = lo_hi_readq(dev->bar + NVME_REG_CAP);

K
Keith Busch 已提交
1705 1706
	dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
	dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
1707 1708
	dev->dbs = dev->bar + 4096;
	if (readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 2))
1709
		dev->cmb = nvme_map_cmb(dev);
1710 1711 1712

	return 0;

K
Keith Busch 已提交
1713 1714 1715
 unmap:
	iounmap(dev->bar);
	dev->bar = NULL;
1716 1717 1718 1719 1720 1721 1722 1723 1724
 disable:
	pci_release_regions(pdev);
 disable_pci:
	pci_disable_device(pdev);
	return result;
}

static void nvme_dev_unmap(struct nvme_dev *dev)
{
1725 1726 1727 1728 1729 1730
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	if (pdev->msi_enabled)
		pci_disable_msi(pdev);
	else if (pdev->msix_enabled)
		pci_disable_msix(pdev);
1731 1732 1733 1734

	if (dev->bar) {
		iounmap(dev->bar);
		dev->bar = NULL;
1735
		pci_release_regions(pdev);
1736 1737
	}

1738 1739
	if (pci_is_enabled(pdev))
		pci_disable_device(pdev);
1740 1741
}

K
Keith Busch 已提交
1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758
struct nvme_delq_ctx {
	struct task_struct *waiter;
	struct kthread_worker *worker;
	atomic_t refcount;
};

static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
{
	dq->waiter = current;
	mb();

	for (;;) {
		set_current_state(TASK_KILLABLE);
		if (!atomic_read(&dq->refcount))
			break;
		if (!schedule_timeout(ADMIN_TIMEOUT) ||
					fatal_signal_pending(current)) {
K
Keith Busch 已提交
1759 1760 1761 1762 1763 1764 1765
			/*
			 * Disable the controller first since we can't trust it
			 * at this point, but leave the admin queue enabled
			 * until all queue deletion requests are flushed.
			 * FIXME: This may take a while if there are more h/w
			 * queues than admin tags.
			 */
K
Keith Busch 已提交
1766
			set_current_state(TASK_RUNNING);
1767
			nvme_disable_ctrl(&dev->ctrl,
1768
				lo_hi_readq(dev->bar + NVME_REG_CAP));
K
Keith Busch 已提交
1769
			nvme_clear_queue(dev->queues[0]);
K
Keith Busch 已提交
1770
			flush_kthread_worker(dq->worker);
K
Keith Busch 已提交
1771
			nvme_disable_queue(dev, 0);
K
Keith Busch 已提交
1772 1773 1774 1775 1776 1777 1778 1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794
			return;
		}
	}
	set_current_state(TASK_RUNNING);
}

static void nvme_put_dq(struct nvme_delq_ctx *dq)
{
	atomic_dec(&dq->refcount);
	if (dq->waiter)
		wake_up_process(dq->waiter);
}

static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq)
{
	atomic_inc(&dq->refcount);
	return dq;
}

static void nvme_del_queue_end(struct nvme_queue *nvmeq)
{
	struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx;
	nvme_put_dq(dq);
1795 1796 1797 1798

	spin_lock_irq(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
K
Keith Busch 已提交
1799 1800 1801 1802 1803
}

static int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode,
						kthread_work_func_t fn)
{
1804
	struct request *req;
K
Keith Busch 已提交
1805 1806 1807 1808 1809 1810 1811
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(nvmeq->qid);

	init_kthread_work(&nvmeq->cmdinfo.work, fn);
1812 1813 1814 1815 1816 1817 1818 1819 1820

	req = nvme_alloc_request(nvmeq->dev->ctrl.admin_q, &c, 0);
	if (IS_ERR(req))
		return PTR_ERR(req);

	req->timeout = ADMIN_TIMEOUT;
	req->end_io_data = &nvmeq->cmdinfo;
	blk_execute_rq_nowait(req->q, NULL, req, 0, async_cmd_info_endio);
	return 0;
K
Keith Busch 已提交
1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841 1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867
}

static void nvme_del_cq_work_handler(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	nvme_del_queue_end(nvmeq);
}

static int nvme_delete_cq(struct nvme_queue *nvmeq)
{
	return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq,
						nvme_del_cq_work_handler);
}

static void nvme_del_sq_work_handler(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	int status = nvmeq->cmdinfo.status;

	if (!status)
		status = nvme_delete_cq(nvmeq);
	if (status)
		nvme_del_queue_end(nvmeq);
}

static int nvme_delete_sq(struct nvme_queue *nvmeq)
{
	return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq,
						nvme_del_sq_work_handler);
}

static void nvme_del_queue_start(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	if (nvme_delete_sq(nvmeq))
		nvme_del_queue_end(nvmeq);
}

static void nvme_disable_io_queues(struct nvme_dev *dev)
{
	int i;
	DEFINE_KTHREAD_WORKER_ONSTACK(worker);
	struct nvme_delq_ctx dq;
	struct task_struct *kworker_task = kthread_run(kthread_worker_fn,
1868
					&worker, "nvme%d", dev->ctrl.instance);
K
Keith Busch 已提交
1869 1870

	if (IS_ERR(kworker_task)) {
1871
		dev_err(dev->dev,
K
Keith Busch 已提交
1872 1873 1874 1875 1876 1877 1878 1879 1880 1881
			"Failed to create queue del task\n");
		for (i = dev->queue_count - 1; i > 0; i--)
			nvme_disable_queue(dev, i);
		return;
	}

	dq.waiter = NULL;
	atomic_set(&dq.refcount, 0);
	dq.worker = &worker;
	for (i = dev->queue_count - 1; i > 0; i--) {
M
Matias Bjørling 已提交
1882
		struct nvme_queue *nvmeq = dev->queues[i];
K
Keith Busch 已提交
1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894

		if (nvme_suspend_queue(nvmeq))
			continue;
		nvmeq->cmdinfo.ctx = nvme_get_dq(&dq);
		nvmeq->cmdinfo.worker = dq.worker;
		init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start);
		queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work);
	}
	nvme_wait_dq(&dq, dev);
	kthread_stop(kworker_task);
}

1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918
static int nvme_dev_list_add(struct nvme_dev *dev)
{
	bool start_thread = false;

	spin_lock(&dev_list_lock);
	if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
		start_thread = true;
		nvme_thread = NULL;
	}
	list_add(&dev->node, &dev_list);
	spin_unlock(&dev_list_lock);

	if (start_thread) {
		nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
		wake_up_all(&nvme_kthread_wait);
	} else
		wait_event_killable(nvme_kthread_wait, nvme_thread);

	if (IS_ERR_OR_NULL(nvme_thread))
		return nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;

	return 0;
}

1919 1920 1921 1922 1923 1924 1925 1926 1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938
/*
* Remove the node from the device list and check
* for whether or not we need to stop the nvme_thread.
*/
static void nvme_dev_list_remove(struct nvme_dev *dev)
{
	struct task_struct *tmp = NULL;

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) {
		tmp = nvme_thread;
		nvme_thread = NULL;
	}
	spin_unlock(&dev_list_lock);

	if (tmp)
		kthread_stop(tmp);
}

1939 1940 1941 1942
static void nvme_freeze_queues(struct nvme_dev *dev)
{
	struct nvme_ns *ns;

1943
	list_for_each_entry(ns, &dev->ctrl.namespaces, list) {
1944 1945
		blk_mq_freeze_queue_start(ns->queue);

1946
		spin_lock_irq(ns->queue->queue_lock);
1947
		queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
1948
		spin_unlock_irq(ns->queue->queue_lock);
1949 1950 1951 1952 1953 1954 1955 1956 1957 1958

		blk_mq_cancel_requeue_work(ns->queue);
		blk_mq_stop_hw_queues(ns->queue);
	}
}

static void nvme_unfreeze_queues(struct nvme_dev *dev)
{
	struct nvme_ns *ns;

1959
	list_for_each_entry(ns, &dev->ctrl.namespaces, list) {
1960 1961 1962 1963 1964 1965 1966
		queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
		blk_mq_unfreeze_queue(ns->queue);
		blk_mq_start_stopped_hw_queues(ns->queue, true);
		blk_mq_kick_requeue_list(ns->queue);
	}
}

1967
static void nvme_dev_shutdown(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1968
{
1969
	int i;
1970
	u32 csts = -1;
1971

1972
	nvme_dev_list_remove(dev);
1973

1974
	mutex_lock(&dev->shutdown_lock);
1975 1976
	if (dev->bar) {
		nvme_freeze_queues(dev);
1977
		csts = readl(dev->bar + NVME_REG_CSTS);
1978
	}
1979
	if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
K
Keith Busch 已提交
1980
		for (i = dev->queue_count - 1; i >= 0; i--) {
M
Matias Bjørling 已提交
1981
			struct nvme_queue *nvmeq = dev->queues[i];
K
Keith Busch 已提交
1982 1983 1984 1985
			nvme_suspend_queue(nvmeq);
		}
	} else {
		nvme_disable_io_queues(dev);
1986
		nvme_shutdown_ctrl(&dev->ctrl);
K
Keith Busch 已提交
1987 1988
		nvme_disable_queue(dev, 0);
	}
1989
	nvme_dev_unmap(dev);
1990 1991 1992

	for (i = dev->queue_count - 1; i >= 0; i--)
		nvme_clear_queue(dev->queues[i]);
1993
	mutex_unlock(&dev->shutdown_lock);
1994 1995
}

M
Matthew Wilcox 已提交
1996 1997
static int nvme_setup_prp_pools(struct nvme_dev *dev)
{
1998
	dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
M
Matthew Wilcox 已提交
1999 2000 2001 2002
						PAGE_SIZE, PAGE_SIZE, 0);
	if (!dev->prp_page_pool)
		return -ENOMEM;

2003
	/* Optimisation for I/Os between 4k and 128k */
2004
	dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2005 2006 2007 2008 2009
						256, 256, 0);
	if (!dev->prp_small_pool) {
		dma_pool_destroy(dev->prp_page_pool);
		return -ENOMEM;
	}
M
Matthew Wilcox 已提交
2010 2011 2012 2013 2014 2015
	return 0;
}

static void nvme_release_prp_pools(struct nvme_dev *dev)
{
	dma_pool_destroy(dev->prp_page_pool);
2016
	dma_pool_destroy(dev->prp_small_pool);
M
Matthew Wilcox 已提交
2017 2018
}

2019
static void nvme_pci_free_ctrl(struct nvme_ctrl *ctrl)
2020
{
2021
	struct nvme_dev *dev = to_nvme_dev(ctrl);
2022

2023
	put_device(dev->dev);
2024 2025
	if (dev->tagset.tags)
		blk_mq_free_tag_set(&dev->tagset);
2026 2027
	if (dev->ctrl.admin_q)
		blk_put_queue(dev->ctrl.admin_q);
2028 2029 2030 2031 2032
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
}

2033
static void nvme_reset_work(struct work_struct *work)
2034
{
2035
	struct nvme_dev *dev = container_of(work, struct nvme_dev, reset_work);
2036
	int result;
2037

2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049
	if (WARN_ON(test_bit(NVME_CTRL_RESETTING, &dev->flags)))
		goto out;

	/*
	 * If we're called to reset a live controller first shut it down before
	 * moving on.
	 */
	if (dev->bar)
		nvme_dev_shutdown(dev);

	set_bit(NVME_CTRL_RESETTING, &dev->flags);

2050 2051
	result = nvme_dev_map(dev);
	if (result)
2052
		goto out;
2053 2054 2055 2056 2057

	result = nvme_configure_admin_queue(dev);
	if (result)
		goto unmap;

M
Matias Bjørling 已提交
2058
	nvme_init_queue(dev->queues[0], 0);
K
Keith Busch 已提交
2059 2060 2061
	result = nvme_alloc_admin_tags(dev);
	if (result)
		goto disable;
2062

2063 2064 2065 2066
	result = nvme_init_identify(&dev->ctrl);
	if (result)
		goto free_tags;

2067
	result = nvme_setup_io_queues(dev);
2068
	if (result)
K
Keith Busch 已提交
2069
		goto free_tags;
2070

2071
	dev->ctrl.event_limit = NVME_NR_AEN_COMMANDS;
2072

2073 2074 2075 2076
	result = nvme_dev_list_add(dev);
	if (result)
		goto remove;

2077 2078 2079 2080
	/*
	 * Keep the controller around but remove all namespaces if we don't have
	 * any working I/O queue.
	 */
2081 2082
	if (dev->online_queues < 2) {
		dev_warn(dev->dev, "IO queues not created\n");
2083
		nvme_remove_namespaces(&dev->ctrl);
2084 2085 2086 2087 2088
	} else {
		nvme_unfreeze_queues(dev);
		nvme_dev_add(dev);
	}

2089
	clear_bit(NVME_CTRL_RESETTING, &dev->flags);
2090
	return;
2091

2092 2093
 remove:
	nvme_dev_list_remove(dev);
K
Keith Busch 已提交
2094 2095
 free_tags:
	nvme_dev_remove_admin(dev);
2096 2097
	blk_put_queue(dev->ctrl.admin_q);
	dev->ctrl.admin_q = NULL;
2098
	dev->queues[0]->tags = NULL;
2099
 disable:
2100
	nvme_disable_queue(dev, 0);
2101 2102
 unmap:
	nvme_dev_unmap(dev);
2103
 out:
2104
	nvme_remove_dead_ctrl(dev);
2105 2106
}

2107
static void nvme_remove_dead_ctrl_work(struct work_struct *work)
K
Keith Busch 已提交
2108
{
2109
	struct nvme_dev *dev = container_of(work, struct nvme_dev, remove_work);
2110
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
2111 2112

	if (pci_get_drvdata(pdev))
2113
		pci_stop_and_remove_bus_device_locked(pdev);
2114
	nvme_put_ctrl(&dev->ctrl);
K
Keith Busch 已提交
2115 2116
}

2117
static void nvme_remove_dead_ctrl(struct nvme_dev *dev)
2118
{
2119
	dev_warn(dev->dev, "Removing after probe failure\n");
2120
	kref_get(&dev->ctrl.kref);
2121
	if (!schedule_work(&dev->remove_work))
2122
		nvme_put_ctrl(&dev->ctrl);
2123 2124
}

2125 2126
static int nvme_reset(struct nvme_dev *dev)
{
2127
	if (!dev->ctrl.admin_q || blk_queue_dying(dev->ctrl.admin_q))
2128 2129
		return -ENODEV;

C
Christoph Hellwig 已提交
2130 2131
	if (!queue_work(nvme_workq, &dev->reset_work))
		return -EBUSY;
2132

C
Christoph Hellwig 已提交
2133 2134
	flush_work(&dev->reset_work);
	return 0;
2135 2136
}

2137 2138 2139 2140 2141 2142
static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
{
	*val = readl(to_nvme_dev(ctrl)->bar + off);
	return 0;
}

2143 2144 2145 2146 2147 2148
static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val)
{
	writel(val, to_nvme_dev(ctrl)->bar + off);
	return 0;
}

2149 2150 2151 2152 2153 2154
static int nvme_pci_reg_read64(struct nvme_ctrl *ctrl, u32 off, u64 *val)
{
	*val = readq(to_nvme_dev(ctrl)->bar + off);
	return 0;
}

2155 2156 2157 2158 2159 2160 2161
static bool nvme_pci_io_incapable(struct nvme_ctrl *ctrl)
{
	struct nvme_dev *dev = to_nvme_dev(ctrl);

	return !dev->bar || dev->online_queues < 2;
}

2162 2163 2164 2165 2166
static int nvme_pci_reset_ctrl(struct nvme_ctrl *ctrl)
{
	return nvme_reset(to_nvme_dev(ctrl));
}

2167 2168
static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
	.reg_read32		= nvme_pci_reg_read32,
2169
	.reg_write32		= nvme_pci_reg_write32,
2170
	.reg_read64		= nvme_pci_reg_read64,
2171
	.io_incapable		= nvme_pci_io_incapable,
2172
	.reset_ctrl		= nvme_pci_reset_ctrl,
2173
	.free_ctrl		= nvme_pci_free_ctrl,
2174 2175
};

2176
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
M
Matthew Wilcox 已提交
2177
{
M
Matias Bjørling 已提交
2178
	int node, result = -ENOMEM;
M
Matthew Wilcox 已提交
2179 2180
	struct nvme_dev *dev;

M
Matias Bjørling 已提交
2181 2182 2183 2184 2185
	node = dev_to_node(&pdev->dev);
	if (node == NUMA_NO_NODE)
		set_dev_node(&pdev->dev, 0);

	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2186 2187
	if (!dev)
		return -ENOMEM;
M
Matias Bjørling 已提交
2188 2189
	dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2190 2191
	if (!dev->entry)
		goto free;
M
Matias Bjørling 已提交
2192 2193
	dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2194 2195 2196
	if (!dev->queues)
		goto free;

2197
	dev->dev = get_device(&pdev->dev);
K
Keith Busch 已提交
2198
	pci_set_drvdata(pdev, dev);
2199

2200 2201 2202
	INIT_LIST_HEAD(&dev->node);
	INIT_WORK(&dev->scan_work, nvme_dev_scan);
	INIT_WORK(&dev->reset_work, nvme_reset_work);
2203
	INIT_WORK(&dev->remove_work, nvme_remove_dead_ctrl_work);
2204
	mutex_init(&dev->shutdown_lock);
2205

2206
	result = nvme_setup_prp_pools(dev);
2207
	if (result)
K
Keith Busch 已提交
2208
		goto put_pci;
M
Matthew Wilcox 已提交
2209

2210 2211
	result = nvme_init_ctrl(&dev->ctrl, &pdev->dev, &nvme_pci_ctrl_ops,
			id->driver_data);
M
Matthew Wilcox 已提交
2212
	if (result)
2213
		goto release_pools;
2214

2215
	queue_work(nvme_workq, &dev->reset_work);
M
Matthew Wilcox 已提交
2216 2217
	return 0;

2218
 release_pools:
M
Matthew Wilcox 已提交
2219
	nvme_release_prp_pools(dev);
K
Keith Busch 已提交
2220
 put_pci:
2221
	put_device(dev->dev);
M
Matthew Wilcox 已提交
2222 2223 2224 2225 2226 2227 2228
 free:
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
	return result;
}

2229 2230
static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
{
K
Keith Busch 已提交
2231
	struct nvme_dev *dev = pci_get_drvdata(pdev);
2232

K
Keith Busch 已提交
2233 2234 2235
	if (prepare)
		nvme_dev_shutdown(dev);
	else
2236
		queue_work(nvme_workq, &dev->reset_work);
2237 2238
}

2239 2240 2241 2242 2243 2244
static void nvme_shutdown(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
	nvme_dev_shutdown(dev);
}

2245
static void nvme_remove(struct pci_dev *pdev)
M
Matthew Wilcox 已提交
2246 2247
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
K
Keith Busch 已提交
2248 2249 2250 2251 2252 2253 2254

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	spin_unlock(&dev_list_lock);

	pci_set_drvdata(pdev, NULL);
	flush_work(&dev->reset_work);
K
Keith Busch 已提交
2255
	flush_work(&dev->scan_work);
2256
	nvme_remove_namespaces(&dev->ctrl);
2257
	nvme_uninit_ctrl(&dev->ctrl);
2258
	nvme_dev_shutdown(dev);
M
Matias Bjørling 已提交
2259
	nvme_dev_remove_admin(dev);
2260
	nvme_free_queues(dev, 0);
2261
	nvme_release_cmb(dev);
K
Keith Busch 已提交
2262
	nvme_release_prp_pools(dev);
2263
	nvme_put_ctrl(&dev->ctrl);
M
Matthew Wilcox 已提交
2264 2265 2266 2267 2268 2269 2270 2271
}

/* These functions are yet to be implemented */
#define nvme_error_detected NULL
#define nvme_dump_registers NULL
#define nvme_link_reset NULL
#define nvme_slot_reset NULL
#define nvme_error_resume NULL
2272

2273
#ifdef CONFIG_PM_SLEEP
2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287
static int nvme_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

	nvme_dev_shutdown(ndev);
	return 0;
}

static int nvme_resume(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

2288
	queue_work(nvme_workq, &ndev->reset_work);
K
Keith Busch 已提交
2289
	return 0;
2290
}
2291
#endif
2292 2293

static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
M
Matthew Wilcox 已提交
2294

2295
static const struct pci_error_handlers nvme_err_handler = {
M
Matthew Wilcox 已提交
2296 2297 2298 2299 2300
	.error_detected	= nvme_error_detected,
	.mmio_enabled	= nvme_dump_registers,
	.link_reset	= nvme_link_reset,
	.slot_reset	= nvme_slot_reset,
	.resume		= nvme_error_resume,
2301
	.reset_notify	= nvme_reset_notify,
M
Matthew Wilcox 已提交
2302 2303 2304 2305 2306
};

/* Move to pci_ids.h later */
#define PCI_CLASS_STORAGE_EXPRESS	0x010802

2307
static const struct pci_device_id nvme_id_table[] = {
2308 2309
	{ PCI_VDEVICE(INTEL, 0x0953),
		.driver_data = NVME_QUIRK_STRIPE_SIZE, },
2310 2311
	{ PCI_VDEVICE(INTEL, 0x5845),	/* Qemu emulated controller */
		.driver_data = NVME_QUIRK_IDENTIFY_CNS, },
M
Matthew Wilcox 已提交
2312
	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
2313
	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
M
Matthew Wilcox 已提交
2314 2315 2316 2317 2318 2319 2320 2321
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, nvme_id_table);

static struct pci_driver nvme_driver = {
	.name		= "nvme",
	.id_table	= nvme_id_table,
	.probe		= nvme_probe,
2322
	.remove		= nvme_remove,
2323
	.shutdown	= nvme_shutdown,
2324 2325 2326
	.driver		= {
		.pm	= &nvme_dev_pm_ops,
	},
M
Matthew Wilcox 已提交
2327 2328 2329 2330 2331
	.err_handler	= &nvme_err_handler,
};

static int __init nvme_init(void)
{
2332
	int result;
2333

2334
	init_waitqueue_head(&nvme_kthread_wait);
M
Matthew Wilcox 已提交
2335

2336
	nvme_workq = alloc_workqueue("nvme", WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
K
Keith Busch 已提交
2337
	if (!nvme_workq)
2338
		return -ENOMEM;
K
Keith Busch 已提交
2339

2340
	result = nvme_core_init();
2341
	if (result < 0)
K
Keith Busch 已提交
2342
		goto kill_workq;
M
Matthew Wilcox 已提交
2343

2344 2345
	result = pci_register_driver(&nvme_driver);
	if (result)
2346
		goto core_exit;
2347
	return 0;
M
Matthew Wilcox 已提交
2348

2349
 core_exit:
2350
	nvme_core_exit();
K
Keith Busch 已提交
2351 2352
 kill_workq:
	destroy_workqueue(nvme_workq);
M
Matthew Wilcox 已提交
2353 2354 2355 2356 2357 2358
	return result;
}

static void __exit nvme_exit(void)
{
	pci_unregister_driver(&nvme_driver);
2359
	nvme_core_exit();
K
Keith Busch 已提交
2360
	destroy_workqueue(nvme_workq);
2361
	BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
2362
	_nvme_check_size();
M
Matthew Wilcox 已提交
2363 2364 2365 2366
}

MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
MODULE_LICENSE("GPL");
2367
MODULE_VERSION("1.0");
M
Matthew Wilcox 已提交
2368 2369
module_init(nvme_init);
module_exit(nvme_exit);