pci.c 82.7 KB
Newer Older
M
Matthew Wilcox 已提交
1 2
/*
 * NVM Express device driver
3
 * Copyright (c) 2011-2014, Intel Corporation.
M
Matthew Wilcox 已提交
4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

15
#include <linux/bitops.h>
M
Matthew Wilcox 已提交
16
#include <linux/blkdev.h>
M
Matias Bjørling 已提交
17
#include <linux/blk-mq.h>
K
Keith Busch 已提交
18
#include <linux/cpu.h>
19
#include <linux/delay.h>
M
Matthew Wilcox 已提交
20 21 22
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/genhd.h>
K
Keith Busch 已提交
23
#include <linux/hdreg.h>
24
#include <linux/idr.h>
M
Matthew Wilcox 已提交
25 26 27 28
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kdev_t.h>
29
#include <linux/kthread.h>
M
Matthew Wilcox 已提交
30
#include <linux/kernel.h>
K
Keith Busch 已提交
31
#include <linux/list_sort.h>
M
Matthew Wilcox 已提交
32 33 34 35
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pci.h>
36
#include <linux/poison.h>
37
#include <linux/ptrace.h>
M
Matthew Wilcox 已提交
38 39
#include <linux/sched.h>
#include <linux/slab.h>
K
Keith Busch 已提交
40
#include <linux/t10-pi.h>
M
Matthew Wilcox 已提交
41
#include <linux/types.h>
V
Vishal Verma 已提交
42
#include <scsi/sg.h>
43 44
#include <asm-generic/io-64-nonatomic-lo-hi.h>

45
#include <uapi/linux/nvme_ioctl.h>
46 47
#include "nvme.h"

48
#define NVME_MINORS		(1U << MINORBITS)
49
#define NVME_Q_DEPTH		1024
J
Jens Axboe 已提交
50
#define NVME_AQ_DEPTH		256
M
Matthew Wilcox 已提交
51 52
#define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))
#define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))
53
#define ADMIN_TIMEOUT		(admin_timeout * HZ)
54
#define SHUTDOWN_TIMEOUT	(shutdown_timeout * HZ)
55 56 57 58

static unsigned char admin_timeout = 60;
module_param(admin_timeout, byte, 0644);
MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
M
Matthew Wilcox 已提交
59

60 61
unsigned char nvme_io_timeout = 30;
module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
62
MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
M
Matthew Wilcox 已提交
63

64 65 66 67
static unsigned char shutdown_timeout = 5;
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");

M
Matthew Wilcox 已提交
68 69 70
static int nvme_major;
module_param(nvme_major, int, 0);

71 72 73
static int nvme_char_major;
module_param(nvme_char_major, int, 0);

74 75 76
static int use_threaded_interrupts;
module_param(use_threaded_interrupts, int, 0);

77 78 79 80
static bool use_cmb_sqes = true;
module_param(use_cmb_sqes, bool, 0644);
MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");

81 82 83
static DEFINE_SPINLOCK(dev_list_lock);
static LIST_HEAD(dev_list);
static struct task_struct *nvme_thread;
K
Keith Busch 已提交
84
static struct workqueue_struct *nvme_workq;
85
static wait_queue_head_t nvme_kthread_wait;
86

87 88
static struct class *nvme_class;

89
static int __nvme_reset(struct nvme_dev *dev);
90
static int nvme_reset(struct nvme_dev *dev);
M
Matias Bjørling 已提交
91
static int nvme_process_cq(struct nvme_queue *nvmeq);
92
static void nvme_dead_ctrl(struct nvme_dev *dev);
93

K
Keith Busch 已提交
94 95 96
struct async_cmd_info {
	struct kthread_work work;
	struct kthread_worker *worker;
M
Matias Bjørling 已提交
97
	struct request *req;
K
Keith Busch 已提交
98 99 100 101
	u32 result;
	int status;
	void *ctx;
};
102

M
Matthew Wilcox 已提交
103 104 105 106 107 108
/*
 * An NVM Express queue.  Each device has at least two (one for admin
 * commands and one for I/O commands).
 */
struct nvme_queue {
	struct device *q_dmadev;
M
Matthew Wilcox 已提交
109
	struct nvme_dev *dev;
110
	char irqname[24];	/* nvme4294967295-65535\0 */
M
Matthew Wilcox 已提交
111 112
	spinlock_t q_lock;
	struct nvme_command *sq_cmds;
113
	struct nvme_command __iomem *sq_cmds_io;
M
Matthew Wilcox 已提交
114
	volatile struct nvme_completion *cqes;
115
	struct blk_mq_tags **tags;
M
Matthew Wilcox 已提交
116 117 118 119
	dma_addr_t sq_dma_addr;
	dma_addr_t cq_dma_addr;
	u32 __iomem *q_db;
	u16 q_depth;
J
Jens Axboe 已提交
120
	s16 cq_vector;
M
Matthew Wilcox 已提交
121 122 123
	u16 sq_head;
	u16 sq_tail;
	u16 cq_head;
K
Keith Busch 已提交
124
	u16 qid;
125 126
	u8 cq_phase;
	u8 cqe_seen;
K
Keith Busch 已提交
127
	struct async_cmd_info cmdinfo;
M
Matthew Wilcox 已提交
128 129 130 131 132 133 134 135 136 137 138 139
};

/*
 * Check we didin't inadvertently grow the command struct
 */
static inline void _nvme_check_size(void)
{
	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
140
	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
K
Keith Busch 已提交
141
	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
M
Matthew Wilcox 已提交
142 143 144 145
	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
K
Keith Busch 已提交
146
	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
M
Matthew Wilcox 已提交
147 148
}

149
typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
150 151
						struct nvme_completion *);

152
struct nvme_cmd_info {
153 154
	nvme_completion_fn fn;
	void *ctx;
K
Keith Busch 已提交
155
	int aborted;
M
Matias Bjørling 已提交
156
	struct nvme_queue *nvmeq;
157
	struct nvme_iod iod[0];
158 159
};

160 161 162 163 164
/*
 * Max size of iod being embedded in the request payload
 */
#define NVME_INT_PAGES		2
#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->page_size)
C
Chong Yuan 已提交
165
#define NVME_INT_MASK		0x01
166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188

/*
 * Will slightly overestimate the number of pages needed.  This is OK
 * as it only leads to a small amount of wasted memory for the lifetime of
 * the I/O.
 */
static int nvme_npages(unsigned size, struct nvme_dev *dev)
{
	unsigned nprps = DIV_ROUND_UP(size + dev->page_size, dev->page_size);
	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}

static unsigned int nvme_cmd_size(struct nvme_dev *dev)
{
	unsigned int ret = sizeof(struct nvme_cmd_info);

	ret += sizeof(struct nvme_iod);
	ret += sizeof(__le64 *) * nvme_npages(NVME_INT_BYTES(dev), dev);
	ret += sizeof(struct scatterlist) * NVME_INT_PAGES;

	return ret;
}

M
Matias Bjørling 已提交
189 190
static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
				unsigned int hctx_idx)
191
{
M
Matias Bjørling 已提交
192 193 194
	struct nvme_dev *dev = data;
	struct nvme_queue *nvmeq = dev->queues[0];

195 196 197 198
	WARN_ON(hctx_idx != 0);
	WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
	WARN_ON(nvmeq->tags);

M
Matias Bjørling 已提交
199
	hctx->driver_data = nvmeq;
200
	nvmeq->tags = &dev->admin_tagset.tags[0];
M
Matias Bjørling 已提交
201
	return 0;
202 203
}

204 205 206 207 208 209 210
static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	nvmeq->tags = NULL;
}

M
Matias Bjørling 已提交
211 212 213
static int nvme_admin_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
214
{
M
Matias Bjørling 已提交
215 216 217 218 219 220 221
	struct nvme_dev *dev = data;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = dev->queues[0];

	BUG_ON(!nvmeq);
	cmd->nvmeq = nvmeq;
	return 0;
222 223
}

M
Matias Bjørling 已提交
224 225
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			  unsigned int hctx_idx)
M
Matthew Wilcox 已提交
226
{
M
Matias Bjørling 已提交
227
	struct nvme_dev *dev = data;
228
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
M
Matias Bjørling 已提交
229

230 231
	if (!nvmeq->tags)
		nvmeq->tags = &dev->tagset.tags[hctx_idx];
M
Matthew Wilcox 已提交
232

233
	WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
M
Matias Bjørling 已提交
234 235
	hctx->driver_data = nvmeq;
	return 0;
M
Matthew Wilcox 已提交
236 237
}

M
Matias Bjørling 已提交
238 239 240
static int nvme_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
M
Matthew Wilcox 已提交
241
{
M
Matias Bjørling 已提交
242 243 244 245 246 247 248 249 250 251 252 253 254 255 256
	struct nvme_dev *dev = data;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];

	BUG_ON(!nvmeq);
	cmd->nvmeq = nvmeq;
	return 0;
}

static void nvme_set_info(struct nvme_cmd_info *cmd, void *ctx,
				nvme_completion_fn handler)
{
	cmd->fn = handler;
	cmd->ctx = ctx;
	cmd->aborted = 0;
K
Keith Busch 已提交
257
	blk_mq_start_request(blk_mq_rq_from_pdu(cmd));
M
Matthew Wilcox 已提交
258 259
}

260 261 262 263 264 265 266 267 268 269
static void *iod_get_private(struct nvme_iod *iod)
{
	return (void *) (iod->private & ~0x1UL);
}

/*
 * If bit 0 is set, the iod is embedded in the request payload.
 */
static bool iod_should_kfree(struct nvme_iod *iod)
{
C
Chong Yuan 已提交
270
	return (iod->private & NVME_INT_MASK) == 0;
271 272
}

273 274
/* Special values must be less than 0x1000 */
#define CMD_CTX_BASE		((void *)POISON_POINTER_DELTA)
275 276 277
#define CMD_CTX_CANCELLED	(0x30C + CMD_CTX_BASE)
#define CMD_CTX_COMPLETED	(0x310 + CMD_CTX_BASE)
#define CMD_CTX_INVALID		(0x314 + CMD_CTX_BASE)
278

279
static void special_completion(struct nvme_queue *nvmeq, void *ctx,
280 281 282 283 284
						struct nvme_completion *cqe)
{
	if (ctx == CMD_CTX_CANCELLED)
		return;
	if (ctx == CMD_CTX_COMPLETED) {
285
		dev_warn(nvmeq->q_dmadev,
286 287 288 289 290
				"completed id %d twice on queue %d\n",
				cqe->command_id, le16_to_cpup(&cqe->sq_id));
		return;
	}
	if (ctx == CMD_CTX_INVALID) {
291
		dev_warn(nvmeq->q_dmadev,
292 293 294 295
				"invalid id %d completed on queue %d\n",
				cqe->command_id, le16_to_cpup(&cqe->sq_id));
		return;
	}
296
	dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
297 298
}

M
Matias Bjørling 已提交
299
static void *cancel_cmd_info(struct nvme_cmd_info *cmd, nvme_completion_fn *fn)
M
Matthew Wilcox 已提交
300
{
301
	void *ctx;
M
Matthew Wilcox 已提交
302

303
	if (fn)
M
Matias Bjørling 已提交
304 305 306 307
		*fn = cmd->fn;
	ctx = cmd->ctx;
	cmd->fn = special_completion;
	cmd->ctx = CMD_CTX_CANCELLED;
308
	return ctx;
M
Matthew Wilcox 已提交
309 310
}

M
Matias Bjørling 已提交
311 312
static void async_req_completion(struct nvme_queue *nvmeq, void *ctx,
						struct nvme_completion *cqe)
313
{
M
Matias Bjørling 已提交
314 315 316 317 318
	u32 result = le32_to_cpup(&cqe->result);
	u16 status = le16_to_cpup(&cqe->status) >> 1;

	if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
		++nvmeq->dev->event_limit;
K
Keith Busch 已提交
319 320 321 322 323 324 325 326 327 328
	if (status != NVME_SC_SUCCESS)
		return;

	switch (result & 0xff07) {
	case NVME_AER_NOTICE_NS_CHANGED:
		dev_info(nvmeq->q_dmadev, "rescanning\n");
		schedule_work(&nvmeq->dev->scan_work);
	default:
		dev_warn(nvmeq->q_dmadev, "async event result %08x\n", result);
	}
M
Matthew Wilcox 已提交
329 330
}

M
Matias Bjørling 已提交
331 332
static void abort_completion(struct nvme_queue *nvmeq, void *ctx,
						struct nvme_completion *cqe)
333
{
M
Matias Bjørling 已提交
334 335 336 337
	struct request *req = ctx;

	u16 status = le16_to_cpup(&cqe->status) >> 1;
	u32 result = le32_to_cpup(&cqe->result);
338

339
	blk_mq_free_request(req);
340

M
Matias Bjørling 已提交
341 342
	dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
	++nvmeq->dev->abort_limit;
343 344
}

M
Matias Bjørling 已提交
345 346
static void async_completion(struct nvme_queue *nvmeq, void *ctx,
						struct nvme_completion *cqe)
M
Matthew Wilcox 已提交
347
{
M
Matias Bjørling 已提交
348 349 350 351
	struct async_cmd_info *cmdinfo = ctx;
	cmdinfo->result = le32_to_cpup(&cqe->result);
	cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
	queue_kthread_work(cmdinfo->worker, &cmdinfo->work);
352
	blk_mq_free_request(cmdinfo->req);
M
Matthew Wilcox 已提交
353 354
}

M
Matias Bjørling 已提交
355 356
static inline struct nvme_cmd_info *get_cmd_from_tag(struct nvme_queue *nvmeq,
				  unsigned int tag)
M
Matthew Wilcox 已提交
357
{
358
	struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, tag);
359

M
Matias Bjørling 已提交
360
	return blk_mq_rq_to_pdu(req);
361 362
}

M
Matias Bjørling 已提交
363 364 365 366 367
/*
 * Called with local interrupts disabled and the q_lock held.  May not sleep.
 */
static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag,
						nvme_completion_fn *fn)
368
{
M
Matias Bjørling 已提交
369 370 371 372 373 374 375 376 377 378 379 380
	struct nvme_cmd_info *cmd = get_cmd_from_tag(nvmeq, tag);
	void *ctx;
	if (tag >= nvmeq->q_depth) {
		*fn = special_completion;
		return CMD_CTX_INVALID;
	}
	if (fn)
		*fn = cmd->fn;
	ctx = cmd->ctx;
	cmd->fn = special_completion;
	cmd->ctx = CMD_CTX_COMPLETED;
	return ctx;
M
Matthew Wilcox 已提交
381 382 383
}

/**
384
 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
M
Matthew Wilcox 已提交
385 386 387 388 389
 * @nvmeq: The queue to use
 * @cmd: The command to send
 *
 * Safe to use from interrupt context
 */
390 391
static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
						struct nvme_command *cmd)
M
Matthew Wilcox 已提交
392
{
M
Matias Bjørling 已提交
393 394
	u16 tail = nvmeq->sq_tail;

395 396 397 398 399
	if (nvmeq->sq_cmds_io)
		memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd));
	else
		memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));

M
Matthew Wilcox 已提交
400 401
	if (++tail == nvmeq->q_depth)
		tail = 0;
402
	writel(tail, nvmeq->q_db);
M
Matthew Wilcox 已提交
403 404 405
	nvmeq->sq_tail = tail;
}

406
static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
M
Matias Bjørling 已提交
407 408 409
{
	unsigned long flags;
	spin_lock_irqsave(&nvmeq->q_lock, flags);
410
	__nvme_submit_cmd(nvmeq, cmd);
M
Matias Bjørling 已提交
411 412 413
	spin_unlock_irqrestore(&nvmeq->q_lock, flags);
}

414
static __le64 **iod_list(struct nvme_iod *iod)
415
{
416
	return ((void *)iod) + iod->offset;
417 418
}

419 420
static inline void iod_init(struct nvme_iod *iod, unsigned nbytes,
			    unsigned nseg, unsigned long private)
421
{
422 423 424 425 426
	iod->private = private;
	iod->offset = offsetof(struct nvme_iod, sg[nseg]);
	iod->npages = -1;
	iod->length = nbytes;
	iod->nents = 0;
427
}
M
Matthew Wilcox 已提交
428

429
static struct nvme_iod *
430 431
__nvme_alloc_iod(unsigned nseg, unsigned bytes, struct nvme_dev *dev,
		 unsigned long priv, gfp_t gfp)
M
Matthew Wilcox 已提交
432
{
433
	struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
434
				sizeof(__le64 *) * nvme_npages(bytes, dev) +
435 436
				sizeof(struct scatterlist) * nseg, gfp);

437 438
	if (iod)
		iod_init(iod, bytes, nseg, priv);
439 440

	return iod;
M
Matthew Wilcox 已提交
441 442
}

443 444 445 446 447 448 449 450 451 452 453 454 455
static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
			               gfp_t gfp)
{
	unsigned size = !(rq->cmd_flags & REQ_DISCARD) ? blk_rq_bytes(rq) :
                                                sizeof(struct nvme_dsm_range);
	struct nvme_iod *iod;

	if (rq->nr_phys_segments <= NVME_INT_PAGES &&
	    size <= NVME_INT_BYTES(dev)) {
		struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);

		iod = cmd->iod;
		iod_init(iod, size, rq->nr_phys_segments,
C
Chong Yuan 已提交
456
				(unsigned long) rq | NVME_INT_MASK);
457 458 459 460 461 462 463
		return iod;
	}

	return __nvme_alloc_iod(rq->nr_phys_segments, size, dev,
				(unsigned long) rq, gfp);
}

464
static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
M
Matthew Wilcox 已提交
465
{
466
	const int last_prp = dev->page_size / 8 - 1;
467 468 469 470 471 472 473 474 475 476 477 478
	int i;
	__le64 **list = iod_list(iod);
	dma_addr_t prp_dma = iod->first_dma;

	if (iod->npages == 0)
		dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
	for (i = 0; i < iod->npages; i++) {
		__le64 *prp_list = list[i];
		dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
		dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
		prp_dma = next_prp_dma;
	}
479 480 481

	if (iod_should_kfree(iod))
		kfree(iod);
M
Matthew Wilcox 已提交
482 483
}

K
Keith Busch 已提交
484 485 486 487 488 489 490 491 492 493 494 495
static int nvme_error_status(u16 status)
{
	switch (status & 0x7ff) {
	case NVME_SC_SUCCESS:
		return 0;
	case NVME_SC_CAP_EXCEEDED:
		return -ENOSPC;
	default:
		return -EIO;
	}
}

496
#ifdef CONFIG_BLK_DEV_INTEGRITY
K
Keith Busch 已提交
497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == v)
		pi->ref_tag = cpu_to_be32(p);
}

static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == p)
		pi->ref_tag = cpu_to_be32(v);
}

/**
 * nvme_dif_remap - remaps ref tags to bip seed and physical lba
 *
 * The virtual start sector is the one that was originally submitted by the
 * block layer.	Due to partitioning, MD/DM cloning, etc. the actual physical
 * start sector may be different. Remap protection information to match the
 * physical LBA on writes, and back to the original seed on reads.
 *
 * Type 0 and 3 do not have a ref tag, so no remapping required.
 */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
	struct nvme_ns *ns = req->rq_disk->private_data;
	struct bio_integrity_payload *bip;
	struct t10_pi_tuple *pi;
	void *p, *pmap;
	u32 i, nlb, ts, phys, virt;

	if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
		return;

	bip = bio_integrity(req->bio);
	if (!bip)
		return;

	pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;

	p = pmap;
	virt = bip_get_seed(bip);
	phys = nvme_block_nr(ns, blk_rq_pos(req));
	nlb = (blk_rq_bytes(req) >> ns->lba_shift);
541
	ts = ns->disk->integrity.tuple_size;
K
Keith Busch 已提交
542 543 544 545 546 547 548 549 550

	for (i = 0; i < nlb; i++, virt++, phys++) {
		pi = (struct t10_pi_tuple *)p;
		dif_swap(phys, virt, pi);
		p += ts;
	}
	kunmap_atomic(pmap);
}

551 552 553 554 555 556 557 558 559 560
static int nvme_noop_verify(struct blk_integrity_iter *iter)
{
	return 0;
}

static int nvme_noop_generate(struct blk_integrity_iter *iter)
{
	return 0;
}

561
struct blk_integrity_profile nvme_meta_noop = {
562 563 564 565 566 567 568 569 570 571 572
	.name			= "NVME_META_NOOP",
	.generate_fn		= nvme_noop_generate,
	.verify_fn		= nvme_noop_verify,
};

static void nvme_init_integrity(struct nvme_ns *ns)
{
	struct blk_integrity integrity;

	switch (ns->pi_type) {
	case NVME_NS_DPS_PI_TYPE3:
573
		integrity.profile = &t10_pi_type3_crc;
574 575 576
		break;
	case NVME_NS_DPS_PI_TYPE1:
	case NVME_NS_DPS_PI_TYPE2:
577
		integrity.profile = &t10_pi_type1_crc;
578 579
		break;
	default:
580
		integrity.profile = &nvme_meta_noop;
581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602
		break;
	}
	integrity.tuple_size = ns->ms;
	blk_integrity_register(ns->disk, &integrity);
	blk_queue_max_integrity_segments(ns->queue, 1);
}
#else /* CONFIG_BLK_DEV_INTEGRITY */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
}
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
static void nvme_init_integrity(struct nvme_ns *ns)
{
}
#endif

M
Matias Bjørling 已提交
603
static void req_completion(struct nvme_queue *nvmeq, void *ctx,
M
Matthew Wilcox 已提交
604 605
						struct nvme_completion *cqe)
{
606
	struct nvme_iod *iod = ctx;
607
	struct request *req = iod_get_private(iod);
M
Matias Bjørling 已提交
608
	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
M
Matthew Wilcox 已提交
609
	u16 status = le16_to_cpup(&cqe->status) >> 1;
J
Jens Axboe 已提交
610
	int error = 0;
M
Matthew Wilcox 已提交
611

612
	if (unlikely(status)) {
M
Matias Bjørling 已提交
613 614
		if (!(status & NVME_SC_DNR || blk_noretry_request(req))
		    && (jiffies - req->start_time) < req->timeout) {
615 616
			unsigned long flags;

M
Matias Bjørling 已提交
617
			blk_mq_requeue_request(req);
618 619 620 621
			spin_lock_irqsave(req->q->queue_lock, flags);
			if (!blk_queue_stopped(req->q))
				blk_mq_kick_requeue_list(req->q);
			spin_unlock_irqrestore(req->q->queue_lock, flags);
622 623
			return;
		}
624

625
		if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
626
			if (cmd_rq->ctx == CMD_CTX_CANCELLED)
627 628 629
				error = -EINTR;
			else
				error = status;
630
		} else {
631
			error = nvme_error_status(status);
632
		}
633 634
	}

K
Keith Busch 已提交
635 636 637 638
	if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
		u32 result = le32_to_cpup(&cqe->result);
		req->special = (void *)(uintptr_t)result;
	}
M
Matias Bjørling 已提交
639 640

	if (cmd_rq->aborted)
641
		dev_warn(nvmeq->dev->dev,
M
Matias Bjørling 已提交
642
			"completing aborted command with status:%04x\n",
643
			error);
M
Matias Bjørling 已提交
644

K
Keith Busch 已提交
645
	if (iod->nents) {
646
		dma_unmap_sg(nvmeq->dev->dev, iod->sg, iod->nents,
M
Matias Bjørling 已提交
647
			rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
K
Keith Busch 已提交
648 649 650
		if (blk_integrity_rq(req)) {
			if (!rq_data_dir(req))
				nvme_dif_remap(req, nvme_dif_complete);
651
			dma_unmap_sg(nvmeq->dev->dev, iod->meta_sg, 1,
K
Keith Busch 已提交
652 653 654
				rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
		}
	}
655
	nvme_free_iod(nvmeq->dev, iod);
K
Keith Busch 已提交
656

657
	blk_mq_complete_request(req, error);
M
Matthew Wilcox 已提交
658 659
}

660
/* length is in bytes.  gfp flags indicates whether we may sleep. */
661 662
static int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
		int total_len, gfp_t gfp)
M
Matthew Wilcox 已提交
663
{
664
	struct dma_pool *pool;
665 666
	int length = total_len;
	struct scatterlist *sg = iod->sg;
M
Matthew Wilcox 已提交
667 668
	int dma_len = sg_dma_len(sg);
	u64 dma_addr = sg_dma_address(sg);
669 670
	u32 page_size = dev->page_size;
	int offset = dma_addr & (page_size - 1);
671
	__le64 *prp_list;
672
	__le64 **list = iod_list(iod);
673
	dma_addr_t prp_dma;
674
	int nprps, i;
M
Matthew Wilcox 已提交
675

676
	length -= (page_size - offset);
M
Matthew Wilcox 已提交
677
	if (length <= 0)
678
		return total_len;
M
Matthew Wilcox 已提交
679

680
	dma_len -= (page_size - offset);
M
Matthew Wilcox 已提交
681
	if (dma_len) {
682
		dma_addr += (page_size - offset);
M
Matthew Wilcox 已提交
683 684 685 686 687 688
	} else {
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
	}

689
	if (length <= page_size) {
690
		iod->first_dma = dma_addr;
691
		return total_len;
692 693
	}

694
	nprps = DIV_ROUND_UP(length, page_size);
695 696
	if (nprps <= (256 / 8)) {
		pool = dev->prp_small_pool;
697
		iod->npages = 0;
698 699
	} else {
		pool = dev->prp_page_pool;
700
		iod->npages = 1;
701 702
	}

703 704
	prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
	if (!prp_list) {
705
		iod->first_dma = dma_addr;
706
		iod->npages = -1;
707
		return (total_len - length) + page_size;
708
	}
709 710
	list[0] = prp_list;
	iod->first_dma = prp_dma;
711 712
	i = 0;
	for (;;) {
713
		if (i == page_size >> 3) {
714
			__le64 *old_prp_list = prp_list;
715
			prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
716 717 718
			if (!prp_list)
				return total_len - length;
			list[iod->npages++] = prp_list;
719 720 721
			prp_list[0] = old_prp_list[i - 1];
			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
			i = 1;
722 723
		}
		prp_list[i++] = cpu_to_le64(dma_addr);
724 725 726
		dma_len -= page_size;
		dma_addr += page_size;
		length -= page_size;
727 728 729 730 731 732 733 734
		if (length <= 0)
			break;
		if (dma_len > 0)
			continue;
		BUG_ON(dma_len < 0);
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
M
Matthew Wilcox 已提交
735 736
	}

737
	return total_len;
M
Matthew Wilcox 已提交
738 739
}

740 741 742
static void nvme_submit_priv(struct nvme_queue *nvmeq, struct request *req,
		struct nvme_iod *iod)
{
743
	struct nvme_command cmnd;
744

745 746
	memcpy(&cmnd, req->cmd, sizeof(cmnd));
	cmnd.rw.command_id = req->tag;
747
	if (req->nr_phys_segments) {
748 749
		cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
		cmnd.rw.prp2 = cpu_to_le64(iod->first_dma);
750 751
	}

752
	__nvme_submit_cmd(nvmeq, &cmnd);
753 754
}

M
Matias Bjørling 已提交
755 756 757 758 759 760 761
/*
 * We reuse the small pool to allocate the 16-byte range here as it is not
 * worth having a special pool for these or additional cases to handle freeing
 * the iod.
 */
static void nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
		struct request *req, struct nvme_iod *iod)
762
{
763 764
	struct nvme_dsm_range *range =
				(struct nvme_dsm_range *)iod_list(iod)[0];
765
	struct nvme_command cmnd;
766 767

	range->cattr = cpu_to_le32(0);
M
Matias Bjørling 已提交
768 769
	range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
	range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
770

771 772 773 774 775 776 777
	memset(&cmnd, 0, sizeof(cmnd));
	cmnd.dsm.opcode = nvme_cmd_dsm;
	cmnd.dsm.command_id = req->tag;
	cmnd.dsm.nsid = cpu_to_le32(ns->ns_id);
	cmnd.dsm.prp1 = cpu_to_le64(iod->first_dma);
	cmnd.dsm.nr = 0;
	cmnd.dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
778

779
	__nvme_submit_cmd(nvmeq, &cmnd);
780 781
}

M
Matias Bjørling 已提交
782
static void nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
M
Matthew Wilcox 已提交
783 784
								int cmdid)
{
785
	struct nvme_command cmnd;
M
Matthew Wilcox 已提交
786

787 788 789 790
	memset(&cmnd, 0, sizeof(cmnd));
	cmnd.common.opcode = nvme_cmd_flush;
	cmnd.common.command_id = cmdid;
	cmnd.common.nsid = cpu_to_le32(ns->ns_id);
M
Matthew Wilcox 已提交
791

792
	__nvme_submit_cmd(nvmeq, &cmnd);
M
Matthew Wilcox 已提交
793 794
}

M
Matias Bjørling 已提交
795 796
static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod,
							struct nvme_ns *ns)
M
Matthew Wilcox 已提交
797
{
798
	struct request *req = iod_get_private(iod);
799
	struct nvme_command cmnd;
M
Matias Bjørling 已提交
800 801
	u16 control = 0;
	u32 dsmgmt = 0;
M
Matthew Wilcox 已提交
802

M
Matias Bjørling 已提交
803
	if (req->cmd_flags & REQ_FUA)
M
Matthew Wilcox 已提交
804
		control |= NVME_RW_FUA;
M
Matias Bjørling 已提交
805
	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
M
Matthew Wilcox 已提交
806 807
		control |= NVME_RW_LR;

M
Matias Bjørling 已提交
808
	if (req->cmd_flags & REQ_RAHEAD)
M
Matthew Wilcox 已提交
809 810
		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;

811 812 813 814 815 816 817 818
	memset(&cmnd, 0, sizeof(cmnd));
	cmnd.rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
	cmnd.rw.command_id = req->tag;
	cmnd.rw.nsid = cpu_to_le32(ns->ns_id);
	cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
	cmnd.rw.prp2 = cpu_to_le64(iod->first_dma);
	cmnd.rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
	cmnd.rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
M
Matthew Wilcox 已提交
819

820
	if (ns->ms) {
K
Keith Busch 已提交
821 822 823 824 825 826 827 828
		switch (ns->pi_type) {
		case NVME_NS_DPS_PI_TYPE3:
			control |= NVME_RW_PRINFO_PRCHK_GUARD;
			break;
		case NVME_NS_DPS_PI_TYPE1:
		case NVME_NS_DPS_PI_TYPE2:
			control |= NVME_RW_PRINFO_PRCHK_GUARD |
					NVME_RW_PRINFO_PRCHK_REF;
829
			cmnd.rw.reftag = cpu_to_le32(
K
Keith Busch 已提交
830 831 832
					nvme_block_nr(ns, blk_rq_pos(req)));
			break;
		}
833 834 835 836 837 838
		if (blk_integrity_rq(req))
			cmnd.rw.metadata =
				cpu_to_le64(sg_dma_address(iod->meta_sg));
		else
			control |= NVME_RW_PRINFO_PRACT;
	}
K
Keith Busch 已提交
839

840 841
	cmnd.rw.control = cpu_to_le16(control);
	cmnd.rw.dsmgmt = cpu_to_le32(dsmgmt);
M
Matthew Wilcox 已提交
842

843
	__nvme_submit_cmd(nvmeq, &cmnd);
M
Matthew Wilcox 已提交
844

845
	return 0;
846 847
}

848 849 850
/*
 * NOTE: ns is NULL when called on the admin queue.
 */
M
Matias Bjørling 已提交
851 852
static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
			 const struct blk_mq_queue_data *bd)
853
{
M
Matias Bjørling 已提交
854 855
	struct nvme_ns *ns = hctx->queue->queuedata;
	struct nvme_queue *nvmeq = hctx->driver_data;
856
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
857 858
	struct request *req = bd->rq;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
859
	struct nvme_iod *iod;
M
Matias Bjørling 已提交
860
	enum dma_data_direction dma_dir;
861

K
Keith Busch 已提交
862 863 864 865 866
	/*
	 * If formated with metadata, require the block layer provide a buffer
	 * unless this namespace is formated such that the metadata can be
	 * stripped/generated by the controller with PRACT=1.
	 */
867
	if (ns && ns->ms && !blk_integrity_rq(req)) {
868 869
		if (!(ns->pi_type && ns->ms == 8) &&
					req->cmd_type != REQ_TYPE_DRV_PRIV) {
870
			blk_mq_complete_request(req, -EFAULT);
K
Keith Busch 已提交
871 872 873 874
			return BLK_MQ_RQ_QUEUE_OK;
		}
	}

875
	iod = nvme_alloc_iod(req, dev, GFP_ATOMIC);
876
	if (!iod)
877
		return BLK_MQ_RQ_QUEUE_BUSY;
M
Matias Bjørling 已提交
878 879

	if (req->cmd_flags & REQ_DISCARD) {
880 881 882 883 884 885
		void *range;
		/*
		 * We reuse the small pool to allocate the 16-byte range here
		 * as it is not worth having a special pool for these or
		 * additional cases to handle freeing the iod.
		 */
886
		range = dma_pool_alloc(dev->prp_small_pool, GFP_ATOMIC,
887
						&iod->first_dma);
M
Matias Bjørling 已提交
888
		if (!range)
889
			goto retry_cmd;
890 891
		iod_list(iod)[0] = (__le64 *)range;
		iod->npages = 0;
892
	} else if (req->nr_phys_segments) {
M
Matias Bjørling 已提交
893 894
		dma_dir = rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;

895
		sg_init_table(iod->sg, req->nr_phys_segments);
M
Matias Bjørling 已提交
896
		iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
897 898
		if (!iod->nents)
			goto error_cmd;
M
Matias Bjørling 已提交
899 900

		if (!dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir))
901
			goto retry_cmd;
M
Matias Bjørling 已提交
902

903
		if (blk_rq_bytes(req) !=
904 905
                    nvme_setup_prps(dev, iod, blk_rq_bytes(req), GFP_ATOMIC)) {
			dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
906 907
			goto retry_cmd;
		}
K
Keith Busch 已提交
908 909 910 911 912 913 914 915 916 917 918 919 920 921 922
		if (blk_integrity_rq(req)) {
			if (blk_rq_count_integrity_sg(req->q, req->bio) != 1)
				goto error_cmd;

			sg_init_table(iod->meta_sg, 1);
			if (blk_rq_map_integrity_sg(
					req->q, req->bio, iod->meta_sg) != 1)
				goto error_cmd;

			if (rq_data_dir(req))
				nvme_dif_remap(req, nvme_dif_prep);

			if (!dma_map_sg(nvmeq->q_dmadev, iod->meta_sg, 1, dma_dir))
				goto error_cmd;
		}
923
	}
924

K
Keith Busch 已提交
925
	nvme_set_info(cmd, iod, req_completion);
M
Matias Bjørling 已提交
926
	spin_lock_irq(&nvmeq->q_lock);
927 928 929
	if (req->cmd_type == REQ_TYPE_DRV_PRIV)
		nvme_submit_priv(nvmeq, req, iod);
	else if (req->cmd_flags & REQ_DISCARD)
M
Matias Bjørling 已提交
930 931 932 933 934 935 936 937 938 939
		nvme_submit_discard(nvmeq, ns, req, iod);
	else if (req->cmd_flags & REQ_FLUSH)
		nvme_submit_flush(nvmeq, ns, req->tag);
	else
		nvme_submit_iod(nvmeq, iod, ns);

	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
	return BLK_MQ_RQ_QUEUE_OK;

940
 error_cmd:
941
	nvme_free_iod(dev, iod);
942 943
	return BLK_MQ_RQ_QUEUE_ERROR;
 retry_cmd:
944
	nvme_free_iod(dev, iod);
945
	return BLK_MQ_RQ_QUEUE_BUSY;
M
Matthew Wilcox 已提交
946 947
}

948
static int nvme_process_cq(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
949
{
M
Matthew Wilcox 已提交
950
	u16 head, phase;
M
Matthew Wilcox 已提交
951 952

	head = nvmeq->cq_head;
M
Matthew Wilcox 已提交
953
	phase = nvmeq->cq_phase;
M
Matthew Wilcox 已提交
954 955

	for (;;) {
956 957
		void *ctx;
		nvme_completion_fn fn;
M
Matthew Wilcox 已提交
958
		struct nvme_completion cqe = nvmeq->cqes[head];
M
Matthew Wilcox 已提交
959
		if ((le16_to_cpu(cqe.status) & 1) != phase)
M
Matthew Wilcox 已提交
960 961 962 963
			break;
		nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
		if (++head == nvmeq->q_depth) {
			head = 0;
M
Matthew Wilcox 已提交
964
			phase = !phase;
M
Matthew Wilcox 已提交
965
		}
M
Matias Bjørling 已提交
966
		ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
967
		fn(nvmeq, ctx, &cqe);
M
Matthew Wilcox 已提交
968 969 970 971 972 973 974 975
	}

	/* If the controller ignores the cq head doorbell and continuously
	 * writes to the queue, it is theoretically possible to wrap around
	 * the queue twice and mistakenly return IRQ_NONE.  Linux only
	 * requires that 0.1% of your interrupts are handled, so this isn't
	 * a big problem.
	 */
M
Matthew Wilcox 已提交
976
	if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
977
		return 0;
M
Matthew Wilcox 已提交
978

979
	writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
M
Matthew Wilcox 已提交
980
	nvmeq->cq_head = head;
M
Matthew Wilcox 已提交
981
	nvmeq->cq_phase = phase;
M
Matthew Wilcox 已提交
982

983 984
	nvmeq->cqe_seen = 1;
	return 1;
M
Matthew Wilcox 已提交
985 986 987
}

static irqreturn_t nvme_irq(int irq, void *data)
988 989 990 991
{
	irqreturn_t result;
	struct nvme_queue *nvmeq = data;
	spin_lock(&nvmeq->q_lock);
992 993 994
	nvme_process_cq(nvmeq);
	result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
	nvmeq->cqe_seen = 0;
995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007
	spin_unlock(&nvmeq->q_lock);
	return result;
}

static irqreturn_t nvme_irq_check(int irq, void *data)
{
	struct nvme_queue *nvmeq = data;
	struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
	if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
		return IRQ_NONE;
	return IRQ_WAKE_THREAD;
}

M
Matthew Wilcox 已提交
1008 1009 1010 1011
/*
 * Returns 0 on success.  If the result is negative, it's a Linux error code;
 * if the result is positive, it's an NVM Express status code
 */
1012 1013 1014
int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
		void *buffer, void __user *ubuffer, unsigned bufflen,
		u32 *result, unsigned timeout)
M
Matthew Wilcox 已提交
1015
{
1016 1017
	bool write = cmd->common.opcode & 1;
	struct bio *bio = NULL;
1018
	struct request *req;
1019
	int ret;
M
Matthew Wilcox 已提交
1020

1021
	req = blk_mq_alloc_request(q, write, GFP_KERNEL, false);
1022 1023
	if (IS_ERR(req))
		return PTR_ERR(req);
M
Matthew Wilcox 已提交
1024

1025
	req->cmd_type = REQ_TYPE_DRV_PRIV;
1026
	req->cmd_flags |= REQ_FAILFAST_DRIVER;
1027 1028 1029
	req->__data_len = 0;
	req->__sector = (sector_t) -1;
	req->bio = req->biotail = NULL;
M
Matthew Wilcox 已提交
1030

1031
	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
M
Matias Bjørling 已提交
1032

1033 1034
	req->cmd = (unsigned char *)cmd;
	req->cmd_len = sizeof(struct nvme_command);
K
Keith Busch 已提交
1035
	req->special = (void *)0;
M
Matthew Wilcox 已提交
1036

1037 1038 1039 1040 1041 1042 1043 1044 1045 1046
	if (buffer && bufflen) {
		ret = blk_rq_map_kern(q, req, buffer, bufflen, __GFP_WAIT);
		if (ret)
			goto out;
	} else if (ubuffer && bufflen) {
		ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen, __GFP_WAIT);
		if (ret)
			goto out;
		bio = req->bio;
	}
1047

1048 1049 1050
	blk_execute_rq(req->q, NULL, req, 0);
	if (bio)
		blk_rq_unmap_user(bio);
M
Matthew Wilcox 已提交
1051
	if (result)
K
Keith Busch 已提交
1052
		*result = (u32)(uintptr_t)req->special;
1053 1054
	ret = req->errors;
 out:
1055
	blk_mq_free_request(req);
1056
	return ret;
1057 1058
}

1059 1060
int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
		void *buffer, unsigned bufflen)
1061
{
1062
	return __nvme_submit_sync_cmd(q, cmd, buffer, NULL, bufflen, NULL, 0);
M
Matthew Wilcox 已提交
1063 1064
}

M
Matias Bjørling 已提交
1065 1066 1067 1068 1069 1070 1071
static int nvme_submit_async_admin_req(struct nvme_dev *dev)
{
	struct nvme_queue *nvmeq = dev->queues[0];
	struct nvme_command c;
	struct nvme_cmd_info *cmd_info;
	struct request *req;

1072
	req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC, true);
1073 1074
	if (IS_ERR(req))
		return PTR_ERR(req);
M
Matias Bjørling 已提交
1075

K
Keith Busch 已提交
1076
	req->cmd_flags |= REQ_NO_TIMEOUT;
M
Matias Bjørling 已提交
1077
	cmd_info = blk_mq_rq_to_pdu(req);
1078
	nvme_set_info(cmd_info, NULL, async_req_completion);
M
Matias Bjørling 已提交
1079 1080 1081 1082 1083

	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_async_event;
	c.common.command_id = req->tag;

1084
	blk_mq_free_request(req);
1085 1086
	__nvme_submit_cmd(nvmeq, &c);
	return 0;
M
Matias Bjørling 已提交
1087 1088 1089
}

static int nvme_submit_admin_async_cmd(struct nvme_dev *dev,
K
Keith Busch 已提交
1090 1091 1092
			struct nvme_command *cmd,
			struct async_cmd_info *cmdinfo, unsigned timeout)
{
M
Matias Bjørling 已提交
1093 1094 1095
	struct nvme_queue *nvmeq = dev->queues[0];
	struct request *req;
	struct nvme_cmd_info *cmd_rq;
K
Keith Busch 已提交
1096

M
Matias Bjørling 已提交
1097
	req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_KERNEL, false);
1098 1099
	if (IS_ERR(req))
		return PTR_ERR(req);
M
Matias Bjørling 已提交
1100 1101 1102 1103 1104

	req->timeout = timeout;
	cmd_rq = blk_mq_rq_to_pdu(req);
	cmdinfo->req = req;
	nvme_set_info(cmd_rq, cmdinfo, async_completion);
K
Keith Busch 已提交
1105
	cmdinfo->status = -EINTR;
M
Matias Bjørling 已提交
1106 1107 1108

	cmd->common.command_id = req->tag;

1109 1110
	nvme_submit_cmd(nvmeq, cmd);
	return 0;
K
Keith Busch 已提交
1111 1112
}

M
Matthew Wilcox 已提交
1113 1114 1115 1116 1117 1118 1119 1120
static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(id);

1121
	return nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1122 1123 1124 1125 1126 1127 1128 1129
}

static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;

1130 1131 1132 1133
	/*
	 * Note: we (ab)use the fact the the prp fields survive if no data
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
1134 1135 1136 1137 1138 1139 1140 1141
	memset(&c, 0, sizeof(c));
	c.create_cq.opcode = nvme_admin_create_cq;
	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
	c.create_cq.cqid = cpu_to_le16(qid);
	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_cq.cq_flags = cpu_to_le16(flags);
	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);

1142
	return nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1143 1144 1145 1146 1147 1148 1149 1150
}

static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;

1151 1152 1153 1154
	/*
	 * Note: we (ab)use the fact the the prp fields survive if no data
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
1155 1156 1157 1158 1159 1160 1161 1162
	memset(&c, 0, sizeof(c));
	c.create_sq.opcode = nvme_admin_create_sq;
	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
	c.create_sq.sqid = cpu_to_le16(qid);
	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_sq.sq_flags = cpu_to_le16(flags);
	c.create_sq.cqid = cpu_to_le16(qid);

1163
	return nvme_submit_sync_cmd(dev->admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175
}

static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
}

static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
}

1176
int nvme_identify_ctrl(struct nvme_dev *dev, struct nvme_id_ctrl **id)
1177
{
1178
	struct nvme_command c = { };
1179
	int error;
1180

1181 1182 1183 1184
	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
	c.identify.opcode = nvme_admin_identify;
	c.identify.cns = cpu_to_le32(1);

1185 1186 1187
	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
	if (!*id)
		return -ENOMEM;
1188

1189 1190 1191 1192 1193 1194 1195 1196 1197 1198
	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
			sizeof(struct nvme_id_ctrl));
	if (error)
		kfree(*id);
	return error;
}

int nvme_identify_ns(struct nvme_dev *dev, unsigned nsid,
		struct nvme_id_ns **id)
{
1199
	struct nvme_command c = { };
1200
	int error;
1201

1202 1203 1204 1205
	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
	c.identify.opcode = nvme_admin_identify,
	c.identify.nsid = cpu_to_le32(nsid),

1206 1207 1208 1209 1210 1211 1212 1213 1214
	*id = kmalloc(sizeof(struct nvme_id_ns), GFP_KERNEL);
	if (!*id)
		return -ENOMEM;

	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
			sizeof(struct nvme_id_ns));
	if (error)
		kfree(*id);
	return error;
1215 1216
}

V
Vishal Verma 已提交
1217
int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
1218
					dma_addr_t dma_addr, u32 *result)
1219 1220 1221 1222 1223
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.features.opcode = nvme_admin_get_features;
1224
	c.features.nsid = cpu_to_le32(nsid);
1225 1226 1227
	c.features.prp1 = cpu_to_le64(dma_addr);
	c.features.fid = cpu_to_le32(fid);

1228 1229
	return __nvme_submit_sync_cmd(dev->admin_q, &c, NULL, NULL, 0,
			result, 0);
1230 1231
}

V
Vishal Verma 已提交
1232 1233
int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
					dma_addr_t dma_addr, u32 *result)
1234 1235 1236 1237 1238 1239 1240 1241 1242
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.features.opcode = nvme_admin_set_features;
	c.features.prp1 = cpu_to_le64(dma_addr);
	c.features.fid = cpu_to_le32(fid);
	c.features.dword11 = cpu_to_le32(dword11);

1243 1244 1245 1246 1247 1248
	return __nvme_submit_sync_cmd(dev->admin_q, &c, NULL, NULL, 0,
			result, 0);
}

int nvme_get_log_page(struct nvme_dev *dev, struct nvme_smart_log **log)
{
1249 1250 1251 1252 1253 1254
	struct nvme_command c = { };
	int error;

	c.common.opcode = nvme_admin_get_log_page,
	c.common.nsid = cpu_to_le32(0xFFFFFFFF),
	c.common.cdw10[0] = cpu_to_le32(
1255 1256 1257 1258 1259 1260 1261 1262 1263 1264 1265 1266
			(((sizeof(struct nvme_smart_log) / 4) - 1) << 16) |
			 NVME_LOG_SMART),

	*log = kmalloc(sizeof(struct nvme_smart_log), GFP_KERNEL);
	if (!*log)
		return -ENOMEM;

	error = nvme_submit_sync_cmd(dev->admin_q, &c, *log,
			sizeof(struct nvme_smart_log));
	if (error)
		kfree(*log);
	return error;
1267 1268
}

K
Keith Busch 已提交
1269
/**
M
Matias Bjørling 已提交
1270
 * nvme_abort_req - Attempt aborting a request
K
Keith Busch 已提交
1271 1272 1273 1274
 *
 * Schedule controller reset if the command was already aborted once before and
 * still hasn't been returned to the driver, or if this is the admin queue.
 */
M
Matias Bjørling 已提交
1275
static void nvme_abort_req(struct request *req)
K
Keith Busch 已提交
1276
{
M
Matias Bjørling 已提交
1277 1278
	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd_rq->nvmeq;
K
Keith Busch 已提交
1279
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
1280 1281 1282
	struct request *abort_req;
	struct nvme_cmd_info *abort_cmd;
	struct nvme_command cmd;
K
Keith Busch 已提交
1283

M
Matias Bjørling 已提交
1284
	if (!nvmeq->qid || cmd_rq->aborted) {
1285 1286 1287 1288 1289 1290 1291
		spin_lock(&dev_list_lock);
		if (!__nvme_reset(dev)) {
			dev_warn(dev->dev,
				 "I/O %d QID %d timeout, reset controller\n",
				 req->tag, nvmeq->qid);
		}
		spin_unlock(&dev_list_lock);
K
Keith Busch 已提交
1292 1293 1294 1295 1296 1297
		return;
	}

	if (!dev->abort_limit)
		return;

M
Matias Bjørling 已提交
1298 1299
	abort_req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC,
									false);
1300
	if (IS_ERR(abort_req))
K
Keith Busch 已提交
1301 1302
		return;

M
Matias Bjørling 已提交
1303 1304 1305
	abort_cmd = blk_mq_rq_to_pdu(abort_req);
	nvme_set_info(abort_cmd, abort_req, abort_completion);

K
Keith Busch 已提交
1306 1307
	memset(&cmd, 0, sizeof(cmd));
	cmd.abort.opcode = nvme_admin_abort_cmd;
M
Matias Bjørling 已提交
1308
	cmd.abort.cid = req->tag;
K
Keith Busch 已提交
1309
	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
M
Matias Bjørling 已提交
1310
	cmd.abort.command_id = abort_req->tag;
K
Keith Busch 已提交
1311 1312

	--dev->abort_limit;
M
Matias Bjørling 已提交
1313
	cmd_rq->aborted = 1;
K
Keith Busch 已提交
1314

M
Matias Bjørling 已提交
1315
	dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", req->tag,
K
Keith Busch 已提交
1316
							nvmeq->qid);
1317
	nvme_submit_cmd(dev->queues[0], &cmd);
K
Keith Busch 已提交
1318 1319
}

1320
static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved)
1321
{
M
Matias Bjørling 已提交
1322 1323 1324 1325
	struct nvme_queue *nvmeq = data;
	void *ctx;
	nvme_completion_fn fn;
	struct nvme_cmd_info *cmd;
K
Keith Busch 已提交
1326 1327 1328 1329
	struct nvme_completion cqe;

	if (!blk_mq_request_started(req))
		return;
1330

M
Matias Bjørling 已提交
1331
	cmd = blk_mq_rq_to_pdu(req);
1332

M
Matias Bjørling 已提交
1333 1334 1335
	if (cmd->ctx == CMD_CTX_CANCELLED)
		return;

K
Keith Busch 已提交
1336 1337 1338 1339 1340 1341
	if (blk_queue_dying(req->q))
		cqe.status = cpu_to_le16((NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1);
	else
		cqe.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);


M
Matias Bjørling 已提交
1342 1343 1344 1345
	dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n",
						req->tag, nvmeq->qid);
	ctx = cancel_cmd_info(cmd, &fn);
	fn(nvmeq, ctx, &cqe);
1346 1347
}

M
Matias Bjørling 已提交
1348
static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
1349
{
M
Matias Bjørling 已提交
1350 1351 1352 1353 1354
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd->nvmeq;

	dev_warn(nvmeq->q_dmadev, "Timeout I/O %d QID %d\n", req->tag,
							nvmeq->qid);
1355
	spin_lock_irq(&nvmeq->q_lock);
1356
	nvme_abort_req(req);
1357
	spin_unlock_irq(&nvmeq->q_lock);
M
Matias Bjørling 已提交
1358

1359 1360 1361 1362 1363 1364
	/*
	 * The aborted req will be completed on receiving the abort req.
	 * We enable the timer again. If hit twice, it'll cause a device reset,
	 * as the device then is in a faulty state.
	 */
	return BLK_EH_RESET_TIMER;
M
Matias Bjørling 已提交
1365
}
1366

M
Matias Bjørling 已提交
1367 1368
static void nvme_free_queue(struct nvme_queue *nvmeq)
{
1369 1370
	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1371 1372
	if (nvmeq->sq_cmds)
		dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1373 1374 1375 1376
					nvmeq->sq_cmds, nvmeq->sq_dma_addr);
	kfree(nvmeq);
}

1377
static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1378 1379 1380
{
	int i;

1381
	for (i = dev->queue_count - 1; i >= lowest; i--) {
M
Matias Bjørling 已提交
1382
		struct nvme_queue *nvmeq = dev->queues[i];
1383
		dev->queue_count--;
M
Matias Bjørling 已提交
1384
		dev->queues[i] = NULL;
K
Keith Busch 已提交
1385
		nvme_free_queue(nvmeq);
1386
	}
1387 1388
}

K
Keith Busch 已提交
1389 1390 1391 1392 1393
/**
 * nvme_suspend_queue - put queue into suspended state
 * @nvmeq - queue to suspend
 */
static int nvme_suspend_queue(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
1394
{
K
Keith Busch 已提交
1395
	int vector;
M
Matthew Wilcox 已提交
1396

1397
	spin_lock_irq(&nvmeq->q_lock);
K
Keith Busch 已提交
1398 1399 1400 1401 1402
	if (nvmeq->cq_vector == -1) {
		spin_unlock_irq(&nvmeq->q_lock);
		return 1;
	}
	vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
K
Keith Busch 已提交
1403
	nvmeq->dev->online_queues--;
K
Keith Busch 已提交
1404
	nvmeq->cq_vector = -1;
1405 1406
	spin_unlock_irq(&nvmeq->q_lock);

1407 1408 1409
	if (!nvmeq->qid && nvmeq->dev->admin_q)
		blk_mq_freeze_queue_start(nvmeq->dev->admin_q);

M
Matthew Wilcox 已提交
1410 1411
	irq_set_affinity_hint(vector, NULL);
	free_irq(vector, nvmeq);
M
Matthew Wilcox 已提交
1412

K
Keith Busch 已提交
1413 1414
	return 0;
}
M
Matthew Wilcox 已提交
1415

K
Keith Busch 已提交
1416 1417
static void nvme_clear_queue(struct nvme_queue *nvmeq)
{
1418
	spin_lock_irq(&nvmeq->q_lock);
1419 1420
	if (nvmeq->tags && *nvmeq->tags)
		blk_mq_all_tag_busy_iter(*nvmeq->tags, nvme_cancel_queue_ios, nvmeq);
1421
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1422 1423
}

K
Keith Busch 已提交
1424 1425
static void nvme_disable_queue(struct nvme_dev *dev, int qid)
{
M
Matias Bjørling 已提交
1426
	struct nvme_queue *nvmeq = dev->queues[qid];
K
Keith Busch 已提交
1427 1428 1429 1430 1431 1432

	if (!nvmeq)
		return;
	if (nvme_suspend_queue(nvmeq))
		return;

K
Keith Busch 已提交
1433 1434 1435
	/* Don't tell the adapter to delete the admin queue.
	 * Don't tell a removed adapter to delete IO queues. */
	if (qid && readl(&dev->bar->csts) != -1) {
M
Matthew Wilcox 已提交
1436 1437 1438
		adapter_delete_sq(dev, qid);
		adapter_delete_cq(dev, qid);
	}
1439 1440 1441 1442

	spin_lock_irq(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1443 1444
}

1445 1446 1447 1448 1449 1450 1451
static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
				int entry_size)
{
	int q_depth = dev->q_depth;
	unsigned q_size_aligned = roundup(q_depth * entry_size, dev->page_size);

	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1452 1453 1454
		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
		mem_per_q = round_down(mem_per_q, dev->page_size);
		q_depth = div_u64(mem_per_q, entry_size);
1455 1456 1457 1458 1459 1460 1461 1462 1463 1464 1465 1466 1467 1468 1469 1470 1471 1472 1473 1474 1475 1476 1477 1478 1479 1480 1481 1482 1483 1484 1485

		/*
		 * Ensure the reduced q_depth is above some threshold where it
		 * would be better to map queues in system memory with the
		 * original depth
		 */
		if (q_depth < 64)
			return -ENOMEM;
	}

	return q_depth;
}

static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
				int qid, int depth)
{
	if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
		unsigned offset = (qid - 1) *
					roundup(SQ_SIZE(depth), dev->page_size);
		nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
		nvmeq->sq_cmds_io = dev->cmb + offset;
	} else {
		nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
					&nvmeq->sq_dma_addr, GFP_KERNEL);
		if (!nvmeq->sq_cmds)
			return -ENOMEM;
	}

	return 0;
}

M
Matthew Wilcox 已提交
1486
static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
K
Keith Busch 已提交
1487
							int depth)
M
Matthew Wilcox 已提交
1488
{
M
Matias Bjørling 已提交
1489
	struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
M
Matthew Wilcox 已提交
1490 1491 1492
	if (!nvmeq)
		return NULL;

1493
	nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
J
Joe Perches 已提交
1494
					  &nvmeq->cq_dma_addr, GFP_KERNEL);
M
Matthew Wilcox 已提交
1495 1496 1497
	if (!nvmeq->cqes)
		goto free_nvmeq;

1498
	if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
M
Matthew Wilcox 已提交
1499 1500
		goto free_cqdma;

1501
	nvmeq->q_dmadev = dev->dev;
M
Matthew Wilcox 已提交
1502
	nvmeq->dev = dev;
1503 1504
	snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
			dev->instance, qid);
M
Matthew Wilcox 已提交
1505 1506
	spin_lock_init(&nvmeq->q_lock);
	nvmeq->cq_head = 0;
M
Matthew Wilcox 已提交
1507
	nvmeq->cq_phase = 1;
1508
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
M
Matthew Wilcox 已提交
1509
	nvmeq->q_depth = depth;
K
Keith Busch 已提交
1510
	nvmeq->qid = qid;
1511
	nvmeq->cq_vector = -1;
M
Matias Bjørling 已提交
1512
	dev->queues[qid] = nvmeq;
M
Matthew Wilcox 已提交
1513

1514 1515 1516 1517
	/* make sure queue descriptor is set before queue count, for kthread */
	mb();
	dev->queue_count++;

M
Matthew Wilcox 已提交
1518 1519 1520
	return nvmeq;

 free_cqdma:
1521
	dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
M
Matthew Wilcox 已提交
1522 1523 1524 1525 1526 1527
							nvmeq->cq_dma_addr);
 free_nvmeq:
	kfree(nvmeq);
	return NULL;
}

1528 1529 1530
static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
							const char *name)
{
1531 1532
	if (use_threaded_interrupts)
		return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1533
					nvme_irq_check, nvme_irq, IRQF_SHARED,
1534
					name, nvmeq);
1535
	return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1536
				IRQF_SHARED, name, nvmeq);
1537 1538
}

1539
static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
M
Matthew Wilcox 已提交
1540
{
1541
	struct nvme_dev *dev = nvmeq->dev;
M
Matthew Wilcox 已提交
1542

1543
	spin_lock_irq(&nvmeq->q_lock);
1544 1545 1546
	nvmeq->sq_tail = 0;
	nvmeq->cq_head = 0;
	nvmeq->cq_phase = 1;
1547
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1548
	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
K
Keith Busch 已提交
1549
	dev->online_queues++;
1550
	spin_unlock_irq(&nvmeq->q_lock);
1551 1552 1553 1554 1555 1556
}

static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
{
	struct nvme_dev *dev = nvmeq->dev;
	int result;
1557

K
Keith Busch 已提交
1558
	nvmeq->cq_vector = qid - 1;
M
Matthew Wilcox 已提交
1559 1560
	result = adapter_alloc_cq(dev, qid, nvmeq);
	if (result < 0)
1561
		return result;
M
Matthew Wilcox 已提交
1562 1563 1564 1565 1566

	result = adapter_alloc_sq(dev, qid, nvmeq);
	if (result < 0)
		goto release_cq;

1567
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
M
Matthew Wilcox 已提交
1568 1569 1570
	if (result < 0)
		goto release_sq;

1571 1572
	nvme_init_queue(nvmeq, qid);
	return result;
M
Matthew Wilcox 已提交
1573 1574 1575 1576 1577

 release_sq:
	adapter_delete_sq(dev, qid);
 release_cq:
	adapter_delete_cq(dev, qid);
1578
	return result;
M
Matthew Wilcox 已提交
1579 1580
}

1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592
static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
{
	unsigned long timeout;
	u32 bit = enabled ? NVME_CSTS_RDY : 0;

	timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;

	while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) {
		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
1593
			dev_err(dev->dev,
M
Matthew Wilcox 已提交
1594 1595
				"Device not ready; aborting %s\n", enabled ?
						"initialisation" : "reset");
1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610
			return -ENODEV;
		}
	}

	return 0;
}

/*
 * If the device has been passed off to us in an enabled state, just clear
 * the enabled bit.  The spec says we should set the 'shutdown notification
 * bits', but doing so may cause the device to complete commands to the
 * admin queue ... and we don't know what memory that might be pointing at!
 */
static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
{
1611 1612 1613
	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
	dev->ctrl_config &= ~NVME_CC_ENABLE;
	writel(dev->ctrl_config, &dev->bar->cc);
1614

1615 1616 1617 1618 1619
	return nvme_wait_ready(dev, cap, false);
}

static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
{
1620 1621 1622 1623
	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
	dev->ctrl_config |= NVME_CC_ENABLE;
	writel(dev->ctrl_config, &dev->bar->cc);

1624 1625 1626
	return nvme_wait_ready(dev, cap, true);
}

K
Keith Busch 已提交
1627 1628 1629 1630
static int nvme_shutdown_ctrl(struct nvme_dev *dev)
{
	unsigned long timeout;

1631 1632 1633 1634
	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
	dev->ctrl_config |= NVME_CC_SHN_NORMAL;

	writel(dev->ctrl_config, &dev->bar->cc);
K
Keith Busch 已提交
1635

1636
	timeout = SHUTDOWN_TIMEOUT + jiffies;
K
Keith Busch 已提交
1637 1638 1639 1640 1641 1642
	while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) !=
							NVME_CSTS_SHST_CMPLT) {
		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
1643
			dev_err(dev->dev,
K
Keith Busch 已提交
1644 1645 1646 1647 1648 1649 1650 1651
				"Device shutdown incomplete; abort shutdown\n");
			return -ENODEV;
		}
	}

	return 0;
}

M
Matias Bjørling 已提交
1652
static struct blk_mq_ops nvme_mq_admin_ops = {
1653
	.queue_rq	= nvme_queue_rq,
M
Matias Bjørling 已提交
1654 1655
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_admin_init_hctx,
1656
	.exit_hctx      = nvme_admin_exit_hctx,
M
Matias Bjørling 已提交
1657 1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668
	.init_request	= nvme_admin_init_request,
	.timeout	= nvme_timeout,
};

static struct blk_mq_ops nvme_mq_ops = {
	.queue_rq	= nvme_queue_rq,
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_init_hctx,
	.init_request	= nvme_init_request,
	.timeout	= nvme_timeout,
};

1669 1670 1671 1672 1673 1674 1675 1676
static void nvme_dev_remove_admin(struct nvme_dev *dev)
{
	if (dev->admin_q && !blk_queue_dying(dev->admin_q)) {
		blk_cleanup_queue(dev->admin_q);
		blk_mq_free_tag_set(&dev->admin_tagset);
	}
}

M
Matias Bjørling 已提交
1677 1678 1679 1680 1681 1682
static int nvme_alloc_admin_tags(struct nvme_dev *dev)
{
	if (!dev->admin_q) {
		dev->admin_tagset.ops = &nvme_mq_admin_ops;
		dev->admin_tagset.nr_hw_queues = 1;
		dev->admin_tagset.queue_depth = NVME_AQ_DEPTH - 1;
1683
		dev->admin_tagset.reserved_tags = 1;
M
Matias Bjørling 已提交
1684
		dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1685
		dev->admin_tagset.numa_node = dev_to_node(dev->dev);
1686
		dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
M
Matias Bjørling 已提交
1687 1688 1689 1690 1691 1692
		dev->admin_tagset.driver_data = dev;

		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
			return -ENOMEM;

		dev->admin_q = blk_mq_init_queue(&dev->admin_tagset);
1693
		if (IS_ERR(dev->admin_q)) {
M
Matias Bjørling 已提交
1694 1695 1696
			blk_mq_free_tag_set(&dev->admin_tagset);
			return -ENOMEM;
		}
1697 1698
		if (!blk_get_queue(dev->admin_q)) {
			nvme_dev_remove_admin(dev);
1699
			dev->admin_q = NULL;
1700 1701
			return -ENODEV;
		}
K
Keith Busch 已提交
1702 1703
	} else
		blk_mq_unfreeze_queue(dev->admin_q);
M
Matias Bjørling 已提交
1704 1705 1706 1707

	return 0;
}

1708
static int nvme_configure_admin_queue(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1709
{
1710
	int result;
M
Matthew Wilcox 已提交
1711
	u32 aqa;
1712
	u64 cap = readq(&dev->bar->cap);
M
Matthew Wilcox 已提交
1713
	struct nvme_queue *nvmeq;
1714 1715 1716 1717 1718
	unsigned page_shift = PAGE_SHIFT;
	unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;
	unsigned dev_page_max = NVME_CAP_MPSMAX(cap) + 12;

	if (page_shift < dev_page_min) {
1719
		dev_err(dev->dev,
1720 1721 1722 1723 1724 1725
				"Minimum device page size (%u) too large for "
				"host (%u)\n", 1 << dev_page_min,
				1 << page_shift);
		return -ENODEV;
	}
	if (page_shift > dev_page_max) {
1726
		dev_info(dev->dev,
1727 1728 1729 1730 1731
				"Device maximum page size (%u) smaller than "
				"host (%u); enabling work-around\n",
				1 << dev_page_max, 1 << page_shift);
		page_shift = dev_page_max;
	}
M
Matthew Wilcox 已提交
1732

1733 1734 1735 1736 1737 1738
	dev->subsystem = readl(&dev->bar->vs) >= NVME_VS(1, 1) ?
						NVME_CAP_NSSRC(cap) : 0;

	if (dev->subsystem && (readl(&dev->bar->csts) & NVME_CSTS_NSSRO))
		writel(NVME_CSTS_NSSRO, &dev->bar->csts);

1739 1740 1741
	result = nvme_disable_ctrl(dev, cap);
	if (result < 0)
		return result;
M
Matthew Wilcox 已提交
1742

M
Matias Bjørling 已提交
1743
	nvmeq = dev->queues[0];
1744
	if (!nvmeq) {
K
Keith Busch 已提交
1745
		nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1746 1747 1748
		if (!nvmeq)
			return -ENOMEM;
	}
M
Matthew Wilcox 已提交
1749 1750 1751 1752

	aqa = nvmeq->q_depth - 1;
	aqa |= aqa << 16;

1753 1754
	dev->page_size = 1 << page_shift;

1755
	dev->ctrl_config = NVME_CC_CSS_NVM;
1756
	dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
M
Matthew Wilcox 已提交
1757
	dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
1758
	dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
M
Matthew Wilcox 已提交
1759 1760 1761 1762 1763

	writel(aqa, &dev->bar->aqa);
	writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
	writeq(nvmeq->cq_dma_addr, &dev->bar->acq);

1764
	result = nvme_enable_ctrl(dev, cap);
1765
	if (result)
M
Matias Bjørling 已提交
1766 1767
		goto free_nvmeq;

K
Keith Busch 已提交
1768
	nvmeq->cq_vector = 0;
1769
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
1770 1771
	if (result) {
		nvmeq->cq_vector = -1;
K
Keith Busch 已提交
1772
		goto free_nvmeq;
1773
	}
1774

M
Matthew Wilcox 已提交
1775
	return result;
M
Matias Bjørling 已提交
1776 1777 1778 1779

 free_nvmeq:
	nvme_free_queues(dev, 0);
	return result;
M
Matthew Wilcox 已提交
1780 1781
}

M
Matthew Wilcox 已提交
1782 1783 1784 1785 1786
static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
{
	struct nvme_dev *dev = ns->dev;
	struct nvme_user_io io;
	struct nvme_command c;
1787
	unsigned length, meta_len;
1788 1789 1790
	int status, write;
	dma_addr_t meta_dma = 0;
	void *meta = NULL;
A
Arnd Bergmann 已提交
1791
	void __user *metadata;
M
Matthew Wilcox 已提交
1792 1793 1794

	if (copy_from_user(&io, uio, sizeof(io)))
		return -EFAULT;
1795 1796 1797 1798

	switch (io.opcode) {
	case nvme_cmd_write:
	case nvme_cmd_read:
M
Matthew Wilcox 已提交
1799
	case nvme_cmd_compare:
M
Matthew Wilcox 已提交
1800
		break;
1801
	default:
M
Matthew Wilcox 已提交
1802
		return -EINVAL;
1803 1804
	}

1805 1806
	length = (io.nblocks + 1) << ns->lba_shift;
	meta_len = (io.nblocks + 1) * ns->ms;
A
Arnd Bergmann 已提交
1807
	metadata = (void __user *)(uintptr_t)io.metadata;
1808
	write = io.opcode & 1;
M
Matthew Wilcox 已提交
1809

1810 1811 1812
	if (ns->ext) {
		length += meta_len;
		meta_len = 0;
1813 1814
	}
	if (meta_len) {
1815 1816 1817
		if (((io.metadata & 3) || !io.metadata) && !ns->ext)
			return -EINVAL;

1818
		meta = dma_alloc_coherent(dev->dev, meta_len,
1819
						&meta_dma, GFP_KERNEL);
A
Arnd Bergmann 已提交
1820

1821 1822 1823 1824 1825
		if (!meta) {
			status = -ENOMEM;
			goto unmap;
		}
		if (write) {
A
Arnd Bergmann 已提交
1826
			if (copy_from_user(meta, metadata, meta_len)) {
1827 1828 1829 1830 1831 1832
				status = -EFAULT;
				goto unmap;
			}
		}
	}

M
Matthew Wilcox 已提交
1833 1834 1835
	memset(&c, 0, sizeof(c));
	c.rw.opcode = io.opcode;
	c.rw.flags = io.flags;
1836
	c.rw.nsid = cpu_to_le32(ns->ns_id);
M
Matthew Wilcox 已提交
1837
	c.rw.slba = cpu_to_le64(io.slba);
1838
	c.rw.length = cpu_to_le16(io.nblocks);
M
Matthew Wilcox 已提交
1839
	c.rw.control = cpu_to_le16(io.control);
1840 1841 1842 1843
	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
	c.rw.reftag = cpu_to_le32(io.reftag);
	c.rw.apptag = cpu_to_le16(io.apptag);
	c.rw.appmask = cpu_to_le16(io.appmask);
1844
	c.rw.metadata = cpu_to_le64(meta_dma);
1845 1846

	status = __nvme_submit_sync_cmd(ns->queue, &c, NULL,
A
Arnd Bergmann 已提交
1847
			(void __user *)(uintptr_t)io.addr, length, NULL, 0);
1848
 unmap:
1849 1850
	if (meta) {
		if (status == NVME_SC_SUCCESS && !write) {
A
Arnd Bergmann 已提交
1851
			if (copy_to_user(metadata, meta, meta_len))
1852 1853
				status = -EFAULT;
		}
1854
		dma_free_coherent(dev->dev, meta_len, meta, meta_dma);
1855
	}
M
Matthew Wilcox 已提交
1856 1857 1858
	return status;
}

M
Matias Bjørling 已提交
1859 1860
static int nvme_user_cmd(struct nvme_dev *dev, struct nvme_ns *ns,
			struct nvme_passthru_cmd __user *ucmd)
1861
{
1862
	struct nvme_passthru_cmd cmd;
1863
	struct nvme_command c;
1864 1865
	unsigned timeout = 0;
	int status;
1866

M
Matthew Wilcox 已提交
1867 1868 1869
	if (!capable(CAP_SYS_ADMIN))
		return -EACCES;
	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1870 1871 1872
		return -EFAULT;

	memset(&c, 0, sizeof(c));
M
Matthew Wilcox 已提交
1873 1874 1875 1876 1877 1878 1879 1880 1881 1882 1883 1884
	c.common.opcode = cmd.opcode;
	c.common.flags = cmd.flags;
	c.common.nsid = cpu_to_le32(cmd.nsid);
	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
	c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
	c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
	c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
	c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
	c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
	c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);

1885 1886
	if (cmd.timeout_ms)
		timeout = msecs_to_jiffies(cmd.timeout_ms);
1887

1888
	status = __nvme_submit_sync_cmd(ns ? ns->queue : dev->admin_q, &c,
A
Arnd Bergmann 已提交
1889
			NULL, (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1890 1891 1892 1893
			&cmd.result, timeout);
	if (status >= 0) {
		if (put_user(cmd.result, &ucmd->result))
			return -EFAULT;
M
Matthew Wilcox 已提交
1894
	}
1895

1896 1897 1898
	return status;
}

1899 1900 1901 1902 1903 1904 1905 1906 1907
static int nvme_subsys_reset(struct nvme_dev *dev)
{
	if (!dev->subsystem)
		return -ENOTTY;

	writel(0x4E564D65, &dev->bar->nssr); /* "NVMe" */
	return 0;
}

M
Matthew Wilcox 已提交
1908 1909 1910 1911 1912 1913
static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
							unsigned long arg)
{
	struct nvme_ns *ns = bdev->bd_disk->private_data;

	switch (cmd) {
M
Matthew Wilcox 已提交
1914
	case NVME_IOCTL_ID:
1915
		force_successful_syscall_return();
M
Matthew Wilcox 已提交
1916 1917
		return ns->ns_id;
	case NVME_IOCTL_ADMIN_CMD:
M
Matias Bjørling 已提交
1918
		return nvme_user_cmd(ns->dev, NULL, (void __user *)arg);
1919
	case NVME_IOCTL_IO_CMD:
M
Matias Bjørling 已提交
1920
		return nvme_user_cmd(ns->dev, ns, (void __user *)arg);
M
Matthew Wilcox 已提交
1921 1922
	case NVME_IOCTL_SUBMIT_IO:
		return nvme_submit_io(ns, (void __user *)arg);
V
Vishal Verma 已提交
1923 1924 1925 1926
	case SG_GET_VERSION_NUM:
		return nvme_sg_get_version_num((void __user *)arg);
	case SG_IO:
		return nvme_sg_io(ns, (void __user *)arg);
M
Matthew Wilcox 已提交
1927 1928 1929 1930 1931
	default:
		return -ENOTTY;
	}
}

K
Keith Busch 已提交
1932 1933 1934 1935 1936 1937
#ifdef CONFIG_COMPAT
static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
					unsigned int cmd, unsigned long arg)
{
	switch (cmd) {
	case SG_IO:
1938
		return -ENOIOCTLCMD;
K
Keith Busch 已提交
1939 1940 1941 1942 1943 1944 1945
	}
	return nvme_ioctl(bdev, mode, cmd, arg);
}
#else
#define nvme_compat_ioctl	NULL
#endif

1946
static void nvme_free_dev(struct kref *kref);
1947 1948 1949 1950 1951 1952 1953 1954
static void nvme_free_ns(struct kref *kref)
{
	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);

	spin_lock(&dev_list_lock);
	ns->disk->private_data = NULL;
	spin_unlock(&dev_list_lock);

1955
	kref_put(&ns->dev->kref, nvme_free_dev);
1956 1957 1958 1959
	put_disk(ns->disk);
	kfree(ns);
}

1960 1961
static int nvme_open(struct block_device *bdev, fmode_t mode)
{
1962 1963
	int ret = 0;
	struct nvme_ns *ns;
1964

1965 1966 1967 1968
	spin_lock(&dev_list_lock);
	ns = bdev->bd_disk->private_data;
	if (!ns)
		ret = -ENXIO;
1969
	else if (!kref_get_unless_zero(&ns->kref))
1970 1971 1972 1973
		ret = -ENXIO;
	spin_unlock(&dev_list_lock);

	return ret;
1974 1975 1976 1977 1978
}

static void nvme_release(struct gendisk *disk, fmode_t mode)
{
	struct nvme_ns *ns = disk->private_data;
1979
	kref_put(&ns->kref, nvme_free_ns);
1980 1981
}

K
Keith Busch 已提交
1982 1983 1984 1985 1986 1987 1988 1989 1990
static int nvme_getgeo(struct block_device *bd, struct hd_geometry *geo)
{
	/* some standard values */
	geo->heads = 1 << 6;
	geo->sectors = 1 << 5;
	geo->cylinders = get_capacity(bd->bd_disk) >> 11;
	return 0;
}

K
Keith Busch 已提交
1991 1992 1993 1994 1995 1996
static void nvme_config_discard(struct nvme_ns *ns)
{
	u32 logical_block_size = queue_logical_block_size(ns->queue);
	ns->queue->limits.discard_zeroes_data = 0;
	ns->queue->limits.discard_alignment = logical_block_size;
	ns->queue->limits.discard_granularity = logical_block_size;
1997
	blk_queue_max_discard_sectors(ns->queue, 0xffffffff);
K
Keith Busch 已提交
1998 1999 2000
	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
}

K
Keith Busch 已提交
2001 2002 2003 2004 2005
static int nvme_revalidate_disk(struct gendisk *disk)
{
	struct nvme_ns *ns = disk->private_data;
	struct nvme_dev *dev = ns->dev;
	struct nvme_id_ns *id;
2006 2007
	u8 lbaf, pi_type;
	u16 old_ms;
K
Keith Busch 已提交
2008
	unsigned short bs;
K
Keith Busch 已提交
2009

2010
	if (nvme_identify_ns(dev, ns->ns_id, &id)) {
K
Keith Busch 已提交
2011 2012 2013
		dev_warn(dev->dev, "%s: Identify failure nvme%dn%d\n", __func__,
						dev->instance, ns->ns_id);
		return -ENODEV;
K
Keith Busch 已提交
2014
	}
K
Keith Busch 已提交
2015 2016 2017
	if (id->ncap == 0) {
		kfree(id);
		return -ENODEV;
K
Keith Busch 已提交
2018
	}
K
Keith Busch 已提交
2019

K
Keith Busch 已提交
2020 2021
	old_ms = ns->ms;
	lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
K
Keith Busch 已提交
2022
	ns->lba_shift = id->lbaf[lbaf].ds;
K
Keith Busch 已提交
2023
	ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
2024
	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
K
Keith Busch 已提交
2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037

	/*
	 * If identify namespace failed, use default 512 byte block size so
	 * block layer can use before failing read/write for 0 capacity.
	 */
	if (ns->lba_shift == 0)
		ns->lba_shift = 9;
	bs = 1 << ns->lba_shift;

	/* XXX: PI implementation requires metadata equal t10 pi tuple size */
	pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
					id->dps & NVME_NS_DPS_PI_MASK : 0;

2038 2039
	if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
				ns->ms != old_ms ||
K
Keith Busch 已提交
2040
				bs != queue_logical_block_size(disk->queue) ||
2041
				(ns->ms && ns->ext)))
K
Keith Busch 已提交
2042 2043 2044 2045 2046
		blk_integrity_unregister(disk);

	ns->pi_type = pi_type;
	blk_queue_logical_block_size(ns->queue, bs);

2047
	if (ns->ms && !ns->ext)
K
Keith Busch 已提交
2048 2049
		nvme_init_integrity(ns);

2050
	if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk))
K
Keith Busch 已提交
2051 2052 2053 2054 2055 2056
		set_capacity(disk, 0);
	else
		set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));

	if (dev->oncs & NVME_CTRL_ONCS_DSM)
		nvme_config_discard(ns);
K
Keith Busch 已提交
2057

2058
	kfree(id);
K
Keith Busch 已提交
2059 2060 2061
	return 0;
}

M
Matthew Wilcox 已提交
2062 2063 2064
static const struct block_device_operations nvme_fops = {
	.owner		= THIS_MODULE,
	.ioctl		= nvme_ioctl,
K
Keith Busch 已提交
2065
	.compat_ioctl	= nvme_compat_ioctl,
2066 2067
	.open		= nvme_open,
	.release	= nvme_release,
K
Keith Busch 已提交
2068
	.getgeo		= nvme_getgeo,
K
Keith Busch 已提交
2069
	.revalidate_disk= nvme_revalidate_disk,
M
Matthew Wilcox 已提交
2070 2071
};

2072 2073
static int nvme_kthread(void *data)
{
2074
	struct nvme_dev *dev, *next;
2075 2076

	while (!kthread_should_stop()) {
2077
		set_current_state(TASK_INTERRUPTIBLE);
2078
		spin_lock(&dev_list_lock);
2079
		list_for_each_entry_safe(dev, next, &dev_list, node) {
2080
			int i;
2081 2082 2083 2084
			u32 csts = readl(&dev->bar->csts);

			if ((dev->subsystem && (csts & NVME_CSTS_NSSRO)) ||
							csts & NVME_CSTS_CFS) {
2085 2086 2087 2088 2089
				if (!__nvme_reset(dev)) {
					dev_warn(dev->dev,
						"Failed status: %x, reset controller\n",
						readl(&dev->bar->csts));
				}
2090 2091
				continue;
			}
2092
			for (i = 0; i < dev->queue_count; i++) {
M
Matias Bjørling 已提交
2093
				struct nvme_queue *nvmeq = dev->queues[i];
2094 2095
				if (!nvmeq)
					continue;
2096
				spin_lock_irq(&nvmeq->q_lock);
2097
				nvme_process_cq(nvmeq);
K
Keith Busch 已提交
2098 2099

				while ((i == 0) && (dev->event_limit > 0)) {
M
Matias Bjørling 已提交
2100
					if (nvme_submit_async_admin_req(dev))
K
Keith Busch 已提交
2101 2102 2103
						break;
					dev->event_limit--;
				}
2104 2105 2106 2107
				spin_unlock_irq(&nvmeq->q_lock);
			}
		}
		spin_unlock(&dev_list_lock);
2108
		schedule_timeout(round_jiffies_relative(HZ));
2109 2110 2111 2112
	}
	return 0;
}

K
Keith Busch 已提交
2113
static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
M
Matthew Wilcox 已提交
2114 2115 2116
{
	struct nvme_ns *ns;
	struct gendisk *disk;
2117
	int node = dev_to_node(dev->dev);
M
Matthew Wilcox 已提交
2118

M
Matias Bjørling 已提交
2119
	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2120
	if (!ns)
K
Keith Busch 已提交
2121 2122
		return;

M
Matias Bjørling 已提交
2123
	ns->queue = blk_mq_init_queue(&dev->tagset);
2124
	if (IS_ERR(ns->queue))
M
Matthew Wilcox 已提交
2125
		goto out_free_ns;
M
Matthew Wilcox 已提交
2126 2127
	queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
M
Matthew Wilcox 已提交
2128 2129 2130
	ns->dev = dev;
	ns->queue->queuedata = ns;

M
Matias Bjørling 已提交
2131
	disk = alloc_disk_node(0, node);
M
Matthew Wilcox 已提交
2132 2133
	if (!disk)
		goto out_free_queue;
M
Matias Bjørling 已提交
2134

2135
	kref_init(&ns->kref);
2136
	ns->ns_id = nsid;
M
Matthew Wilcox 已提交
2137
	ns->disk = disk;
K
Keith Busch 已提交
2138 2139 2140
	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
	list_add_tail(&ns->list, &dev->namespaces);

2141
	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
K
Keith Busch 已提交
2142
	if (dev->max_hw_sectors) {
K
Keith Busch 已提交
2143
		blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
K
Keith Busch 已提交
2144 2145 2146
		blk_queue_max_segments(ns->queue,
			((dev->max_hw_sectors << 9) / dev->page_size) + 1);
	}
M
Matias Bjørling 已提交
2147 2148
	if (dev->stripe_size)
		blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
2149 2150
	if (dev->vwc & NVME_CTRL_VWC_PRESENT)
		blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
2151
	blk_queue_virt_boundary(ns->queue, dev->page_size - 1);
M
Matthew Wilcox 已提交
2152 2153

	disk->major = nvme_major;
2154
	disk->first_minor = 0;
M
Matthew Wilcox 已提交
2155 2156 2157
	disk->fops = &nvme_fops;
	disk->private_data = ns;
	disk->queue = ns->queue;
2158
	disk->driverfs_dev = dev->device;
2159
	disk->flags = GENHD_FL_EXT_DEVT;
2160
	sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
M
Matthew Wilcox 已提交
2161

K
Keith Busch 已提交
2162 2163 2164 2165 2166 2167 2168
	/*
	 * Initialize capacity to 0 until we establish the namespace format and
	 * setup integrity extentions if necessary. The revalidate_disk after
	 * add_disk allows the driver to register with integrity if the format
	 * requires it.
	 */
	set_capacity(disk, 0);
K
Keith Busch 已提交
2169 2170 2171
	if (nvme_revalidate_disk(ns->disk))
		goto out_free_disk;

2172
	kref_get(&dev->kref);
K
Keith Busch 已提交
2173
	add_disk(ns->disk);
2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184
	if (ns->ms) {
		struct block_device *bd = bdget_disk(ns->disk, 0);
		if (!bd)
			return;
		if (blkdev_get(bd, FMODE_READ, NULL)) {
			bdput(bd);
			return;
		}
		blkdev_reread_part(bd);
		blkdev_put(bd, FMODE_READ);
	}
K
Keith Busch 已提交
2185
	return;
K
Keith Busch 已提交
2186 2187 2188
 out_free_disk:
	kfree(disk);
	list_del(&ns->list);
M
Matthew Wilcox 已提交
2189 2190 2191 2192 2193 2194
 out_free_queue:
	blk_cleanup_queue(ns->queue);
 out_free_ns:
	kfree(ns);
}

2195 2196 2197 2198 2199 2200 2201
/*
 * Create I/O queues.  Failing to create an I/O queue is not an issue,
 * we can continue with less than the desired amount of queues, and
 * even a controller without I/O queues an still be used to issue
 * admin commands.  This might be useful to upgrade a buggy firmware
 * for example.
 */
K
Keith Busch 已提交
2202 2203
static void nvme_create_io_queues(struct nvme_dev *dev)
{
M
Matias Bjørling 已提交
2204
	unsigned i;
K
Keith Busch 已提交
2205

M
Matias Bjørling 已提交
2206
	for (i = dev->queue_count; i <= dev->max_qid; i++)
K
Keith Busch 已提交
2207
		if (!nvme_alloc_queue(dev, i, dev->q_depth))
K
Keith Busch 已提交
2208 2209
			break;

M
Matias Bjørling 已提交
2210
	for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
2211 2212
		if (nvme_create_queue(dev->queues[i], i)) {
			nvme_free_queues(dev, i);
K
Keith Busch 已提交
2213
			break;
2214
		}
K
Keith Busch 已提交
2215 2216
}

2217
static int set_queue_count(struct nvme_dev *dev, int count)
M
Matthew Wilcox 已提交
2218 2219 2220
{
	int status;
	u32 result;
2221
	u32 q_count = (count - 1) | ((count - 1) << 16);
M
Matthew Wilcox 已提交
2222

2223
	status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
2224
								&result);
M
Matthew Wilcox 已提交
2225 2226 2227
	if (status < 0)
		return status;
	if (status > 0) {
2228
		dev_err(dev->dev, "Could not set queue count (%d)\n", status);
2229
		return 0;
M
Matthew Wilcox 已提交
2230
	}
M
Matthew Wilcox 已提交
2231 2232 2233
	return min(result & 0xffff, result >> 16) + 1;
}

2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259 2260 2261 2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285
static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
{
	u64 szu, size, offset;
	u32 cmbloc;
	resource_size_t bar_size;
	struct pci_dev *pdev = to_pci_dev(dev->dev);
	void __iomem *cmb;
	dma_addr_t dma_addr;

	if (!use_cmb_sqes)
		return NULL;

	dev->cmbsz = readl(&dev->bar->cmbsz);
	if (!(NVME_CMB_SZ(dev->cmbsz)))
		return NULL;

	cmbloc = readl(&dev->bar->cmbloc);

	szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
	size = szu * NVME_CMB_SZ(dev->cmbsz);
	offset = szu * NVME_CMB_OFST(cmbloc);
	bar_size = pci_resource_len(pdev, NVME_CMB_BIR(cmbloc));

	if (offset > bar_size)
		return NULL;

	/*
	 * Controllers may support a CMB size larger than their BAR,
	 * for example, due to being behind a bridge. Reduce the CMB to
	 * the reported size of the BAR
	 */
	if (size > bar_size - offset)
		size = bar_size - offset;

	dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(cmbloc)) + offset;
	cmb = ioremap_wc(dma_addr, size);
	if (!cmb)
		return NULL;

	dev->cmb_dma_addr = dma_addr;
	dev->cmb_size = size;
	return cmb;
}

static inline void nvme_release_cmb(struct nvme_dev *dev)
{
	if (dev->cmb) {
		iounmap(dev->cmb);
		dev->cmb = NULL;
	}
}

K
Keith Busch 已提交
2286 2287
static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
{
2288
	return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
K
Keith Busch 已提交
2289 2290
}

2291
static int nvme_setup_io_queues(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2292
{
M
Matias Bjørling 已提交
2293
	struct nvme_queue *adminq = dev->queues[0];
2294
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
2295
	int result, i, vecs, nr_io_queues, size;
M
Matthew Wilcox 已提交
2296

K
Keith Busch 已提交
2297
	nr_io_queues = num_possible_cpus();
2298
	result = set_queue_count(dev, nr_io_queues);
2299
	if (result <= 0)
M
Matthew Wilcox 已提交
2300
		return result;
2301 2302
	if (result < nr_io_queues)
		nr_io_queues = result;
M
Matthew Wilcox 已提交
2303

2304 2305 2306 2307 2308 2309 2310 2311 2312
	if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
		result = nvme_cmb_qdepth(dev, nr_io_queues,
				sizeof(struct nvme_command));
		if (result > 0)
			dev->q_depth = result;
		else
			nvme_release_cmb(dev);
	}

K
Keith Busch 已提交
2313 2314
	size = db_bar_size(dev, nr_io_queues);
	if (size > 8192) {
2315
		iounmap(dev->bar);
K
Keith Busch 已提交
2316 2317 2318 2319 2320 2321 2322 2323
		do {
			dev->bar = ioremap(pci_resource_start(pdev, 0), size);
			if (dev->bar)
				break;
			if (!--nr_io_queues)
				return -ENOMEM;
			size = db_bar_size(dev, nr_io_queues);
		} while (1);
2324
		dev->dbs = ((void __iomem *)dev->bar) + 4096;
2325
		adminq->q_db = dev->dbs;
2326 2327
	}

K
Keith Busch 已提交
2328
	/* Deregister the admin queue's interrupt */
2329
	free_irq(dev->entry[0].vector, adminq);
K
Keith Busch 已提交
2330

2331 2332 2333 2334 2335 2336 2337
	/*
	 * If we enable msix early due to not intx, disable it again before
	 * setting up the full range we need.
	 */
	if (!pdev->irq)
		pci_disable_msix(pdev);

2338
	for (i = 0; i < nr_io_queues; i++)
M
Matthew Wilcox 已提交
2339
		dev->entry[i].entry = i;
2340 2341 2342 2343 2344 2345 2346 2347
	vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
	if (vecs < 0) {
		vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
		if (vecs < 0) {
			vecs = 1;
		} else {
			for (i = 0; i < vecs; i++)
				dev->entry[i].vector = i + pdev->irq;
R
Ramachandra Rao Gajula 已提交
2348 2349 2350
		}
	}

2351 2352 2353 2354 2355 2356 2357
	/*
	 * Should investigate if there's a performance win from allocating
	 * more queues than interrupt vectors; it might allow the submission
	 * path to scale better, even if the receive path is limited by the
	 * number of interrupts.
	 */
	nr_io_queues = vecs;
K
Keith Busch 已提交
2358
	dev->max_qid = nr_io_queues;
2359

2360
	result = queue_request_irq(dev, adminq, adminq->irqname);
2361 2362
	if (result) {
		adminq->cq_vector = -1;
2363
		goto free_queues;
2364
	}
M
Matthew Wilcox 已提交
2365

2366
	/* Free previously allocated queues that are no longer usable */
K
Keith Busch 已提交
2367
	nvme_free_queues(dev, nr_io_queues + 1);
M
Matias Bjørling 已提交
2368
	nvme_create_io_queues(dev);
M
Matthew Wilcox 已提交
2369

2370
	return 0;
M
Matthew Wilcox 已提交
2371

2372
 free_queues:
2373
	nvme_free_queues(dev, 1);
2374
	return result;
M
Matthew Wilcox 已提交
2375 2376
}

K
Keith Busch 已提交
2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416 2417
static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
	struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);

	return nsa->ns_id - nsb->ns_id;
}

static struct nvme_ns *nvme_find_ns(struct nvme_dev *dev, unsigned nsid)
{
	struct nvme_ns *ns;

	list_for_each_entry(ns, &dev->namespaces, list) {
		if (ns->ns_id == nsid)
			return ns;
		if (ns->ns_id > nsid)
			break;
	}
	return NULL;
}

static inline bool nvme_io_incapable(struct nvme_dev *dev)
{
	return (!dev->bar || readl(&dev->bar->csts) & NVME_CSTS_CFS ||
							dev->online_queues < 2);
}

static void nvme_ns_remove(struct nvme_ns *ns)
{
	bool kill = nvme_io_incapable(ns->dev) && !blk_queue_dying(ns->queue);

	if (kill)
		blk_set_queue_dying(ns->queue);
	if (ns->disk->flags & GENHD_FL_UP) {
		if (blk_get_integrity(ns->disk))
			blk_integrity_unregister(ns->disk);
		del_gendisk(ns->disk);
	}
	if (kill || !blk_queue_dying(ns->queue)) {
		blk_mq_abort_requeue_list(ns->queue);
		blk_cleanup_queue(ns->queue);
2418 2419 2420
	}
	list_del_init(&ns->list);
	kref_put(&ns->kref, nvme_free_ns);
K
Keith Busch 已提交
2421 2422 2423 2424 2425 2426 2427 2428 2429 2430
}

static void nvme_scan_namespaces(struct nvme_dev *dev, unsigned nn)
{
	struct nvme_ns *ns, *next;
	unsigned i;

	for (i = 1; i <= nn; i++) {
		ns = nvme_find_ns(dev, i);
		if (ns) {
2431
			if (revalidate_disk(ns->disk))
K
Keith Busch 已提交
2432 2433 2434 2435 2436
				nvme_ns_remove(ns);
		} else
			nvme_alloc_ns(dev, i);
	}
	list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
2437
		if (ns->ns_id > nn)
K
Keith Busch 已提交
2438 2439 2440 2441 2442
			nvme_ns_remove(ns);
	}
	list_sort(NULL, &dev->namespaces, ns_cmp);
}

2443 2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456 2457 2458
static void nvme_set_irq_hints(struct nvme_dev *dev)
{
	struct nvme_queue *nvmeq;
	int i;

	for (i = 0; i < dev->online_queues; i++) {
		nvmeq = dev->queues[i];

		if (!nvmeq->tags || !(*nvmeq->tags))
			continue;

		irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
					blk_mq_tags_cpumask(*nvmeq->tags));
	}
}

K
Keith Busch 已提交
2459 2460 2461 2462 2463 2464 2465 2466 2467 2468 2469
static void nvme_dev_scan(struct work_struct *work)
{
	struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);
	struct nvme_id_ctrl *ctrl;

	if (!dev->tagset.tags)
		return;
	if (nvme_identify_ctrl(dev, &ctrl))
		return;
	nvme_scan_namespaces(dev, le32_to_cpup(&ctrl->nn));
	kfree(ctrl);
2470
	nvme_set_irq_hints(dev);
K
Keith Busch 已提交
2471 2472
}

2473 2474 2475 2476 2477 2478
/*
 * Return: error value if an error occurred setting up the queues or calling
 * Identify Device.  0 if these succeeded, even if adding some of the
 * namespaces failed.  At the moment, these failures are silent.  TBD which
 * failures should be reported.
 */
2479
static int nvme_dev_add(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2480
{
2481
	struct pci_dev *pdev = to_pci_dev(dev->dev);
2482
	int res;
2483
	struct nvme_id_ctrl *ctrl;
2484
	int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
M
Matthew Wilcox 已提交
2485

2486
	res = nvme_identify_ctrl(dev, &ctrl);
M
Matthew Wilcox 已提交
2487
	if (res) {
2488
		dev_err(dev->dev, "Identify Controller failed (%d)\n", res);
K
Keith Busch 已提交
2489
		return -EIO;
M
Matthew Wilcox 已提交
2490 2491
	}

2492
	dev->oncs = le16_to_cpup(&ctrl->oncs);
K
Keith Busch 已提交
2493
	dev->abort_limit = ctrl->acl + 1;
2494
	dev->vwc = ctrl->vwc;
2495 2496 2497
	memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
	memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
	memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
2498
	if (ctrl->mdts)
K
Keith Busch 已提交
2499
		dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
2500
	if ((pdev->vendor == PCI_VENDOR_ID_INTEL) &&
M
Matias Bjørling 已提交
2501 2502 2503
			(pdev->device == 0x0953) && ctrl->vs[3]) {
		unsigned int max_hw_sectors;

2504
		dev->stripe_size = 1 << (ctrl->vs[3] + shift);
M
Matias Bjørling 已提交
2505 2506 2507 2508 2509 2510 2511
		max_hw_sectors = dev->stripe_size >> (shift - 9);
		if (dev->max_hw_sectors) {
			dev->max_hw_sectors = min(max_hw_sectors,
							dev->max_hw_sectors);
		} else
			dev->max_hw_sectors = max_hw_sectors;
	}
2512
	kfree(ctrl);
M
Matias Bjørling 已提交
2513

2514 2515 2516 2517 2518 2519
	if (!dev->tagset.tags) {
		dev->tagset.ops = &nvme_mq_ops;
		dev->tagset.nr_hw_queues = dev->online_queues - 1;
		dev->tagset.timeout = NVME_IO_TIMEOUT;
		dev->tagset.numa_node = dev_to_node(dev->dev);
		dev->tagset.queue_depth =
M
Matias Bjørling 已提交
2520
				min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
2521 2522 2523
		dev->tagset.cmd_size = nvme_cmd_size(dev);
		dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
		dev->tagset.driver_data = dev;
M
Matthew Wilcox 已提交
2524

2525 2526 2527
		if (blk_mq_alloc_tag_set(&dev->tagset))
			return 0;
	}
K
Keith Busch 已提交
2528
	schedule_work(&dev->scan_work);
K
Keith Busch 已提交
2529
	return 0;
M
Matthew Wilcox 已提交
2530 2531
}

2532 2533
static int nvme_dev_map(struct nvme_dev *dev)
{
K
Keith Busch 已提交
2534
	u64 cap;
2535
	int bars, result = -ENOMEM;
2536
	struct pci_dev *pdev = to_pci_dev(dev->dev);
2537 2538 2539 2540 2541 2542 2543

	if (pci_enable_device_mem(pdev))
		return result;

	dev->entry[0].vector = pdev->irq;
	pci_set_master(pdev);
	bars = pci_select_bars(pdev, IORESOURCE_MEM);
2544 2545 2546
	if (!bars)
		goto disable_pci;

2547 2548 2549
	if (pci_request_selected_regions(pdev, bars, "nvme"))
		goto disable_pci;

2550 2551
	if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
	    dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
2552
		goto disable;
2553 2554 2555 2556

	dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
	if (!dev->bar)
		goto disable;
2557

K
Keith Busch 已提交
2558 2559 2560 2561
	if (readl(&dev->bar->csts) == -1) {
		result = -ENODEV;
		goto unmap;
	}
2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572

	/*
	 * Some devices don't advertse INTx interrupts, pre-enable a single
	 * MSIX vec for setup. We'll adjust this later.
	 */
	if (!pdev->irq) {
		result = pci_enable_msix(pdev, dev->entry, 1);
		if (result < 0)
			goto unmap;
	}

K
Keith Busch 已提交
2573 2574 2575
	cap = readq(&dev->bar->cap);
	dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
	dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
2576
	dev->dbs = ((void __iomem *)dev->bar) + 4096;
2577 2578
	if (readl(&dev->bar->vs) >= NVME_VS(1, 2))
		dev->cmb = nvme_map_cmb(dev);
2579 2580 2581

	return 0;

K
Keith Busch 已提交
2582 2583 2584
 unmap:
	iounmap(dev->bar);
	dev->bar = NULL;
2585 2586 2587 2588 2589 2590 2591 2592 2593
 disable:
	pci_release_regions(pdev);
 disable_pci:
	pci_disable_device(pdev);
	return result;
}

static void nvme_dev_unmap(struct nvme_dev *dev)
{
2594 2595 2596 2597 2598 2599
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	if (pdev->msi_enabled)
		pci_disable_msi(pdev);
	else if (pdev->msix_enabled)
		pci_disable_msix(pdev);
2600 2601 2602 2603

	if (dev->bar) {
		iounmap(dev->bar);
		dev->bar = NULL;
2604
		pci_release_regions(pdev);
2605 2606
	}

2607 2608
	if (pci_is_enabled(pdev))
		pci_disable_device(pdev);
2609 2610
}

K
Keith Busch 已提交
2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621 2622 2623 2624 2625 2626 2627
struct nvme_delq_ctx {
	struct task_struct *waiter;
	struct kthread_worker *worker;
	atomic_t refcount;
};

static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
{
	dq->waiter = current;
	mb();

	for (;;) {
		set_current_state(TASK_KILLABLE);
		if (!atomic_read(&dq->refcount))
			break;
		if (!schedule_timeout(ADMIN_TIMEOUT) ||
					fatal_signal_pending(current)) {
K
Keith Busch 已提交
2628 2629 2630 2631 2632 2633 2634
			/*
			 * Disable the controller first since we can't trust it
			 * at this point, but leave the admin queue enabled
			 * until all queue deletion requests are flushed.
			 * FIXME: This may take a while if there are more h/w
			 * queues than admin tags.
			 */
K
Keith Busch 已提交
2635 2636
			set_current_state(TASK_RUNNING);
			nvme_disable_ctrl(dev, readq(&dev->bar->cap));
K
Keith Busch 已提交
2637
			nvme_clear_queue(dev->queues[0]);
K
Keith Busch 已提交
2638
			flush_kthread_worker(dq->worker);
K
Keith Busch 已提交
2639
			nvme_disable_queue(dev, 0);
K
Keith Busch 已提交
2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651 2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674
			return;
		}
	}
	set_current_state(TASK_RUNNING);
}

static void nvme_put_dq(struct nvme_delq_ctx *dq)
{
	atomic_dec(&dq->refcount);
	if (dq->waiter)
		wake_up_process(dq->waiter);
}

static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq)
{
	atomic_inc(&dq->refcount);
	return dq;
}

static void nvme_del_queue_end(struct nvme_queue *nvmeq)
{
	struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx;
	nvme_put_dq(dq);
}

static int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode,
						kthread_work_func_t fn)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(nvmeq->qid);

	init_kthread_work(&nvmeq->cmdinfo.work, fn);
M
Matias Bjørling 已提交
2675 2676
	return nvme_submit_admin_async_cmd(nvmeq->dev, &c, &nvmeq->cmdinfo,
								ADMIN_TIMEOUT);
K
Keith Busch 已提交
2677 2678 2679 2680 2681 2682 2683 2684 2685 2686 2687 2688 2689 2690 2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718 2719 2720 2721 2722 2723 2724 2725 2726
}

static void nvme_del_cq_work_handler(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	nvme_del_queue_end(nvmeq);
}

static int nvme_delete_cq(struct nvme_queue *nvmeq)
{
	return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq,
						nvme_del_cq_work_handler);
}

static void nvme_del_sq_work_handler(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	int status = nvmeq->cmdinfo.status;

	if (!status)
		status = nvme_delete_cq(nvmeq);
	if (status)
		nvme_del_queue_end(nvmeq);
}

static int nvme_delete_sq(struct nvme_queue *nvmeq)
{
	return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq,
						nvme_del_sq_work_handler);
}

static void nvme_del_queue_start(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	if (nvme_delete_sq(nvmeq))
		nvme_del_queue_end(nvmeq);
}

static void nvme_disable_io_queues(struct nvme_dev *dev)
{
	int i;
	DEFINE_KTHREAD_WORKER_ONSTACK(worker);
	struct nvme_delq_ctx dq;
	struct task_struct *kworker_task = kthread_run(kthread_worker_fn,
					&worker, "nvme%d", dev->instance);

	if (IS_ERR(kworker_task)) {
2727
		dev_err(dev->dev,
K
Keith Busch 已提交
2728 2729 2730 2731 2732 2733 2734 2735 2736 2737
			"Failed to create queue del task\n");
		for (i = dev->queue_count - 1; i > 0; i--)
			nvme_disable_queue(dev, i);
		return;
	}

	dq.waiter = NULL;
	atomic_set(&dq.refcount, 0);
	dq.worker = &worker;
	for (i = dev->queue_count - 1; i > 0; i--) {
M
Matias Bjørling 已提交
2738
		struct nvme_queue *nvmeq = dev->queues[i];
K
Keith Busch 已提交
2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750

		if (nvme_suspend_queue(nvmeq))
			continue;
		nvmeq->cmdinfo.ctx = nvme_get_dq(&dq);
		nvmeq->cmdinfo.worker = dq.worker;
		init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start);
		queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work);
	}
	nvme_wait_dq(&dq, dev);
	kthread_stop(kworker_task);
}

2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770
/*
* Remove the node from the device list and check
* for whether or not we need to stop the nvme_thread.
*/
static void nvme_dev_list_remove(struct nvme_dev *dev)
{
	struct task_struct *tmp = NULL;

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) {
		tmp = nvme_thread;
		nvme_thread = NULL;
	}
	spin_unlock(&dev_list_lock);

	if (tmp)
		kthread_stop(tmp);
}

2771 2772 2773 2774 2775 2776 2777
static void nvme_freeze_queues(struct nvme_dev *dev)
{
	struct nvme_ns *ns;

	list_for_each_entry(ns, &dev->namespaces, list) {
		blk_mq_freeze_queue_start(ns->queue);

2778
		spin_lock_irq(ns->queue->queue_lock);
2779
		queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
2780
		spin_unlock_irq(ns->queue->queue_lock);
2781 2782 2783 2784 2785 2786 2787 2788 2789 2790 2791 2792 2793 2794 2795 2796 2797 2798

		blk_mq_cancel_requeue_work(ns->queue);
		blk_mq_stop_hw_queues(ns->queue);
	}
}

static void nvme_unfreeze_queues(struct nvme_dev *dev)
{
	struct nvme_ns *ns;

	list_for_each_entry(ns, &dev->namespaces, list) {
		queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
		blk_mq_unfreeze_queue(ns->queue);
		blk_mq_start_stopped_hw_queues(ns->queue, true);
		blk_mq_kick_requeue_list(ns->queue);
	}
}

2799
static void nvme_dev_shutdown(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2800
{
2801
	int i;
2802
	u32 csts = -1;
2803

2804
	nvme_dev_list_remove(dev);
2805

2806 2807
	if (dev->bar) {
		nvme_freeze_queues(dev);
2808
		csts = readl(&dev->bar->csts);
2809
	}
2810
	if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
K
Keith Busch 已提交
2811
		for (i = dev->queue_count - 1; i >= 0; i--) {
M
Matias Bjørling 已提交
2812
			struct nvme_queue *nvmeq = dev->queues[i];
K
Keith Busch 已提交
2813 2814 2815 2816
			nvme_suspend_queue(nvmeq);
		}
	} else {
		nvme_disable_io_queues(dev);
K
Keith Busch 已提交
2817
		nvme_shutdown_ctrl(dev);
K
Keith Busch 已提交
2818 2819
		nvme_disable_queue(dev, 0);
	}
2820
	nvme_dev_unmap(dev);
2821 2822 2823

	for (i = dev->queue_count - 1; i >= 0; i--)
		nvme_clear_queue(dev->queues[i]);
2824 2825 2826 2827
}

static void nvme_dev_remove(struct nvme_dev *dev)
{
2828
	struct nvme_ns *ns, *next;
2829

2830
	list_for_each_entry_safe(ns, next, &dev->namespaces, list)
K
Keith Busch 已提交
2831
		nvme_ns_remove(ns);
M
Matthew Wilcox 已提交
2832 2833
}

M
Matthew Wilcox 已提交
2834 2835
static int nvme_setup_prp_pools(struct nvme_dev *dev)
{
2836
	dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
M
Matthew Wilcox 已提交
2837 2838 2839 2840
						PAGE_SIZE, PAGE_SIZE, 0);
	if (!dev->prp_page_pool)
		return -ENOMEM;

2841
	/* Optimisation for I/Os between 4k and 128k */
2842
	dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2843 2844 2845 2846 2847
						256, 256, 0);
	if (!dev->prp_small_pool) {
		dma_pool_destroy(dev->prp_page_pool);
		return -ENOMEM;
	}
M
Matthew Wilcox 已提交
2848 2849 2850 2851 2852 2853
	return 0;
}

static void nvme_release_prp_pools(struct nvme_dev *dev)
{
	dma_pool_destroy(dev->prp_page_pool);
2854
	dma_pool_destroy(dev->prp_small_pool);
M
Matthew Wilcox 已提交
2855 2856
}

2857 2858 2859
static DEFINE_IDA(nvme_instance_ida);

static int nvme_set_instance(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2860
{
2861 2862 2863 2864 2865 2866 2867 2868 2869 2870 2871 2872 2873 2874 2875 2876
	int instance, error;

	do {
		if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
			return -ENODEV;

		spin_lock(&dev_list_lock);
		error = ida_get_new(&nvme_instance_ida, &instance);
		spin_unlock(&dev_list_lock);
	} while (error == -EAGAIN);

	if (error)
		return -ENODEV;

	dev->instance = instance;
	return 0;
M
Matthew Wilcox 已提交
2877 2878 2879 2880
}

static void nvme_release_instance(struct nvme_dev *dev)
{
2881 2882 2883
	spin_lock(&dev_list_lock);
	ida_remove(&nvme_instance_ida, dev->instance);
	spin_unlock(&dev_list_lock);
M
Matthew Wilcox 已提交
2884 2885
}

2886 2887 2888
static void nvme_free_dev(struct kref *kref)
{
	struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
2889

2890
	put_device(dev->dev);
2891
	put_device(dev->device);
2892
	nvme_release_instance(dev);
2893 2894 2895 2896
	if (dev->tagset.tags)
		blk_mq_free_tag_set(&dev->tagset);
	if (dev->admin_q)
		blk_put_queue(dev->admin_q);
2897 2898 2899 2900 2901 2902 2903
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
}

static int nvme_dev_open(struct inode *inode, struct file *f)
{
2904 2905 2906 2907 2908 2909 2910
	struct nvme_dev *dev;
	int instance = iminor(inode);
	int ret = -ENODEV;

	spin_lock(&dev_list_lock);
	list_for_each_entry(dev, &dev_list, node) {
		if (dev->instance == instance) {
2911 2912 2913 2914
			if (!dev->admin_q) {
				ret = -EWOULDBLOCK;
				break;
			}
2915 2916 2917 2918 2919 2920 2921 2922 2923 2924
			if (!kref_get_unless_zero(&dev->kref))
				break;
			f->private_data = dev;
			ret = 0;
			break;
		}
	}
	spin_unlock(&dev_list_lock);

	return ret;
2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936
}

static int nvme_dev_release(struct inode *inode, struct file *f)
{
	struct nvme_dev *dev = f->private_data;
	kref_put(&dev->kref, nvme_free_dev);
	return 0;
}

static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
{
	struct nvme_dev *dev = f->private_data;
M
Matias Bjørling 已提交
2937 2938
	struct nvme_ns *ns;

2939 2940
	switch (cmd) {
	case NVME_IOCTL_ADMIN_CMD:
M
Matias Bjørling 已提交
2941
		return nvme_user_cmd(dev, NULL, (void __user *)arg);
2942
	case NVME_IOCTL_IO_CMD:
M
Matias Bjørling 已提交
2943 2944 2945 2946
		if (list_empty(&dev->namespaces))
			return -ENOTTY;
		ns = list_first_entry(&dev->namespaces, struct nvme_ns, list);
		return nvme_user_cmd(dev, ns, (void __user *)arg);
2947 2948 2949
	case NVME_IOCTL_RESET:
		dev_warn(dev->dev, "resetting controller\n");
		return nvme_reset(dev);
2950 2951
	case NVME_IOCTL_SUBSYS_RESET:
		return nvme_subsys_reset(dev);
2952 2953 2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964
	default:
		return -ENOTTY;
	}
}

static const struct file_operations nvme_dev_fops = {
	.owner		= THIS_MODULE,
	.open		= nvme_dev_open,
	.release	= nvme_dev_release,
	.unlocked_ioctl	= nvme_dev_ioctl,
	.compat_ioctl	= nvme_dev_ioctl,
};

2965
static void nvme_probe_work(struct work_struct *work)
2966
{
2967
	struct nvme_dev *dev = container_of(work, struct nvme_dev, probe_work);
2968
	bool start_thread = false;
2969
	int result;
2970 2971 2972

	result = nvme_dev_map(dev);
	if (result)
2973
		goto out;
2974 2975 2976 2977 2978 2979

	result = nvme_configure_admin_queue(dev);
	if (result)
		goto unmap;

	spin_lock(&dev_list_lock);
2980 2981 2982 2983
	if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
		start_thread = true;
		nvme_thread = NULL;
	}
2984 2985 2986
	list_add(&dev->node, &dev_list);
	spin_unlock(&dev_list_lock);

2987 2988
	if (start_thread) {
		nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
2989
		wake_up_all(&nvme_kthread_wait);
2990 2991 2992 2993 2994 2995 2996
	} else
		wait_event_killable(nvme_kthread_wait, nvme_thread);

	if (IS_ERR_OR_NULL(nvme_thread)) {
		result = nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;
		goto disable;
	}
M
Matias Bjørling 已提交
2997 2998

	nvme_init_queue(dev->queues[0], 0);
K
Keith Busch 已提交
2999 3000 3001
	result = nvme_alloc_admin_tags(dev);
	if (result)
		goto disable;
3002

3003
	result = nvme_setup_io_queues(dev);
3004
	if (result)
K
Keith Busch 已提交
3005
		goto free_tags;
3006

3007
	dev->event_limit = 1;
3008

3009 3010 3011 3012
	/*
	 * Keep the controller around but remove all namespaces if we don't have
	 * any working I/O queue.
	 */
3013 3014 3015 3016 3017 3018 3019 3020 3021
	if (dev->online_queues < 2) {
		dev_warn(dev->dev, "IO queues not created\n");
		nvme_dev_remove(dev);
	} else {
		nvme_unfreeze_queues(dev);
		nvme_dev_add(dev);
	}

	return;
3022

K
Keith Busch 已提交
3023 3024
 free_tags:
	nvme_dev_remove_admin(dev);
3025 3026 3027
	blk_put_queue(dev->admin_q);
	dev->admin_q = NULL;
	dev->queues[0]->tags = NULL;
3028
 disable:
3029
	nvme_disable_queue(dev, 0);
3030
	nvme_dev_list_remove(dev);
3031 3032
 unmap:
	nvme_dev_unmap(dev);
3033 3034 3035
 out:
	if (!work_busy(&dev->reset_work))
		nvme_dead_ctrl(dev);
3036 3037
}

K
Keith Busch 已提交
3038 3039 3040
static int nvme_remove_dead_ctrl(void *arg)
{
	struct nvme_dev *dev = (struct nvme_dev *)arg;
3041
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
3042 3043

	if (pci_get_drvdata(pdev))
3044
		pci_stop_and_remove_bus_device_locked(pdev);
K
Keith Busch 已提交
3045 3046 3047 3048
	kref_put(&dev->kref, nvme_free_dev);
	return 0;
}

3049 3050 3051 3052 3053 3054 3055 3056 3057 3058 3059 3060
static void nvme_dead_ctrl(struct nvme_dev *dev)
{
	dev_warn(dev->dev, "Device failed to resume\n");
	kref_get(&dev->kref);
	if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d",
						dev->instance))) {
		dev_err(dev->dev,
			"Failed to start controller remove task\n");
		kref_put(&dev->kref, nvme_free_dev);
	}
}

3061
static void nvme_reset_work(struct work_struct *ws)
K
Keith Busch 已提交
3062
{
3063
	struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
3064 3065
	bool in_probe = work_busy(&dev->probe_work);

K
Keith Busch 已提交
3066
	nvme_dev_shutdown(dev);
3067 3068 3069 3070 3071 3072 3073 3074

	/* Synchronize with device probe so that work will see failure status
	 * and exit gracefully without trying to schedule another reset */
	flush_work(&dev->probe_work);

	/* Fail this device if reset occured during probe to avoid
	 * infinite initialization loops. */
	if (in_probe) {
3075
		nvme_dead_ctrl(dev);
3076
		return;
K
Keith Busch 已提交
3077
	}
3078 3079 3080
	/* Schedule device resume asynchronously so the reset work is available
	 * to cleanup errors that may occur during reinitialization */
	schedule_work(&dev->probe_work);
K
Keith Busch 已提交
3081 3082
}

3083 3084 3085 3086 3087 3088 3089 3090 3091
static int __nvme_reset(struct nvme_dev *dev)
{
	if (work_pending(&dev->reset_work))
		return -EBUSY;
	list_del_init(&dev->node);
	queue_work(nvme_workq, &dev->reset_work);
	return 0;
}

3092 3093
static int nvme_reset(struct nvme_dev *dev)
{
3094
	int ret;
3095 3096 3097 3098 3099

	if (!dev->admin_q || blk_queue_dying(dev->admin_q))
		return -ENODEV;

	spin_lock(&dev_list_lock);
3100
	ret = __nvme_reset(dev);
3101 3102 3103 3104
	spin_unlock(&dev_list_lock);

	if (!ret) {
		flush_work(&dev->reset_work);
3105
		flush_work(&dev->probe_work);
3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126
		return 0;
	}

	return ret;
}

static ssize_t nvme_sysfs_reset(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_dev *ndev = dev_get_drvdata(dev);
	int ret;

	ret = nvme_reset(ndev);
	if (ret < 0)
		return ret;

	return count;
}
static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);

3127
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
M
Matthew Wilcox 已提交
3128
{
M
Matias Bjørling 已提交
3129
	int node, result = -ENOMEM;
M
Matthew Wilcox 已提交
3130 3131
	struct nvme_dev *dev;

M
Matias Bjørling 已提交
3132 3133 3134 3135 3136
	node = dev_to_node(&pdev->dev);
	if (node == NUMA_NO_NODE)
		set_dev_node(&pdev->dev, 0);

	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
3137 3138
	if (!dev)
		return -ENOMEM;
M
Matias Bjørling 已提交
3139 3140
	dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
3141 3142
	if (!dev->entry)
		goto free;
M
Matias Bjørling 已提交
3143 3144
	dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
3145 3146 3147 3148
	if (!dev->queues)
		goto free;

	INIT_LIST_HEAD(&dev->namespaces);
3149
	INIT_WORK(&dev->reset_work, nvme_reset_work);
3150
	dev->dev = get_device(&pdev->dev);
K
Keith Busch 已提交
3151
	pci_set_drvdata(pdev, dev);
3152 3153
	result = nvme_set_instance(dev);
	if (result)
K
Keith Busch 已提交
3154
		goto put_pci;
M
Matthew Wilcox 已提交
3155

M
Matthew Wilcox 已提交
3156 3157
	result = nvme_setup_prp_pools(dev);
	if (result)
3158
		goto release;
M
Matthew Wilcox 已提交
3159

3160
	kref_init(&dev->kref);
3161 3162 3163 3164 3165
	dev->device = device_create(nvme_class, &pdev->dev,
				MKDEV(nvme_char_major, dev->instance),
				dev, "nvme%d", dev->instance);
	if (IS_ERR(dev->device)) {
		result = PTR_ERR(dev->device);
3166
		goto release_pools;
3167 3168
	}
	get_device(dev->device);
3169 3170 3171 3172 3173
	dev_set_drvdata(dev->device, dev);

	result = device_create_file(dev->device, &dev_attr_reset_controller);
	if (result)
		goto put_dev;
3174

3175
	INIT_LIST_HEAD(&dev->node);
K
Keith Busch 已提交
3176
	INIT_WORK(&dev->scan_work, nvme_dev_scan);
3177
	INIT_WORK(&dev->probe_work, nvme_probe_work);
3178
	schedule_work(&dev->probe_work);
M
Matthew Wilcox 已提交
3179 3180
	return 0;

3181 3182 3183
 put_dev:
	device_destroy(nvme_class, MKDEV(nvme_char_major, dev->instance));
	put_device(dev->device);
3184
 release_pools:
M
Matthew Wilcox 已提交
3185
	nvme_release_prp_pools(dev);
3186 3187
 release:
	nvme_release_instance(dev);
K
Keith Busch 已提交
3188
 put_pci:
3189
	put_device(dev->dev);
M
Matthew Wilcox 已提交
3190 3191 3192 3193 3194 3195 3196
 free:
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
	return result;
}

3197 3198
static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
{
K
Keith Busch 已提交
3199
	struct nvme_dev *dev = pci_get_drvdata(pdev);
3200

K
Keith Busch 已提交
3201 3202 3203
	if (prepare)
		nvme_dev_shutdown(dev);
	else
3204
		schedule_work(&dev->probe_work);
3205 3206
}

3207 3208 3209 3210 3211 3212
static void nvme_shutdown(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
	nvme_dev_shutdown(dev);
}

3213
static void nvme_remove(struct pci_dev *pdev)
M
Matthew Wilcox 已提交
3214 3215
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
K
Keith Busch 已提交
3216 3217 3218 3219 3220 3221

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	spin_unlock(&dev_list_lock);

	pci_set_drvdata(pdev, NULL);
3222
	flush_work(&dev->probe_work);
K
Keith Busch 已提交
3223
	flush_work(&dev->reset_work);
K
Keith Busch 已提交
3224
	flush_work(&dev->scan_work);
3225
	device_remove_file(dev->device, &dev_attr_reset_controller);
3226
	nvme_dev_remove(dev);
3227
	nvme_dev_shutdown(dev);
M
Matias Bjørling 已提交
3228
	nvme_dev_remove_admin(dev);
3229
	device_destroy(nvme_class, MKDEV(nvme_char_major, dev->instance));
3230
	nvme_free_queues(dev, 0);
3231
	nvme_release_cmb(dev);
K
Keith Busch 已提交
3232
	nvme_release_prp_pools(dev);
3233
	kref_put(&dev->kref, nvme_free_dev);
M
Matthew Wilcox 已提交
3234 3235 3236 3237 3238 3239 3240 3241
}

/* These functions are yet to be implemented */
#define nvme_error_detected NULL
#define nvme_dump_registers NULL
#define nvme_link_reset NULL
#define nvme_slot_reset NULL
#define nvme_error_resume NULL
3242

3243
#ifdef CONFIG_PM_SLEEP
3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257
static int nvme_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

	nvme_dev_shutdown(ndev);
	return 0;
}

static int nvme_resume(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

3258
	schedule_work(&ndev->probe_work);
K
Keith Busch 已提交
3259
	return 0;
3260
}
3261
#endif
3262 3263

static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
M
Matthew Wilcox 已提交
3264

3265
static const struct pci_error_handlers nvme_err_handler = {
M
Matthew Wilcox 已提交
3266 3267 3268 3269 3270
	.error_detected	= nvme_error_detected,
	.mmio_enabled	= nvme_dump_registers,
	.link_reset	= nvme_link_reset,
	.slot_reset	= nvme_slot_reset,
	.resume		= nvme_error_resume,
3271
	.reset_notify	= nvme_reset_notify,
M
Matthew Wilcox 已提交
3272 3273 3274 3275 3276
};

/* Move to pci_ids.h later */
#define PCI_CLASS_STORAGE_EXPRESS	0x010802

3277
static const struct pci_device_id nvme_id_table[] = {
M
Matthew Wilcox 已提交
3278 3279 3280 3281 3282 3283 3284 3285 3286
	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, nvme_id_table);

static struct pci_driver nvme_driver = {
	.name		= "nvme",
	.id_table	= nvme_id_table,
	.probe		= nvme_probe,
3287
	.remove		= nvme_remove,
3288
	.shutdown	= nvme_shutdown,
3289 3290 3291
	.driver		= {
		.pm	= &nvme_dev_pm_ops,
	},
M
Matthew Wilcox 已提交
3292 3293 3294 3295 3296
	.err_handler	= &nvme_err_handler,
};

static int __init nvme_init(void)
{
3297
	int result;
3298

3299
	init_waitqueue_head(&nvme_kthread_wait);
M
Matthew Wilcox 已提交
3300

K
Keith Busch 已提交
3301 3302
	nvme_workq = create_singlethread_workqueue("nvme");
	if (!nvme_workq)
3303
		return -ENOMEM;
K
Keith Busch 已提交
3304

3305 3306
	result = register_blkdev(nvme_major, "nvme");
	if (result < 0)
K
Keith Busch 已提交
3307
		goto kill_workq;
3308
	else if (result > 0)
3309
		nvme_major = result;
M
Matthew Wilcox 已提交
3310

3311 3312 3313 3314 3315 3316 3317 3318
	result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
							&nvme_dev_fops);
	if (result < 0)
		goto unregister_blkdev;
	else if (result > 0)
		nvme_char_major = result;

	nvme_class = class_create(THIS_MODULE, "nvme");
3319 3320
	if (IS_ERR(nvme_class)) {
		result = PTR_ERR(nvme_class);
3321
		goto unregister_chrdev;
3322
	}
3323

3324 3325
	result = pci_register_driver(&nvme_driver);
	if (result)
3326
		goto destroy_class;
3327
	return 0;
M
Matthew Wilcox 已提交
3328

3329 3330 3331 3332
 destroy_class:
	class_destroy(nvme_class);
 unregister_chrdev:
	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
3333
 unregister_blkdev:
M
Matthew Wilcox 已提交
3334
	unregister_blkdev(nvme_major, "nvme");
K
Keith Busch 已提交
3335 3336
 kill_workq:
	destroy_workqueue(nvme_workq);
M
Matthew Wilcox 已提交
3337 3338 3339 3340 3341 3342 3343
	return result;
}

static void __exit nvme_exit(void)
{
	pci_unregister_driver(&nvme_driver);
	unregister_blkdev(nvme_major, "nvme");
K
Keith Busch 已提交
3344
	destroy_workqueue(nvme_workq);
3345 3346
	class_destroy(nvme_class);
	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
3347
	BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
3348
	_nvme_check_size();
M
Matthew Wilcox 已提交
3349 3350 3351 3352
}

MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
MODULE_LICENSE("GPL");
3353
MODULE_VERSION("1.0");
M
Matthew Wilcox 已提交
3354 3355
module_init(nvme_init);
module_exit(nvme_exit);