pci.c 84.8 KB
Newer Older
M
Matthew Wilcox 已提交
1 2
/*
 * NVM Express device driver
3
 * Copyright (c) 2011-2014, Intel Corporation.
M
Matthew Wilcox 已提交
4 5 6 7 8 9 10 11 12 13 14
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

15
#include <linux/bitops.h>
M
Matthew Wilcox 已提交
16
#include <linux/blkdev.h>
M
Matias Bjørling 已提交
17
#include <linux/blk-mq.h>
K
Keith Busch 已提交
18
#include <linux/cpu.h>
19
#include <linux/delay.h>
M
Matthew Wilcox 已提交
20 21 22
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/genhd.h>
K
Keith Busch 已提交
23
#include <linux/hdreg.h>
24
#include <linux/idr.h>
M
Matthew Wilcox 已提交
25 26 27 28
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kdev_t.h>
29
#include <linux/kthread.h>
M
Matthew Wilcox 已提交
30
#include <linux/kernel.h>
K
Keith Busch 已提交
31
#include <linux/list_sort.h>
M
Matthew Wilcox 已提交
32 33 34 35
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pci.h>
36
#include <linux/poison.h>
37
#include <linux/ptrace.h>
M
Matthew Wilcox 已提交
38 39
#include <linux/sched.h>
#include <linux/slab.h>
K
Keith Busch 已提交
40
#include <linux/t10-pi.h>
M
Matthew Wilcox 已提交
41
#include <linux/types.h>
K
Keith Busch 已提交
42
#include <linux/pr.h>
V
Vishal Verma 已提交
43
#include <scsi/sg.h>
44
#include <linux/io-64-nonatomic-lo-hi.h>
K
Keith Busch 已提交
45
#include <asm/unaligned.h>
46

47
#include <uapi/linux/nvme_ioctl.h>
48 49
#include "nvme.h"

50
#define NVME_MINORS		(1U << MINORBITS)
51
#define NVME_Q_DEPTH		1024
J
Jens Axboe 已提交
52
#define NVME_AQ_DEPTH		256
M
Matthew Wilcox 已提交
53 54
#define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))
#define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))
55
#define SHUTDOWN_TIMEOUT	(shutdown_timeout * HZ)
56

57
unsigned char admin_timeout = 60;
58 59
module_param(admin_timeout, byte, 0644);
MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
M
Matthew Wilcox 已提交
60

61 62
unsigned char nvme_io_timeout = 30;
module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
63
MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
M
Matthew Wilcox 已提交
64

65 66 67 68
static unsigned char shutdown_timeout = 5;
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");

M
Matthew Wilcox 已提交
69 70 71
static int nvme_major;
module_param(nvme_major, int, 0);

72 73 74
static int nvme_char_major;
module_param(nvme_char_major, int, 0);

75 76 77
static int use_threaded_interrupts;
module_param(use_threaded_interrupts, int, 0);

78 79 80 81
static bool use_cmb_sqes = true;
module_param(use_cmb_sqes, bool, 0644);
MODULE_PARM_DESC(use_cmb_sqes, "use controller's memory buffer for I/O SQes");

82 83 84
static DEFINE_SPINLOCK(dev_list_lock);
static LIST_HEAD(dev_list);
static struct task_struct *nvme_thread;
K
Keith Busch 已提交
85
static struct workqueue_struct *nvme_workq;
86
static wait_queue_head_t nvme_kthread_wait;
87

88 89
static struct class *nvme_class;

90 91 92
struct nvme_dev;
struct nvme_queue;

93
static int __nvme_reset(struct nvme_dev *dev);
94
static int nvme_reset(struct nvme_dev *dev);
J
Jens Axboe 已提交
95
static void nvme_process_cq(struct nvme_queue *nvmeq);
96
static void nvme_dead_ctrl(struct nvme_dev *dev);
97

K
Keith Busch 已提交
98 99 100
struct async_cmd_info {
	struct kthread_work work;
	struct kthread_worker *worker;
M
Matias Bjørling 已提交
101
	struct request *req;
K
Keith Busch 已提交
102 103 104 105
	u32 result;
	int status;
	void *ctx;
};
106

107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149
/*
 * Represents an NVM Express device.  Each nvme_dev is a PCI function.
 */
struct nvme_dev {
	struct list_head node;
	struct nvme_queue **queues;
	struct blk_mq_tag_set tagset;
	struct blk_mq_tag_set admin_tagset;
	u32 __iomem *dbs;
	struct device *dev;
	struct dma_pool *prp_page_pool;
	struct dma_pool *prp_small_pool;
	unsigned queue_count;
	unsigned online_queues;
	unsigned max_qid;
	int q_depth;
	u32 db_stride;
	u32 ctrl_config;
	struct msix_entry *entry;
	void __iomem *bar;
	struct list_head namespaces;
	struct kref kref;
	struct device *device;
	struct work_struct reset_work;
	struct work_struct probe_work;
	struct work_struct scan_work;
	bool subsystem;
	u32 max_hw_sectors;
	u32 stripe_size;
	u32 page_size;
	void __iomem *cmb;
	dma_addr_t cmb_dma_addr;
	u64 cmb_size;
	u32 cmbsz;

	struct nvme_ctrl ctrl;
};

static inline struct nvme_dev *to_nvme_dev(struct nvme_ctrl *ctrl)
{
	return container_of(ctrl, struct nvme_dev, ctrl);
}

M
Matthew Wilcox 已提交
150 151 152 153 154 155
/*
 * An NVM Express queue.  Each device has at least two (one for admin
 * commands and one for I/O commands).
 */
struct nvme_queue {
	struct device *q_dmadev;
M
Matthew Wilcox 已提交
156
	struct nvme_dev *dev;
157
	char irqname[24];	/* nvme4294967295-65535\0 */
M
Matthew Wilcox 已提交
158 159
	spinlock_t q_lock;
	struct nvme_command *sq_cmds;
160
	struct nvme_command __iomem *sq_cmds_io;
M
Matthew Wilcox 已提交
161
	volatile struct nvme_completion *cqes;
162
	struct blk_mq_tags **tags;
M
Matthew Wilcox 已提交
163 164 165 166
	dma_addr_t sq_dma_addr;
	dma_addr_t cq_dma_addr;
	u32 __iomem *q_db;
	u16 q_depth;
J
Jens Axboe 已提交
167
	s16 cq_vector;
M
Matthew Wilcox 已提交
168 169 170
	u16 sq_head;
	u16 sq_tail;
	u16 cq_head;
K
Keith Busch 已提交
171
	u16 qid;
172 173
	u8 cq_phase;
	u8 cqe_seen;
K
Keith Busch 已提交
174
	struct async_cmd_info cmdinfo;
M
Matthew Wilcox 已提交
175 176
};

177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193
/*
 * The nvme_iod describes the data in an I/O, including the list of PRP
 * entries.  You can't see it in this data structure because C doesn't let
 * me express that.  Use nvme_alloc_iod to ensure there's enough space
 * allocated to store the PRP list.
 */
struct nvme_iod {
	unsigned long private;	/* For the use of the submitter of the I/O */
	int npages;		/* In the PRP list. 0 means small pool in use */
	int offset;		/* Of PRP list */
	int nents;		/* Used in scatterlist */
	int length;		/* Of data, in bytes */
	dma_addr_t first_dma;
	struct scatterlist meta_sg[1]; /* metadata requires single contiguous buffer */
	struct scatterlist sg[0];
};

M
Matthew Wilcox 已提交
194 195 196 197 198 199 200 201 202 203
/*
 * Check we didin't inadvertently grow the command struct
 */
static inline void _nvme_check_size(void)
{
	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
204
	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
K
Keith Busch 已提交
205
	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
M
Matthew Wilcox 已提交
206 207 208 209
	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
K
Keith Busch 已提交
210
	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
M
Matthew Wilcox 已提交
211 212
}

213
typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
214 215
						struct nvme_completion *);

216
struct nvme_cmd_info {
217 218
	nvme_completion_fn fn;
	void *ctx;
K
Keith Busch 已提交
219
	int aborted;
M
Matias Bjørling 已提交
220
	struct nvme_queue *nvmeq;
221
	struct nvme_iod iod[0];
222 223
};

224 225 226 227 228
/*
 * Max size of iod being embedded in the request payload
 */
#define NVME_INT_PAGES		2
#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->page_size)
C
Chong Yuan 已提交
229
#define NVME_INT_MASK		0x01
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252

/*
 * Will slightly overestimate the number of pages needed.  This is OK
 * as it only leads to a small amount of wasted memory for the lifetime of
 * the I/O.
 */
static int nvme_npages(unsigned size, struct nvme_dev *dev)
{
	unsigned nprps = DIV_ROUND_UP(size + dev->page_size, dev->page_size);
	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}

static unsigned int nvme_cmd_size(struct nvme_dev *dev)
{
	unsigned int ret = sizeof(struct nvme_cmd_info);

	ret += sizeof(struct nvme_iod);
	ret += sizeof(__le64 *) * nvme_npages(NVME_INT_BYTES(dev), dev);
	ret += sizeof(struct scatterlist) * NVME_INT_PAGES;

	return ret;
}

M
Matias Bjørling 已提交
253 254
static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
				unsigned int hctx_idx)
255
{
M
Matias Bjørling 已提交
256 257 258
	struct nvme_dev *dev = data;
	struct nvme_queue *nvmeq = dev->queues[0];

259 260 261 262
	WARN_ON(hctx_idx != 0);
	WARN_ON(dev->admin_tagset.tags[0] != hctx->tags);
	WARN_ON(nvmeq->tags);

M
Matias Bjørling 已提交
263
	hctx->driver_data = nvmeq;
264
	nvmeq->tags = &dev->admin_tagset.tags[0];
M
Matias Bjørling 已提交
265
	return 0;
266 267
}

268 269 270 271 272 273 274
static void nvme_admin_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	nvmeq->tags = NULL;
}

M
Matias Bjørling 已提交
275 276 277
static int nvme_admin_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
278
{
M
Matias Bjørling 已提交
279 280 281 282 283 284 285
	struct nvme_dev *dev = data;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = dev->queues[0];

	BUG_ON(!nvmeq);
	cmd->nvmeq = nvmeq;
	return 0;
286 287
}

M
Matias Bjørling 已提交
288 289
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			  unsigned int hctx_idx)
M
Matthew Wilcox 已提交
290
{
M
Matias Bjørling 已提交
291
	struct nvme_dev *dev = data;
292
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];
M
Matias Bjørling 已提交
293

294 295
	if (!nvmeq->tags)
		nvmeq->tags = &dev->tagset.tags[hctx_idx];
M
Matthew Wilcox 已提交
296

297
	WARN_ON(dev->tagset.tags[hctx_idx] != hctx->tags);
M
Matias Bjørling 已提交
298 299
	hctx->driver_data = nvmeq;
	return 0;
M
Matthew Wilcox 已提交
300 301
}

M
Matias Bjørling 已提交
302 303 304
static int nvme_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
M
Matthew Wilcox 已提交
305
{
M
Matias Bjørling 已提交
306 307 308 309 310 311 312 313 314 315 316 317 318 319 320
	struct nvme_dev *dev = data;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];

	BUG_ON(!nvmeq);
	cmd->nvmeq = nvmeq;
	return 0;
}

static void nvme_set_info(struct nvme_cmd_info *cmd, void *ctx,
				nvme_completion_fn handler)
{
	cmd->fn = handler;
	cmd->ctx = ctx;
	cmd->aborted = 0;
K
Keith Busch 已提交
321
	blk_mq_start_request(blk_mq_rq_from_pdu(cmd));
M
Matthew Wilcox 已提交
322 323
}

324 325 326 327 328 329 330 331 332 333
static void *iod_get_private(struct nvme_iod *iod)
{
	return (void *) (iod->private & ~0x1UL);
}

/*
 * If bit 0 is set, the iod is embedded in the request payload.
 */
static bool iod_should_kfree(struct nvme_iod *iod)
{
C
Chong Yuan 已提交
334
	return (iod->private & NVME_INT_MASK) == 0;
335 336
}

337 338
/* Special values must be less than 0x1000 */
#define CMD_CTX_BASE		((void *)POISON_POINTER_DELTA)
339 340 341
#define CMD_CTX_CANCELLED	(0x30C + CMD_CTX_BASE)
#define CMD_CTX_COMPLETED	(0x310 + CMD_CTX_BASE)
#define CMD_CTX_INVALID		(0x314 + CMD_CTX_BASE)
342

343
static void special_completion(struct nvme_queue *nvmeq, void *ctx,
344 345 346 347 348
						struct nvme_completion *cqe)
{
	if (ctx == CMD_CTX_CANCELLED)
		return;
	if (ctx == CMD_CTX_COMPLETED) {
349
		dev_warn(nvmeq->q_dmadev,
350 351 352 353 354
				"completed id %d twice on queue %d\n",
				cqe->command_id, le16_to_cpup(&cqe->sq_id));
		return;
	}
	if (ctx == CMD_CTX_INVALID) {
355
		dev_warn(nvmeq->q_dmadev,
356 357 358 359
				"invalid id %d completed on queue %d\n",
				cqe->command_id, le16_to_cpup(&cqe->sq_id));
		return;
	}
360
	dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
361 362
}

M
Matias Bjørling 已提交
363
static void *cancel_cmd_info(struct nvme_cmd_info *cmd, nvme_completion_fn *fn)
M
Matthew Wilcox 已提交
364
{
365
	void *ctx;
M
Matthew Wilcox 已提交
366

367
	if (fn)
M
Matias Bjørling 已提交
368 369 370 371
		*fn = cmd->fn;
	ctx = cmd->ctx;
	cmd->fn = special_completion;
	cmd->ctx = CMD_CTX_CANCELLED;
372
	return ctx;
M
Matthew Wilcox 已提交
373 374
}

M
Matias Bjørling 已提交
375 376
static void async_req_completion(struct nvme_queue *nvmeq, void *ctx,
						struct nvme_completion *cqe)
377
{
M
Matias Bjørling 已提交
378 379 380 381
	u32 result = le32_to_cpup(&cqe->result);
	u16 status = le16_to_cpup(&cqe->status) >> 1;

	if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
382
		++nvmeq->dev->ctrl.event_limit;
K
Keith Busch 已提交
383 384 385 386 387 388 389 390 391 392
	if (status != NVME_SC_SUCCESS)
		return;

	switch (result & 0xff07) {
	case NVME_AER_NOTICE_NS_CHANGED:
		dev_info(nvmeq->q_dmadev, "rescanning\n");
		schedule_work(&nvmeq->dev->scan_work);
	default:
		dev_warn(nvmeq->q_dmadev, "async event result %08x\n", result);
	}
M
Matthew Wilcox 已提交
393 394
}

M
Matias Bjørling 已提交
395 396
static void abort_completion(struct nvme_queue *nvmeq, void *ctx,
						struct nvme_completion *cqe)
397
{
M
Matias Bjørling 已提交
398 399 400 401
	struct request *req = ctx;

	u16 status = le16_to_cpup(&cqe->status) >> 1;
	u32 result = le32_to_cpup(&cqe->result);
402

403
	blk_mq_free_request(req);
404

M
Matias Bjørling 已提交
405
	dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
406
	++nvmeq->dev->ctrl.abort_limit;
407 408
}

M
Matias Bjørling 已提交
409 410
static void async_completion(struct nvme_queue *nvmeq, void *ctx,
						struct nvme_completion *cqe)
M
Matthew Wilcox 已提交
411
{
M
Matias Bjørling 已提交
412 413 414 415
	struct async_cmd_info *cmdinfo = ctx;
	cmdinfo->result = le32_to_cpup(&cqe->result);
	cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
	queue_kthread_work(cmdinfo->worker, &cmdinfo->work);
416
	blk_mq_free_request(cmdinfo->req);
M
Matthew Wilcox 已提交
417 418
}

M
Matias Bjørling 已提交
419 420
static inline struct nvme_cmd_info *get_cmd_from_tag(struct nvme_queue *nvmeq,
				  unsigned int tag)
M
Matthew Wilcox 已提交
421
{
422
	struct request *req = blk_mq_tag_to_rq(*nvmeq->tags, tag);
423

M
Matias Bjørling 已提交
424
	return blk_mq_rq_to_pdu(req);
425 426
}

M
Matias Bjørling 已提交
427 428 429 430 431
/*
 * Called with local interrupts disabled and the q_lock held.  May not sleep.
 */
static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag,
						nvme_completion_fn *fn)
432
{
M
Matias Bjørling 已提交
433 434 435 436 437 438 439 440 441 442 443 444
	struct nvme_cmd_info *cmd = get_cmd_from_tag(nvmeq, tag);
	void *ctx;
	if (tag >= nvmeq->q_depth) {
		*fn = special_completion;
		return CMD_CTX_INVALID;
	}
	if (fn)
		*fn = cmd->fn;
	ctx = cmd->ctx;
	cmd->fn = special_completion;
	cmd->ctx = CMD_CTX_COMPLETED;
	return ctx;
M
Matthew Wilcox 已提交
445 446 447
}

/**
448
 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
M
Matthew Wilcox 已提交
449 450 451 452 453
 * @nvmeq: The queue to use
 * @cmd: The command to send
 *
 * Safe to use from interrupt context
 */
454 455
static void __nvme_submit_cmd(struct nvme_queue *nvmeq,
						struct nvme_command *cmd)
M
Matthew Wilcox 已提交
456
{
M
Matias Bjørling 已提交
457 458
	u16 tail = nvmeq->sq_tail;

459 460 461 462 463
	if (nvmeq->sq_cmds_io)
		memcpy_toio(&nvmeq->sq_cmds_io[tail], cmd, sizeof(*cmd));
	else
		memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));

M
Matthew Wilcox 已提交
464 465
	if (++tail == nvmeq->q_depth)
		tail = 0;
466
	writel(tail, nvmeq->q_db);
M
Matthew Wilcox 已提交
467 468 469
	nvmeq->sq_tail = tail;
}

470
static void nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
M
Matias Bjørling 已提交
471 472 473
{
	unsigned long flags;
	spin_lock_irqsave(&nvmeq->q_lock, flags);
474
	__nvme_submit_cmd(nvmeq, cmd);
M
Matias Bjørling 已提交
475 476 477
	spin_unlock_irqrestore(&nvmeq->q_lock, flags);
}

478
static __le64 **iod_list(struct nvme_iod *iod)
479
{
480
	return ((void *)iod) + iod->offset;
481 482
}

483 484
static inline void iod_init(struct nvme_iod *iod, unsigned nbytes,
			    unsigned nseg, unsigned long private)
485
{
486 487 488 489 490
	iod->private = private;
	iod->offset = offsetof(struct nvme_iod, sg[nseg]);
	iod->npages = -1;
	iod->length = nbytes;
	iod->nents = 0;
491
}
M
Matthew Wilcox 已提交
492

493
static struct nvme_iod *
494 495
__nvme_alloc_iod(unsigned nseg, unsigned bytes, struct nvme_dev *dev,
		 unsigned long priv, gfp_t gfp)
M
Matthew Wilcox 已提交
496
{
497
	struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
498
				sizeof(__le64 *) * nvme_npages(bytes, dev) +
499 500
				sizeof(struct scatterlist) * nseg, gfp);

501 502
	if (iod)
		iod_init(iod, bytes, nseg, priv);
503 504

	return iod;
M
Matthew Wilcox 已提交
505 506
}

507 508 509 510 511 512 513 514 515 516 517 518 519
static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
			               gfp_t gfp)
{
	unsigned size = !(rq->cmd_flags & REQ_DISCARD) ? blk_rq_bytes(rq) :
                                                sizeof(struct nvme_dsm_range);
	struct nvme_iod *iod;

	if (rq->nr_phys_segments <= NVME_INT_PAGES &&
	    size <= NVME_INT_BYTES(dev)) {
		struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);

		iod = cmd->iod;
		iod_init(iod, size, rq->nr_phys_segments,
C
Chong Yuan 已提交
520
				(unsigned long) rq | NVME_INT_MASK);
521 522 523 524 525 526 527
		return iod;
	}

	return __nvme_alloc_iod(rq->nr_phys_segments, size, dev,
				(unsigned long) rq, gfp);
}

528
static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
M
Matthew Wilcox 已提交
529
{
530
	const int last_prp = dev->page_size / 8 - 1;
531 532 533 534 535 536 537 538 539 540 541 542
	int i;
	__le64 **list = iod_list(iod);
	dma_addr_t prp_dma = iod->first_dma;

	if (iod->npages == 0)
		dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
	for (i = 0; i < iod->npages; i++) {
		__le64 *prp_list = list[i];
		dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
		dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
		prp_dma = next_prp_dma;
	}
543 544 545

	if (iod_should_kfree(iod))
		kfree(iod);
M
Matthew Wilcox 已提交
546 547
}

K
Keith Busch 已提交
548 549 550 551 552 553 554 555 556 557 558 559
static int nvme_error_status(u16 status)
{
	switch (status & 0x7ff) {
	case NVME_SC_SUCCESS:
		return 0;
	case NVME_SC_CAP_EXCEEDED:
		return -ENOSPC;
	default:
		return -EIO;
	}
}

560
#ifdef CONFIG_BLK_DEV_INTEGRITY
K
Keith Busch 已提交
561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == v)
		pi->ref_tag = cpu_to_be32(p);
}

static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == p)
		pi->ref_tag = cpu_to_be32(v);
}

/**
 * nvme_dif_remap - remaps ref tags to bip seed and physical lba
 *
 * The virtual start sector is the one that was originally submitted by the
 * block layer.	Due to partitioning, MD/DM cloning, etc. the actual physical
 * start sector may be different. Remap protection information to match the
 * physical LBA on writes, and back to the original seed on reads.
 *
 * Type 0 and 3 do not have a ref tag, so no remapping required.
 */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
	struct nvme_ns *ns = req->rq_disk->private_data;
	struct bio_integrity_payload *bip;
	struct t10_pi_tuple *pi;
	void *p, *pmap;
	u32 i, nlb, ts, phys, virt;

	if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
		return;

	bip = bio_integrity(req->bio);
	if (!bip)
		return;

	pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;

	p = pmap;
	virt = bip_get_seed(bip);
	phys = nvme_block_nr(ns, blk_rq_pos(req));
	nlb = (blk_rq_bytes(req) >> ns->lba_shift);
605
	ts = ns->disk->queue->integrity.tuple_size;
K
Keith Busch 已提交
606 607 608 609 610 611 612 613 614

	for (i = 0; i < nlb; i++, virt++, phys++) {
		pi = (struct t10_pi_tuple *)p;
		dif_swap(phys, virt, pi);
		p += ts;
	}
	kunmap_atomic(pmap);
}

615 616 617 618 619 620
static void nvme_init_integrity(struct nvme_ns *ns)
{
	struct blk_integrity integrity;

	switch (ns->pi_type) {
	case NVME_NS_DPS_PI_TYPE3:
621
		integrity.profile = &t10_pi_type3_crc;
622 623 624
		break;
	case NVME_NS_DPS_PI_TYPE1:
	case NVME_NS_DPS_PI_TYPE2:
625
		integrity.profile = &t10_pi_type1_crc;
626 627
		break;
	default:
628
		integrity.profile = NULL;
629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650
		break;
	}
	integrity.tuple_size = ns->ms;
	blk_integrity_register(ns->disk, &integrity);
	blk_queue_max_integrity_segments(ns->queue, 1);
}
#else /* CONFIG_BLK_DEV_INTEGRITY */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
}
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
}
static void nvme_init_integrity(struct nvme_ns *ns)
{
}
#endif

M
Matias Bjørling 已提交
651
static void req_completion(struct nvme_queue *nvmeq, void *ctx,
M
Matthew Wilcox 已提交
652 653
						struct nvme_completion *cqe)
{
654
	struct nvme_iod *iod = ctx;
655
	struct request *req = iod_get_private(iod);
M
Matias Bjørling 已提交
656
	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
M
Matthew Wilcox 已提交
657
	u16 status = le16_to_cpup(&cqe->status) >> 1;
658
	bool requeue = false;
659
	int error = 0;
M
Matthew Wilcox 已提交
660

661
	if (unlikely(status)) {
M
Matias Bjørling 已提交
662 663
		if (!(status & NVME_SC_DNR || blk_noretry_request(req))
		    && (jiffies - req->start_time) < req->timeout) {
664 665
			unsigned long flags;

666
			requeue = true;
M
Matias Bjørling 已提交
667
			blk_mq_requeue_request(req);
668 669 670 671
			spin_lock_irqsave(req->q->queue_lock, flags);
			if (!blk_queue_stopped(req->q))
				blk_mq_kick_requeue_list(req->q);
			spin_unlock_irqrestore(req->q->queue_lock, flags);
672
			goto release_iod;
673
		}
674

675
		if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
676
			if (cmd_rq->ctx == CMD_CTX_CANCELLED)
677 678 679
				error = -EINTR;
			else
				error = status;
680
		} else {
681
			error = nvme_error_status(status);
682
		}
683 684
	}

K
Keith Busch 已提交
685 686 687 688
	if (req->cmd_type == REQ_TYPE_DRV_PRIV) {
		u32 result = le32_to_cpup(&cqe->result);
		req->special = (void *)(uintptr_t)result;
	}
M
Matias Bjørling 已提交
689 690

	if (cmd_rq->aborted)
691
		dev_warn(nvmeq->dev->dev,
M
Matias Bjørling 已提交
692
			"completing aborted command with status:%04x\n",
693
			error);
M
Matias Bjørling 已提交
694

695
release_iod:
K
Keith Busch 已提交
696
	if (iod->nents) {
697
		dma_unmap_sg(nvmeq->dev->dev, iod->sg, iod->nents,
M
Matias Bjørling 已提交
698
			rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
K
Keith Busch 已提交
699 700 701
		if (blk_integrity_rq(req)) {
			if (!rq_data_dir(req))
				nvme_dif_remap(req, nvme_dif_complete);
702
			dma_unmap_sg(nvmeq->dev->dev, iod->meta_sg, 1,
K
Keith Busch 已提交
703 704 705
				rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
		}
	}
706
	nvme_free_iod(nvmeq->dev, iod);
K
Keith Busch 已提交
707

708 709
	if (likely(!requeue))
		blk_mq_complete_request(req, error);
M
Matthew Wilcox 已提交
710 711
}

712
/* length is in bytes.  gfp flags indicates whether we may sleep. */
713 714
static int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod,
		int total_len, gfp_t gfp)
M
Matthew Wilcox 已提交
715
{
716
	struct dma_pool *pool;
717 718
	int length = total_len;
	struct scatterlist *sg = iod->sg;
M
Matthew Wilcox 已提交
719 720
	int dma_len = sg_dma_len(sg);
	u64 dma_addr = sg_dma_address(sg);
721 722
	u32 page_size = dev->page_size;
	int offset = dma_addr & (page_size - 1);
723
	__le64 *prp_list;
724
	__le64 **list = iod_list(iod);
725
	dma_addr_t prp_dma;
726
	int nprps, i;
M
Matthew Wilcox 已提交
727

728
	length -= (page_size - offset);
M
Matthew Wilcox 已提交
729
	if (length <= 0)
730
		return total_len;
M
Matthew Wilcox 已提交
731

732
	dma_len -= (page_size - offset);
M
Matthew Wilcox 已提交
733
	if (dma_len) {
734
		dma_addr += (page_size - offset);
M
Matthew Wilcox 已提交
735 736 737 738 739 740
	} else {
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
	}

741
	if (length <= page_size) {
742
		iod->first_dma = dma_addr;
743
		return total_len;
744 745
	}

746
	nprps = DIV_ROUND_UP(length, page_size);
747 748
	if (nprps <= (256 / 8)) {
		pool = dev->prp_small_pool;
749
		iod->npages = 0;
750 751
	} else {
		pool = dev->prp_page_pool;
752
		iod->npages = 1;
753 754
	}

755 756
	prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
	if (!prp_list) {
757
		iod->first_dma = dma_addr;
758
		iod->npages = -1;
759
		return (total_len - length) + page_size;
760
	}
761 762
	list[0] = prp_list;
	iod->first_dma = prp_dma;
763 764
	i = 0;
	for (;;) {
765
		if (i == page_size >> 3) {
766
			__le64 *old_prp_list = prp_list;
767
			prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
768 769 770
			if (!prp_list)
				return total_len - length;
			list[iod->npages++] = prp_list;
771 772 773
			prp_list[0] = old_prp_list[i - 1];
			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
			i = 1;
774 775
		}
		prp_list[i++] = cpu_to_le64(dma_addr);
776 777 778
		dma_len -= page_size;
		dma_addr += page_size;
		length -= page_size;
779 780 781 782 783 784 785 786
		if (length <= 0)
			break;
		if (dma_len > 0)
			continue;
		BUG_ON(dma_len < 0);
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
M
Matthew Wilcox 已提交
787 788
	}

789
	return total_len;
M
Matthew Wilcox 已提交
790 791
}

792 793 794
static void nvme_submit_priv(struct nvme_queue *nvmeq, struct request *req,
		struct nvme_iod *iod)
{
795
	struct nvme_command cmnd;
796

797 798
	memcpy(&cmnd, req->cmd, sizeof(cmnd));
	cmnd.rw.command_id = req->tag;
799
	if (req->nr_phys_segments) {
800 801
		cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
		cmnd.rw.prp2 = cpu_to_le64(iod->first_dma);
802 803
	}

804
	__nvme_submit_cmd(nvmeq, &cmnd);
805 806
}

M
Matias Bjørling 已提交
807 808 809 810 811 812 813
/*
 * We reuse the small pool to allocate the 16-byte range here as it is not
 * worth having a special pool for these or additional cases to handle freeing
 * the iod.
 */
static void nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
		struct request *req, struct nvme_iod *iod)
814
{
815 816
	struct nvme_dsm_range *range =
				(struct nvme_dsm_range *)iod_list(iod)[0];
817
	struct nvme_command cmnd;
818 819

	range->cattr = cpu_to_le32(0);
M
Matias Bjørling 已提交
820 821
	range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
	range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
822

823 824 825 826 827 828 829
	memset(&cmnd, 0, sizeof(cmnd));
	cmnd.dsm.opcode = nvme_cmd_dsm;
	cmnd.dsm.command_id = req->tag;
	cmnd.dsm.nsid = cpu_to_le32(ns->ns_id);
	cmnd.dsm.prp1 = cpu_to_le64(iod->first_dma);
	cmnd.dsm.nr = 0;
	cmnd.dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);
830

831
	__nvme_submit_cmd(nvmeq, &cmnd);
832 833
}

M
Matias Bjørling 已提交
834
static void nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
M
Matthew Wilcox 已提交
835 836
								int cmdid)
{
837
	struct nvme_command cmnd;
M
Matthew Wilcox 已提交
838

839 840 841 842
	memset(&cmnd, 0, sizeof(cmnd));
	cmnd.common.opcode = nvme_cmd_flush;
	cmnd.common.command_id = cmdid;
	cmnd.common.nsid = cpu_to_le32(ns->ns_id);
M
Matthew Wilcox 已提交
843

844
	__nvme_submit_cmd(nvmeq, &cmnd);
M
Matthew Wilcox 已提交
845 846
}

M
Matias Bjørling 已提交
847 848
static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod,
							struct nvme_ns *ns)
M
Matthew Wilcox 已提交
849
{
850
	struct request *req = iod_get_private(iod);
851
	struct nvme_command cmnd;
M
Matias Bjørling 已提交
852 853
	u16 control = 0;
	u32 dsmgmt = 0;
M
Matthew Wilcox 已提交
854

M
Matias Bjørling 已提交
855
	if (req->cmd_flags & REQ_FUA)
M
Matthew Wilcox 已提交
856
		control |= NVME_RW_FUA;
M
Matias Bjørling 已提交
857
	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
M
Matthew Wilcox 已提交
858 859
		control |= NVME_RW_LR;

M
Matias Bjørling 已提交
860
	if (req->cmd_flags & REQ_RAHEAD)
M
Matthew Wilcox 已提交
861 862
		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;

863 864 865 866 867 868 869 870
	memset(&cmnd, 0, sizeof(cmnd));
	cmnd.rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
	cmnd.rw.command_id = req->tag;
	cmnd.rw.nsid = cpu_to_le32(ns->ns_id);
	cmnd.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
	cmnd.rw.prp2 = cpu_to_le64(iod->first_dma);
	cmnd.rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
	cmnd.rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
M
Matthew Wilcox 已提交
871

872
	if (ns->ms) {
K
Keith Busch 已提交
873 874 875 876 877 878 879 880
		switch (ns->pi_type) {
		case NVME_NS_DPS_PI_TYPE3:
			control |= NVME_RW_PRINFO_PRCHK_GUARD;
			break;
		case NVME_NS_DPS_PI_TYPE1:
		case NVME_NS_DPS_PI_TYPE2:
			control |= NVME_RW_PRINFO_PRCHK_GUARD |
					NVME_RW_PRINFO_PRCHK_REF;
881
			cmnd.rw.reftag = cpu_to_le32(
K
Keith Busch 已提交
882 883 884
					nvme_block_nr(ns, blk_rq_pos(req)));
			break;
		}
885 886 887 888 889 890
		if (blk_integrity_rq(req))
			cmnd.rw.metadata =
				cpu_to_le64(sg_dma_address(iod->meta_sg));
		else
			control |= NVME_RW_PRINFO_PRACT;
	}
K
Keith Busch 已提交
891

892 893
	cmnd.rw.control = cpu_to_le16(control);
	cmnd.rw.dsmgmt = cpu_to_le32(dsmgmt);
M
Matthew Wilcox 已提交
894

895
	__nvme_submit_cmd(nvmeq, &cmnd);
M
Matthew Wilcox 已提交
896

897
	return 0;
898 899
}

900 901 902
/*
 * NOTE: ns is NULL when called on the admin queue.
 */
M
Matias Bjørling 已提交
903 904
static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
			 const struct blk_mq_queue_data *bd)
905
{
M
Matias Bjørling 已提交
906 907
	struct nvme_ns *ns = hctx->queue->queuedata;
	struct nvme_queue *nvmeq = hctx->driver_data;
908
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
909 910
	struct request *req = bd->rq;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
911
	struct nvme_iod *iod;
M
Matias Bjørling 已提交
912
	enum dma_data_direction dma_dir;
913

K
Keith Busch 已提交
914 915 916 917 918
	/*
	 * If formated with metadata, require the block layer provide a buffer
	 * unless this namespace is formated such that the metadata can be
	 * stripped/generated by the controller with PRACT=1.
	 */
919
	if (ns && ns->ms && !blk_integrity_rq(req)) {
920 921
		if (!(ns->pi_type && ns->ms == 8) &&
					req->cmd_type != REQ_TYPE_DRV_PRIV) {
922
			blk_mq_complete_request(req, -EFAULT);
K
Keith Busch 已提交
923 924 925 926
			return BLK_MQ_RQ_QUEUE_OK;
		}
	}

927
	iod = nvme_alloc_iod(req, dev, GFP_ATOMIC);
928
	if (!iod)
929
		return BLK_MQ_RQ_QUEUE_BUSY;
M
Matias Bjørling 已提交
930 931

	if (req->cmd_flags & REQ_DISCARD) {
932 933 934 935 936 937
		void *range;
		/*
		 * We reuse the small pool to allocate the 16-byte range here
		 * as it is not worth having a special pool for these or
		 * additional cases to handle freeing the iod.
		 */
938
		range = dma_pool_alloc(dev->prp_small_pool, GFP_ATOMIC,
939
						&iod->first_dma);
M
Matias Bjørling 已提交
940
		if (!range)
941
			goto retry_cmd;
942 943
		iod_list(iod)[0] = (__le64 *)range;
		iod->npages = 0;
944
	} else if (req->nr_phys_segments) {
M
Matias Bjørling 已提交
945 946
		dma_dir = rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;

947
		sg_init_table(iod->sg, req->nr_phys_segments);
M
Matias Bjørling 已提交
948
		iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
949 950
		if (!iod->nents)
			goto error_cmd;
M
Matias Bjørling 已提交
951 952

		if (!dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir))
953
			goto retry_cmd;
M
Matias Bjørling 已提交
954

955
		if (blk_rq_bytes(req) !=
956 957
                    nvme_setup_prps(dev, iod, blk_rq_bytes(req), GFP_ATOMIC)) {
			dma_unmap_sg(dev->dev, iod->sg, iod->nents, dma_dir);
958 959
			goto retry_cmd;
		}
K
Keith Busch 已提交
960
		if (blk_integrity_rq(req)) {
961 962 963
			if (blk_rq_count_integrity_sg(req->q, req->bio) != 1) {
				dma_unmap_sg(dev->dev, iod->sg, iod->nents,
						dma_dir);
K
Keith Busch 已提交
964
				goto error_cmd;
965
			}
K
Keith Busch 已提交
966 967 968

			sg_init_table(iod->meta_sg, 1);
			if (blk_rq_map_integrity_sg(
969 970 971
					req->q, req->bio, iod->meta_sg) != 1) {
				dma_unmap_sg(dev->dev, iod->sg, iod->nents,
						dma_dir);
K
Keith Busch 已提交
972
				goto error_cmd;
973
			}
K
Keith Busch 已提交
974 975 976 977

			if (rq_data_dir(req))
				nvme_dif_remap(req, nvme_dif_prep);

978 979 980
			if (!dma_map_sg(nvmeq->q_dmadev, iod->meta_sg, 1, dma_dir)) {
				dma_unmap_sg(dev->dev, iod->sg, iod->nents,
						dma_dir);
K
Keith Busch 已提交
981
				goto error_cmd;
982
			}
K
Keith Busch 已提交
983
		}
984
	}
985

K
Keith Busch 已提交
986
	nvme_set_info(cmd, iod, req_completion);
M
Matias Bjørling 已提交
987
	spin_lock_irq(&nvmeq->q_lock);
988 989 990
	if (req->cmd_type == REQ_TYPE_DRV_PRIV)
		nvme_submit_priv(nvmeq, req, iod);
	else if (req->cmd_flags & REQ_DISCARD)
M
Matias Bjørling 已提交
991 992 993 994 995 996 997 998 999 1000
		nvme_submit_discard(nvmeq, ns, req, iod);
	else if (req->cmd_flags & REQ_FLUSH)
		nvme_submit_flush(nvmeq, ns, req->tag);
	else
		nvme_submit_iod(nvmeq, iod, ns);

	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
	return BLK_MQ_RQ_QUEUE_OK;

1001
 error_cmd:
1002
	nvme_free_iod(dev, iod);
1003 1004
	return BLK_MQ_RQ_QUEUE_ERROR;
 retry_cmd:
1005
	nvme_free_iod(dev, iod);
1006
	return BLK_MQ_RQ_QUEUE_BUSY;
M
Matthew Wilcox 已提交
1007 1008
}

J
Jens Axboe 已提交
1009
static void __nvme_process_cq(struct nvme_queue *nvmeq, unsigned int *tag)
M
Matthew Wilcox 已提交
1010
{
M
Matthew Wilcox 已提交
1011
	u16 head, phase;
M
Matthew Wilcox 已提交
1012 1013

	head = nvmeq->cq_head;
M
Matthew Wilcox 已提交
1014
	phase = nvmeq->cq_phase;
M
Matthew Wilcox 已提交
1015 1016

	for (;;) {
1017 1018
		void *ctx;
		nvme_completion_fn fn;
M
Matthew Wilcox 已提交
1019
		struct nvme_completion cqe = nvmeq->cqes[head];
M
Matthew Wilcox 已提交
1020
		if ((le16_to_cpu(cqe.status) & 1) != phase)
M
Matthew Wilcox 已提交
1021 1022 1023 1024
			break;
		nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
		if (++head == nvmeq->q_depth) {
			head = 0;
M
Matthew Wilcox 已提交
1025
			phase = !phase;
M
Matthew Wilcox 已提交
1026
		}
J
Jens Axboe 已提交
1027 1028
		if (tag && *tag == cqe.command_id)
			*tag = -1;
M
Matias Bjørling 已提交
1029
		ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
1030
		fn(nvmeq, ctx, &cqe);
M
Matthew Wilcox 已提交
1031 1032 1033 1034 1035 1036 1037 1038
	}

	/* If the controller ignores the cq head doorbell and continuously
	 * writes to the queue, it is theoretically possible to wrap around
	 * the queue twice and mistakenly return IRQ_NONE.  Linux only
	 * requires that 0.1% of your interrupts are handled, so this isn't
	 * a big problem.
	 */
M
Matthew Wilcox 已提交
1039
	if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
J
Jens Axboe 已提交
1040
		return;
M
Matthew Wilcox 已提交
1041

1042 1043
	if (likely(nvmeq->cq_vector >= 0))
		writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
M
Matthew Wilcox 已提交
1044
	nvmeq->cq_head = head;
M
Matthew Wilcox 已提交
1045
	nvmeq->cq_phase = phase;
M
Matthew Wilcox 已提交
1046

1047
	nvmeq->cqe_seen = 1;
J
Jens Axboe 已提交
1048 1049 1050 1051 1052
}

static void nvme_process_cq(struct nvme_queue *nvmeq)
{
	__nvme_process_cq(nvmeq, NULL);
M
Matthew Wilcox 已提交
1053 1054 1055
}

static irqreturn_t nvme_irq(int irq, void *data)
1056 1057 1058 1059
{
	irqreturn_t result;
	struct nvme_queue *nvmeq = data;
	spin_lock(&nvmeq->q_lock);
1060 1061 1062
	nvme_process_cq(nvmeq);
	result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
	nvmeq->cqe_seen = 0;
1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
	spin_unlock(&nvmeq->q_lock);
	return result;
}

static irqreturn_t nvme_irq_check(int irq, void *data)
{
	struct nvme_queue *nvmeq = data;
	struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
	if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
		return IRQ_NONE;
	return IRQ_WAKE_THREAD;
}

J
Jens Axboe 已提交
1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
static int nvme_poll(struct blk_mq_hw_ctx *hctx, unsigned int tag)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	if ((le16_to_cpu(nvmeq->cqes[nvmeq->cq_head].status) & 1) ==
	    nvmeq->cq_phase) {
		spin_lock_irq(&nvmeq->q_lock);
		__nvme_process_cq(nvmeq, &tag);
		spin_unlock_irq(&nvmeq->q_lock);

		if (tag == -1)
			return 1;
	}

	return 0;
}

M
Matias Bjørling 已提交
1093 1094 1095 1096 1097 1098 1099
static int nvme_submit_async_admin_req(struct nvme_dev *dev)
{
	struct nvme_queue *nvmeq = dev->queues[0];
	struct nvme_command c;
	struct nvme_cmd_info *cmd_info;
	struct request *req;

1100
	req = blk_mq_alloc_request(dev->ctrl.admin_q, WRITE,
1101
			BLK_MQ_REQ_NOWAIT | BLK_MQ_REQ_RESERVED);
1102 1103
	if (IS_ERR(req))
		return PTR_ERR(req);
M
Matias Bjørling 已提交
1104

K
Keith Busch 已提交
1105
	req->cmd_flags |= REQ_NO_TIMEOUT;
M
Matias Bjørling 已提交
1106
	cmd_info = blk_mq_rq_to_pdu(req);
1107
	nvme_set_info(cmd_info, NULL, async_req_completion);
M
Matias Bjørling 已提交
1108 1109 1110 1111 1112

	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_async_event;
	c.common.command_id = req->tag;

1113
	blk_mq_free_request(req);
1114 1115
	__nvme_submit_cmd(nvmeq, &c);
	return 0;
M
Matias Bjørling 已提交
1116 1117 1118
}

static int nvme_submit_admin_async_cmd(struct nvme_dev *dev,
K
Keith Busch 已提交
1119 1120 1121
			struct nvme_command *cmd,
			struct async_cmd_info *cmdinfo, unsigned timeout)
{
M
Matias Bjørling 已提交
1122 1123 1124
	struct nvme_queue *nvmeq = dev->queues[0];
	struct request *req;
	struct nvme_cmd_info *cmd_rq;
K
Keith Busch 已提交
1125

1126
	req = blk_mq_alloc_request(dev->ctrl.admin_q, WRITE, 0);
1127 1128
	if (IS_ERR(req))
		return PTR_ERR(req);
M
Matias Bjørling 已提交
1129 1130 1131 1132 1133

	req->timeout = timeout;
	cmd_rq = blk_mq_rq_to_pdu(req);
	cmdinfo->req = req;
	nvme_set_info(cmd_rq, cmdinfo, async_completion);
K
Keith Busch 已提交
1134
	cmdinfo->status = -EINTR;
M
Matias Bjørling 已提交
1135 1136 1137

	cmd->common.command_id = req->tag;

1138 1139
	nvme_submit_cmd(nvmeq, cmd);
	return 0;
K
Keith Busch 已提交
1140 1141
}

M
Matthew Wilcox 已提交
1142 1143 1144 1145 1146 1147 1148 1149
static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(id);

1150
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1151 1152 1153 1154 1155 1156 1157 1158
}

static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;

1159 1160 1161 1162
	/*
	 * Note: we (ab)use the fact the the prp fields survive if no data
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
1163 1164 1165 1166 1167 1168 1169 1170
	memset(&c, 0, sizeof(c));
	c.create_cq.opcode = nvme_admin_create_cq;
	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
	c.create_cq.cqid = cpu_to_le16(qid);
	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_cq.cq_flags = cpu_to_le16(flags);
	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);

1171
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1172 1173 1174 1175 1176 1177 1178 1179
}

static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;

1180 1181 1182 1183
	/*
	 * Note: we (ab)use the fact the the prp fields survive if no data
	 * is attached to the request.
	 */
M
Matthew Wilcox 已提交
1184 1185 1186 1187 1188 1189 1190 1191
	memset(&c, 0, sizeof(c));
	c.create_sq.opcode = nvme_admin_create_sq;
	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
	c.create_sq.sqid = cpu_to_le16(qid);
	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_sq.sq_flags = cpu_to_le16(flags);
	c.create_sq.cqid = cpu_to_le16(qid);

1192
	return nvme_submit_sync_cmd(dev->ctrl.admin_q, &c, NULL, 0);
M
Matthew Wilcox 已提交
1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204
}

static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
}

static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
}

K
Keith Busch 已提交
1205
/**
M
Matias Bjørling 已提交
1206
 * nvme_abort_req - Attempt aborting a request
K
Keith Busch 已提交
1207 1208 1209 1210
 *
 * Schedule controller reset if the command was already aborted once before and
 * still hasn't been returned to the driver, or if this is the admin queue.
 */
M
Matias Bjørling 已提交
1211
static void nvme_abort_req(struct request *req)
K
Keith Busch 已提交
1212
{
M
Matias Bjørling 已提交
1213 1214
	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd_rq->nvmeq;
K
Keith Busch 已提交
1215
	struct nvme_dev *dev = nvmeq->dev;
M
Matias Bjørling 已提交
1216 1217 1218
	struct request *abort_req;
	struct nvme_cmd_info *abort_cmd;
	struct nvme_command cmd;
K
Keith Busch 已提交
1219

M
Matias Bjørling 已提交
1220
	if (!nvmeq->qid || cmd_rq->aborted) {
1221 1222 1223 1224 1225 1226 1227
		spin_lock(&dev_list_lock);
		if (!__nvme_reset(dev)) {
			dev_warn(dev->dev,
				 "I/O %d QID %d timeout, reset controller\n",
				 req->tag, nvmeq->qid);
		}
		spin_unlock(&dev_list_lock);
K
Keith Busch 已提交
1228 1229 1230
		return;
	}

1231
	if (!dev->ctrl.abort_limit)
K
Keith Busch 已提交
1232 1233
		return;

1234
	abort_req = blk_mq_alloc_request(dev->ctrl.admin_q, WRITE,
1235
			BLK_MQ_REQ_NOWAIT);
1236
	if (IS_ERR(abort_req))
K
Keith Busch 已提交
1237 1238
		return;

M
Matias Bjørling 已提交
1239 1240 1241
	abort_cmd = blk_mq_rq_to_pdu(abort_req);
	nvme_set_info(abort_cmd, abort_req, abort_completion);

K
Keith Busch 已提交
1242 1243
	memset(&cmd, 0, sizeof(cmd));
	cmd.abort.opcode = nvme_admin_abort_cmd;
M
Matias Bjørling 已提交
1244
	cmd.abort.cid = req->tag;
K
Keith Busch 已提交
1245
	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
M
Matias Bjørling 已提交
1246
	cmd.abort.command_id = abort_req->tag;
K
Keith Busch 已提交
1247

1248
	--dev->ctrl.abort_limit;
M
Matias Bjørling 已提交
1249
	cmd_rq->aborted = 1;
K
Keith Busch 已提交
1250

M
Matias Bjørling 已提交
1251
	dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", req->tag,
K
Keith Busch 已提交
1252
							nvmeq->qid);
1253
	nvme_submit_cmd(dev->queues[0], &cmd);
K
Keith Busch 已提交
1254 1255
}

1256
static void nvme_cancel_queue_ios(struct request *req, void *data, bool reserved)
1257
{
M
Matias Bjørling 已提交
1258 1259 1260 1261
	struct nvme_queue *nvmeq = data;
	void *ctx;
	nvme_completion_fn fn;
	struct nvme_cmd_info *cmd;
K
Keith Busch 已提交
1262 1263 1264 1265
	struct nvme_completion cqe;

	if (!blk_mq_request_started(req))
		return;
1266

M
Matias Bjørling 已提交
1267
	cmd = blk_mq_rq_to_pdu(req);
1268

M
Matias Bjørling 已提交
1269 1270 1271
	if (cmd->ctx == CMD_CTX_CANCELLED)
		return;

K
Keith Busch 已提交
1272 1273 1274 1275 1276 1277
	if (blk_queue_dying(req->q))
		cqe.status = cpu_to_le16((NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1);
	else
		cqe.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);


M
Matias Bjørling 已提交
1278 1279 1280 1281
	dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n",
						req->tag, nvmeq->qid);
	ctx = cancel_cmd_info(cmd, &fn);
	fn(nvmeq, ctx, &cqe);
1282 1283
}

M
Matias Bjørling 已提交
1284
static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
1285
{
M
Matias Bjørling 已提交
1286 1287 1288 1289 1290
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd->nvmeq;

	dev_warn(nvmeq->q_dmadev, "Timeout I/O %d QID %d\n", req->tag,
							nvmeq->qid);
1291
	spin_lock_irq(&nvmeq->q_lock);
1292
	nvme_abort_req(req);
1293
	spin_unlock_irq(&nvmeq->q_lock);
M
Matias Bjørling 已提交
1294

1295 1296 1297 1298 1299 1300
	/*
	 * The aborted req will be completed on receiving the abort req.
	 * We enable the timer again. If hit twice, it'll cause a device reset,
	 * as the device then is in a faulty state.
	 */
	return BLK_EH_RESET_TIMER;
M
Matias Bjørling 已提交
1301
}
1302

M
Matias Bjørling 已提交
1303 1304
static void nvme_free_queue(struct nvme_queue *nvmeq)
{
1305 1306
	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
1307 1308
	if (nvmeq->sq_cmds)
		dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
1309 1310 1311 1312
					nvmeq->sq_cmds, nvmeq->sq_dma_addr);
	kfree(nvmeq);
}

1313
static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1314 1315 1316
{
	int i;

1317
	for (i = dev->queue_count - 1; i >= lowest; i--) {
M
Matias Bjørling 已提交
1318
		struct nvme_queue *nvmeq = dev->queues[i];
1319
		dev->queue_count--;
M
Matias Bjørling 已提交
1320
		dev->queues[i] = NULL;
K
Keith Busch 已提交
1321
		nvme_free_queue(nvmeq);
1322
	}
1323 1324
}

K
Keith Busch 已提交
1325 1326 1327 1328 1329
/**
 * nvme_suspend_queue - put queue into suspended state
 * @nvmeq - queue to suspend
 */
static int nvme_suspend_queue(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
1330
{
K
Keith Busch 已提交
1331
	int vector;
M
Matthew Wilcox 已提交
1332

1333
	spin_lock_irq(&nvmeq->q_lock);
K
Keith Busch 已提交
1334 1335 1336 1337 1338
	if (nvmeq->cq_vector == -1) {
		spin_unlock_irq(&nvmeq->q_lock);
		return 1;
	}
	vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
K
Keith Busch 已提交
1339
	nvmeq->dev->online_queues--;
K
Keith Busch 已提交
1340
	nvmeq->cq_vector = -1;
1341 1342
	spin_unlock_irq(&nvmeq->q_lock);

1343 1344
	if (!nvmeq->qid && nvmeq->dev->ctrl.admin_q)
		blk_mq_freeze_queue_start(nvmeq->dev->ctrl.admin_q);
1345

M
Matthew Wilcox 已提交
1346 1347
	irq_set_affinity_hint(vector, NULL);
	free_irq(vector, nvmeq);
M
Matthew Wilcox 已提交
1348

K
Keith Busch 已提交
1349 1350
	return 0;
}
M
Matthew Wilcox 已提交
1351

K
Keith Busch 已提交
1352 1353
static void nvme_clear_queue(struct nvme_queue *nvmeq)
{
1354
	spin_lock_irq(&nvmeq->q_lock);
1355 1356
	if (nvmeq->tags && *nvmeq->tags)
		blk_mq_all_tag_busy_iter(*nvmeq->tags, nvme_cancel_queue_ios, nvmeq);
1357
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1358 1359
}

K
Keith Busch 已提交
1360 1361
static void nvme_disable_queue(struct nvme_dev *dev, int qid)
{
M
Matias Bjørling 已提交
1362
	struct nvme_queue *nvmeq = dev->queues[qid];
K
Keith Busch 已提交
1363 1364 1365 1366 1367 1368

	if (!nvmeq)
		return;
	if (nvme_suspend_queue(nvmeq))
		return;

K
Keith Busch 已提交
1369 1370
	/* Don't tell the adapter to delete the admin queue.
	 * Don't tell a removed adapter to delete IO queues. */
1371
	if (qid && readl(dev->bar + NVME_REG_CSTS) != -1) {
M
Matthew Wilcox 已提交
1372 1373 1374
		adapter_delete_sq(dev, qid);
		adapter_delete_cq(dev, qid);
	}
1375 1376 1377 1378

	spin_lock_irq(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1379 1380
}

1381 1382 1383 1384 1385 1386 1387
static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues,
				int entry_size)
{
	int q_depth = dev->q_depth;
	unsigned q_size_aligned = roundup(q_depth * entry_size, dev->page_size);

	if (q_size_aligned * nr_io_queues > dev->cmb_size) {
1388 1389 1390
		u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues);
		mem_per_q = round_down(mem_per_q, dev->page_size);
		q_depth = div_u64(mem_per_q, entry_size);
1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421

		/*
		 * Ensure the reduced q_depth is above some threshold where it
		 * would be better to map queues in system memory with the
		 * original depth
		 */
		if (q_depth < 64)
			return -ENOMEM;
	}

	return q_depth;
}

static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq,
				int qid, int depth)
{
	if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) {
		unsigned offset = (qid - 1) *
					roundup(SQ_SIZE(depth), dev->page_size);
		nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset;
		nvmeq->sq_cmds_io = dev->cmb + offset;
	} else {
		nvmeq->sq_cmds = dma_alloc_coherent(dev->dev, SQ_SIZE(depth),
					&nvmeq->sq_dma_addr, GFP_KERNEL);
		if (!nvmeq->sq_cmds)
			return -ENOMEM;
	}

	return 0;
}

M
Matthew Wilcox 已提交
1422
static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
K
Keith Busch 已提交
1423
							int depth)
M
Matthew Wilcox 已提交
1424
{
M
Matias Bjørling 已提交
1425
	struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
M
Matthew Wilcox 已提交
1426 1427 1428
	if (!nvmeq)
		return NULL;

1429
	nvmeq->cqes = dma_zalloc_coherent(dev->dev, CQ_SIZE(depth),
J
Joe Perches 已提交
1430
					  &nvmeq->cq_dma_addr, GFP_KERNEL);
M
Matthew Wilcox 已提交
1431 1432 1433
	if (!nvmeq->cqes)
		goto free_nvmeq;

1434
	if (nvme_alloc_sq_cmds(dev, nvmeq, qid, depth))
M
Matthew Wilcox 已提交
1435 1436
		goto free_cqdma;

1437
	nvmeq->q_dmadev = dev->dev;
M
Matthew Wilcox 已提交
1438
	nvmeq->dev = dev;
1439
	snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
1440
			dev->ctrl.instance, qid);
M
Matthew Wilcox 已提交
1441 1442
	spin_lock_init(&nvmeq->q_lock);
	nvmeq->cq_head = 0;
M
Matthew Wilcox 已提交
1443
	nvmeq->cq_phase = 1;
1444
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
M
Matthew Wilcox 已提交
1445
	nvmeq->q_depth = depth;
K
Keith Busch 已提交
1446
	nvmeq->qid = qid;
1447
	nvmeq->cq_vector = -1;
M
Matias Bjørling 已提交
1448
	dev->queues[qid] = nvmeq;
M
Matthew Wilcox 已提交
1449

1450 1451 1452 1453
	/* make sure queue descriptor is set before queue count, for kthread */
	mb();
	dev->queue_count++;

M
Matthew Wilcox 已提交
1454 1455 1456
	return nvmeq;

 free_cqdma:
1457
	dma_free_coherent(dev->dev, CQ_SIZE(depth), (void *)nvmeq->cqes,
M
Matthew Wilcox 已提交
1458 1459 1460 1461 1462 1463
							nvmeq->cq_dma_addr);
 free_nvmeq:
	kfree(nvmeq);
	return NULL;
}

1464 1465 1466
static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
							const char *name)
{
1467 1468
	if (use_threaded_interrupts)
		return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1469
					nvme_irq_check, nvme_irq, IRQF_SHARED,
1470
					name, nvmeq);
1471
	return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1472
				IRQF_SHARED, name, nvmeq);
1473 1474
}

1475
static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
M
Matthew Wilcox 已提交
1476
{
1477
	struct nvme_dev *dev = nvmeq->dev;
M
Matthew Wilcox 已提交
1478

1479
	spin_lock_irq(&nvmeq->q_lock);
1480 1481 1482
	nvmeq->sq_tail = 0;
	nvmeq->cq_head = 0;
	nvmeq->cq_phase = 1;
1483
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1484
	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
K
Keith Busch 已提交
1485
	dev->online_queues++;
1486
	spin_unlock_irq(&nvmeq->q_lock);
1487 1488 1489 1490 1491 1492
}

static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
{
	struct nvme_dev *dev = nvmeq->dev;
	int result;
1493

K
Keith Busch 已提交
1494
	nvmeq->cq_vector = qid - 1;
M
Matthew Wilcox 已提交
1495 1496
	result = adapter_alloc_cq(dev, qid, nvmeq);
	if (result < 0)
1497
		return result;
M
Matthew Wilcox 已提交
1498 1499 1500 1501 1502

	result = adapter_alloc_sq(dev, qid, nvmeq);
	if (result < 0)
		goto release_cq;

1503
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
M
Matthew Wilcox 已提交
1504 1505 1506
	if (result < 0)
		goto release_sq;

1507 1508
	nvme_init_queue(nvmeq, qid);
	return result;
M
Matthew Wilcox 已提交
1509 1510 1511 1512 1513

 release_sq:
	adapter_delete_sq(dev, qid);
 release_cq:
	adapter_delete_cq(dev, qid);
1514
	return result;
M
Matthew Wilcox 已提交
1515 1516
}

1517 1518 1519 1520 1521 1522 1523
static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
{
	unsigned long timeout;
	u32 bit = enabled ? NVME_CSTS_RDY : 0;

	timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;

1524
	while ((readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_RDY) != bit) {
1525 1526 1527 1528
		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
1529
			dev_err(dev->dev,
M
Matthew Wilcox 已提交
1530 1531
				"Device not ready; aborting %s\n", enabled ?
						"initialisation" : "reset");
1532 1533 1534 1535 1536 1537 1538 1539 1540 1541 1542 1543 1544 1545 1546
			return -ENODEV;
		}
	}

	return 0;
}

/*
 * If the device has been passed off to us in an enabled state, just clear
 * the enabled bit.  The spec says we should set the 'shutdown notification
 * bits', but doing so may cause the device to complete commands to the
 * admin queue ... and we don't know what memory that might be pointing at!
 */
static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
{
1547 1548
	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
	dev->ctrl_config &= ~NVME_CC_ENABLE;
1549
	writel(dev->ctrl_config, dev->bar + NVME_REG_CC);
1550

1551 1552 1553 1554 1555
	return nvme_wait_ready(dev, cap, false);
}

static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
{
1556 1557
	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
	dev->ctrl_config |= NVME_CC_ENABLE;
1558
	writel(dev->ctrl_config, dev->bar + NVME_REG_CC);
1559

1560 1561 1562
	return nvme_wait_ready(dev, cap, true);
}

K
Keith Busch 已提交
1563 1564 1565 1566
static int nvme_shutdown_ctrl(struct nvme_dev *dev)
{
	unsigned long timeout;

1567 1568 1569
	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
	dev->ctrl_config |= NVME_CC_SHN_NORMAL;

1570
	writel(dev->ctrl_config, dev->bar + NVME_REG_CC);
K
Keith Busch 已提交
1571

1572
	timeout = SHUTDOWN_TIMEOUT + jiffies;
1573
	while ((readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_SHST_MASK) !=
K
Keith Busch 已提交
1574 1575 1576 1577 1578
							NVME_CSTS_SHST_CMPLT) {
		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
1579
			dev_err(dev->dev,
K
Keith Busch 已提交
1580 1581 1582 1583 1584 1585 1586 1587
				"Device shutdown incomplete; abort shutdown\n");
			return -ENODEV;
		}
	}

	return 0;
}

M
Matias Bjørling 已提交
1588
static struct blk_mq_ops nvme_mq_admin_ops = {
1589
	.queue_rq	= nvme_queue_rq,
M
Matias Bjørling 已提交
1590 1591
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_admin_init_hctx,
1592
	.exit_hctx      = nvme_admin_exit_hctx,
M
Matias Bjørling 已提交
1593 1594 1595 1596 1597 1598 1599 1600 1601 1602
	.init_request	= nvme_admin_init_request,
	.timeout	= nvme_timeout,
};

static struct blk_mq_ops nvme_mq_ops = {
	.queue_rq	= nvme_queue_rq,
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_init_hctx,
	.init_request	= nvme_init_request,
	.timeout	= nvme_timeout,
J
Jens Axboe 已提交
1603
	.poll		= nvme_poll,
M
Matias Bjørling 已提交
1604 1605
};

1606 1607
static void nvme_dev_remove_admin(struct nvme_dev *dev)
{
1608 1609
	if (dev->ctrl.admin_q && !blk_queue_dying(dev->ctrl.admin_q)) {
		blk_cleanup_queue(dev->ctrl.admin_q);
1610 1611 1612 1613
		blk_mq_free_tag_set(&dev->admin_tagset);
	}
}

M
Matias Bjørling 已提交
1614 1615
static int nvme_alloc_admin_tags(struct nvme_dev *dev)
{
1616
	if (!dev->ctrl.admin_q) {
M
Matias Bjørling 已提交
1617 1618 1619
		dev->admin_tagset.ops = &nvme_mq_admin_ops;
		dev->admin_tagset.nr_hw_queues = 1;
		dev->admin_tagset.queue_depth = NVME_AQ_DEPTH - 1;
1620
		dev->admin_tagset.reserved_tags = 1;
M
Matias Bjørling 已提交
1621
		dev->admin_tagset.timeout = ADMIN_TIMEOUT;
1622
		dev->admin_tagset.numa_node = dev_to_node(dev->dev);
1623
		dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
M
Matias Bjørling 已提交
1624 1625 1626 1627 1628
		dev->admin_tagset.driver_data = dev;

		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
			return -ENOMEM;

1629 1630
		dev->ctrl.admin_q = blk_mq_init_queue(&dev->admin_tagset);
		if (IS_ERR(dev->ctrl.admin_q)) {
M
Matias Bjørling 已提交
1631 1632 1633
			blk_mq_free_tag_set(&dev->admin_tagset);
			return -ENOMEM;
		}
1634
		if (!blk_get_queue(dev->ctrl.admin_q)) {
1635
			nvme_dev_remove_admin(dev);
1636
			dev->ctrl.admin_q = NULL;
1637 1638
			return -ENODEV;
		}
K
Keith Busch 已提交
1639
	} else
1640
		blk_mq_unfreeze_queue(dev->ctrl.admin_q);
M
Matias Bjørling 已提交
1641 1642 1643 1644

	return 0;
}

1645
static int nvme_configure_admin_queue(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1646
{
1647
	int result;
M
Matthew Wilcox 已提交
1648
	u32 aqa;
1649
	u64 cap = lo_hi_readq(dev->bar + NVME_REG_CAP);
M
Matthew Wilcox 已提交
1650
	struct nvme_queue *nvmeq;
1651 1652 1653 1654 1655 1656
	/*
	 * default to a 4K page size, with the intention to update this
	 * path in the future to accomodate architectures with differing
	 * kernel and IO page sizes.
	 */
	unsigned page_shift = 12;
1657 1658 1659
	unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;

	if (page_shift < dev_page_min) {
1660
		dev_err(dev->dev,
1661 1662 1663 1664 1665
				"Minimum device page size (%u) too large for "
				"host (%u)\n", 1 << dev_page_min,
				1 << page_shift);
		return -ENODEV;
	}
M
Matthew Wilcox 已提交
1666

1667
	dev->subsystem = readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 1) ?
1668 1669
						NVME_CAP_NSSRC(cap) : 0;

1670 1671 1672
	if (dev->subsystem &&
	    (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO))
		writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS);
1673

1674 1675 1676
	result = nvme_disable_ctrl(dev, cap);
	if (result < 0)
		return result;
M
Matthew Wilcox 已提交
1677

M
Matias Bjørling 已提交
1678
	nvmeq = dev->queues[0];
1679
	if (!nvmeq) {
K
Keith Busch 已提交
1680
		nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1681 1682 1683
		if (!nvmeq)
			return -ENOMEM;
	}
M
Matthew Wilcox 已提交
1684 1685 1686 1687

	aqa = nvmeq->q_depth - 1;
	aqa |= aqa << 16;

1688 1689
	dev->page_size = 1 << page_shift;

1690
	dev->ctrl_config = NVME_CC_CSS_NVM;
1691
	dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
M
Matthew Wilcox 已提交
1692
	dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
1693
	dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
M
Matthew Wilcox 已提交
1694

1695 1696 1697
	writel(aqa, dev->bar + NVME_REG_AQA);
	lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ);
	lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ);
M
Matthew Wilcox 已提交
1698

1699
	result = nvme_enable_ctrl(dev, cap);
1700
	if (result)
M
Matias Bjørling 已提交
1701 1702
		goto free_nvmeq;

K
Keith Busch 已提交
1703
	nvmeq->cq_vector = 0;
1704
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
1705 1706
	if (result) {
		nvmeq->cq_vector = -1;
K
Keith Busch 已提交
1707
		goto free_nvmeq;
1708
	}
1709

M
Matthew Wilcox 已提交
1710
	return result;
M
Matias Bjørling 已提交
1711 1712 1713 1714

 free_nvmeq:
	nvme_free_queues(dev, 0);
	return result;
M
Matthew Wilcox 已提交
1715 1716
}

M
Matthew Wilcox 已提交
1717 1718
static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
{
1719
	struct nvme_dev *dev = to_nvme_dev(ns->ctrl);
M
Matthew Wilcox 已提交
1720 1721
	struct nvme_user_io io;
	struct nvme_command c;
1722
	unsigned length, meta_len;
1723 1724 1725
	int status, write;
	dma_addr_t meta_dma = 0;
	void *meta = NULL;
A
Arnd Bergmann 已提交
1726
	void __user *metadata;
M
Matthew Wilcox 已提交
1727 1728 1729

	if (copy_from_user(&io, uio, sizeof(io)))
		return -EFAULT;
1730 1731 1732 1733

	switch (io.opcode) {
	case nvme_cmd_write:
	case nvme_cmd_read:
M
Matthew Wilcox 已提交
1734
	case nvme_cmd_compare:
M
Matthew Wilcox 已提交
1735
		break;
1736
	default:
M
Matthew Wilcox 已提交
1737
		return -EINVAL;
1738 1739
	}

1740 1741
	length = (io.nblocks + 1) << ns->lba_shift;
	meta_len = (io.nblocks + 1) * ns->ms;
A
Arnd Bergmann 已提交
1742
	metadata = (void __user *)(uintptr_t)io.metadata;
1743
	write = io.opcode & 1;
M
Matthew Wilcox 已提交
1744

1745 1746 1747
	if (ns->ext) {
		length += meta_len;
		meta_len = 0;
1748 1749
	}
	if (meta_len) {
1750 1751 1752
		if (((io.metadata & 3) || !io.metadata) && !ns->ext)
			return -EINVAL;

1753
		meta = dma_alloc_coherent(dev->dev, meta_len,
1754
						&meta_dma, GFP_KERNEL);
A
Arnd Bergmann 已提交
1755

1756 1757 1758 1759 1760
		if (!meta) {
			status = -ENOMEM;
			goto unmap;
		}
		if (write) {
A
Arnd Bergmann 已提交
1761
			if (copy_from_user(meta, metadata, meta_len)) {
1762 1763 1764 1765 1766 1767
				status = -EFAULT;
				goto unmap;
			}
		}
	}

M
Matthew Wilcox 已提交
1768 1769 1770
	memset(&c, 0, sizeof(c));
	c.rw.opcode = io.opcode;
	c.rw.flags = io.flags;
1771
	c.rw.nsid = cpu_to_le32(ns->ns_id);
M
Matthew Wilcox 已提交
1772
	c.rw.slba = cpu_to_le64(io.slba);
1773
	c.rw.length = cpu_to_le16(io.nblocks);
M
Matthew Wilcox 已提交
1774
	c.rw.control = cpu_to_le16(io.control);
1775 1776 1777 1778
	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
	c.rw.reftag = cpu_to_le32(io.reftag);
	c.rw.apptag = cpu_to_le16(io.apptag);
	c.rw.appmask = cpu_to_le16(io.appmask);
1779
	c.rw.metadata = cpu_to_le64(meta_dma);
1780 1781

	status = __nvme_submit_sync_cmd(ns->queue, &c, NULL,
A
Arnd Bergmann 已提交
1782
			(void __user *)(uintptr_t)io.addr, length, NULL, 0);
1783
 unmap:
1784 1785
	if (meta) {
		if (status == NVME_SC_SUCCESS && !write) {
A
Arnd Bergmann 已提交
1786
			if (copy_to_user(metadata, meta, meta_len))
1787 1788
				status = -EFAULT;
		}
1789
		dma_free_coherent(dev->dev, meta_len, meta, meta_dma);
1790
	}
M
Matthew Wilcox 已提交
1791 1792 1793
	return status;
}

1794
static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
M
Matias Bjørling 已提交
1795
			struct nvme_passthru_cmd __user *ucmd)
1796
{
1797
	struct nvme_passthru_cmd cmd;
1798
	struct nvme_command c;
1799 1800
	unsigned timeout = 0;
	int status;
1801

M
Matthew Wilcox 已提交
1802 1803 1804
	if (!capable(CAP_SYS_ADMIN))
		return -EACCES;
	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1805 1806 1807
		return -EFAULT;

	memset(&c, 0, sizeof(c));
M
Matthew Wilcox 已提交
1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818 1819
	c.common.opcode = cmd.opcode;
	c.common.flags = cmd.flags;
	c.common.nsid = cpu_to_le32(cmd.nsid);
	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
	c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
	c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
	c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
	c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
	c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
	c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);

1820 1821
	if (cmd.timeout_ms)
		timeout = msecs_to_jiffies(cmd.timeout_ms);
1822

1823
	status = __nvme_submit_sync_cmd(ns ? ns->queue : ctrl->admin_q, &c,
A
Arnd Bergmann 已提交
1824
			NULL, (void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1825 1826 1827 1828
			&cmd.result, timeout);
	if (status >= 0) {
		if (put_user(cmd.result, &ucmd->result))
			return -EFAULT;
M
Matthew Wilcox 已提交
1829
	}
1830

1831 1832 1833
	return status;
}

1834 1835 1836 1837 1838
static int nvme_subsys_reset(struct nvme_dev *dev)
{
	if (!dev->subsystem)
		return -ENOTTY;

1839
	writel(0x4E564D65, dev->bar + NVME_REG_NSSR); /* "NVMe" */
1840 1841 1842
	return 0;
}

M
Matthew Wilcox 已提交
1843 1844 1845 1846 1847 1848
static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
							unsigned long arg)
{
	struct nvme_ns *ns = bdev->bd_disk->private_data;

	switch (cmd) {
M
Matthew Wilcox 已提交
1849
	case NVME_IOCTL_ID:
1850
		force_successful_syscall_return();
M
Matthew Wilcox 已提交
1851 1852
		return ns->ns_id;
	case NVME_IOCTL_ADMIN_CMD:
1853
		return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
1854
	case NVME_IOCTL_IO_CMD:
1855
		return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
M
Matthew Wilcox 已提交
1856 1857
	case NVME_IOCTL_SUBMIT_IO:
		return nvme_submit_io(ns, (void __user *)arg);
V
Vishal Verma 已提交
1858 1859 1860 1861
	case SG_GET_VERSION_NUM:
		return nvme_sg_get_version_num((void __user *)arg);
	case SG_IO:
		return nvme_sg_io(ns, (void __user *)arg);
M
Matthew Wilcox 已提交
1862 1863 1864 1865 1866
	default:
		return -ENOTTY;
	}
}

K
Keith Busch 已提交
1867 1868 1869 1870 1871 1872
#ifdef CONFIG_COMPAT
static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
					unsigned int cmd, unsigned long arg)
{
	switch (cmd) {
	case SG_IO:
1873
		return -ENOIOCTLCMD;
K
Keith Busch 已提交
1874 1875 1876 1877 1878 1879 1880
	}
	return nvme_ioctl(bdev, mode, cmd, arg);
}
#else
#define nvme_compat_ioctl	NULL
#endif

1881
static void nvme_free_dev(struct kref *kref);
1882 1883 1884
static void nvme_free_ns(struct kref *kref)
{
	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);
1885
	struct nvme_dev *dev = to_nvme_dev(ns->ctrl);
1886

M
Matias Bjørling 已提交
1887 1888 1889
	if (ns->type == NVME_NS_LIGHTNVM)
		nvme_nvm_unregister(ns->queue, ns->disk->disk_name);

1890 1891 1892 1893
	spin_lock(&dev_list_lock);
	ns->disk->private_data = NULL;
	spin_unlock(&dev_list_lock);

1894
	kref_put(&dev->kref, nvme_free_dev);
1895 1896 1897 1898
	put_disk(ns->disk);
	kfree(ns);
}

1899 1900
static int nvme_open(struct block_device *bdev, fmode_t mode)
{
1901 1902
	int ret = 0;
	struct nvme_ns *ns;
1903

1904 1905 1906 1907
	spin_lock(&dev_list_lock);
	ns = bdev->bd_disk->private_data;
	if (!ns)
		ret = -ENXIO;
1908
	else if (!kref_get_unless_zero(&ns->kref))
1909 1910 1911 1912
		ret = -ENXIO;
	spin_unlock(&dev_list_lock);

	return ret;
1913 1914 1915 1916 1917
}

static void nvme_release(struct gendisk *disk, fmode_t mode)
{
	struct nvme_ns *ns = disk->private_data;
1918
	kref_put(&ns->kref, nvme_free_ns);
1919 1920
}

K
Keith Busch 已提交
1921 1922 1923 1924 1925 1926 1927 1928 1929
static int nvme_getgeo(struct block_device *bd, struct hd_geometry *geo)
{
	/* some standard values */
	geo->heads = 1 << 6;
	geo->sectors = 1 << 5;
	geo->cylinders = get_capacity(bd->bd_disk) >> 11;
	return 0;
}

K
Keith Busch 已提交
1930 1931 1932 1933 1934 1935
static void nvme_config_discard(struct nvme_ns *ns)
{
	u32 logical_block_size = queue_logical_block_size(ns->queue);
	ns->queue->limits.discard_zeroes_data = 0;
	ns->queue->limits.discard_alignment = logical_block_size;
	ns->queue->limits.discard_granularity = logical_block_size;
1936
	blk_queue_max_discard_sectors(ns->queue, 0xffffffff);
K
Keith Busch 已提交
1937 1938 1939
	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
}

K
Keith Busch 已提交
1940 1941 1942
static int nvme_revalidate_disk(struct gendisk *disk)
{
	struct nvme_ns *ns = disk->private_data;
1943
	struct nvme_dev *dev = to_nvme_dev(ns->ctrl);
K
Keith Busch 已提交
1944
	struct nvme_id_ns *id;
1945 1946
	u8 lbaf, pi_type;
	u16 old_ms;
K
Keith Busch 已提交
1947
	unsigned short bs;
K
Keith Busch 已提交
1948

1949
	if (nvme_identify_ns(&dev->ctrl, ns->ns_id, &id)) {
K
Keith Busch 已提交
1950
		dev_warn(dev->dev, "%s: Identify failure nvme%dn%d\n", __func__,
1951
						dev->ctrl.instance, ns->ns_id);
K
Keith Busch 已提交
1952
		return -ENODEV;
K
Keith Busch 已提交
1953
	}
K
Keith Busch 已提交
1954 1955 1956
	if (id->ncap == 0) {
		kfree(id);
		return -ENODEV;
K
Keith Busch 已提交
1957
	}
K
Keith Busch 已提交
1958

M
Matias Bjørling 已提交
1959 1960 1961 1962 1963 1964 1965 1966 1967 1968
	if (nvme_nvm_ns_supported(ns, id) && ns->type != NVME_NS_LIGHTNVM) {
		if (nvme_nvm_register(ns->queue, disk->disk_name)) {
			dev_warn(dev->dev,
				"%s: LightNVM init failure\n", __func__);
			kfree(id);
			return -ENODEV;
		}
		ns->type = NVME_NS_LIGHTNVM;
	}

K
Keith Busch 已提交
1969 1970
	old_ms = ns->ms;
	lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
K
Keith Busch 已提交
1971
	ns->lba_shift = id->lbaf[lbaf].ds;
K
Keith Busch 已提交
1972
	ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
1973
	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
K
Keith Busch 已提交
1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986

	/*
	 * If identify namespace failed, use default 512 byte block size so
	 * block layer can use before failing read/write for 0 capacity.
	 */
	if (ns->lba_shift == 0)
		ns->lba_shift = 9;
	bs = 1 << ns->lba_shift;

	/* XXX: PI implementation requires metadata equal t10 pi tuple size */
	pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
					id->dps & NVME_NS_DPS_PI_MASK : 0;

1987
	blk_mq_freeze_queue(disk->queue);
1988 1989
	if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
				ns->ms != old_ms ||
K
Keith Busch 已提交
1990
				bs != queue_logical_block_size(disk->queue) ||
1991
				(ns->ms && ns->ext)))
K
Keith Busch 已提交
1992 1993 1994 1995 1996
		blk_integrity_unregister(disk);

	ns->pi_type = pi_type;
	blk_queue_logical_block_size(ns->queue, bs);

1997
	if (ns->ms && !ns->ext)
K
Keith Busch 已提交
1998 1999
		nvme_init_integrity(ns);

M
Matias Bjørling 已提交
2000 2001 2002
	if ((ns->ms && !(ns->ms == 8 && ns->pi_type) &&
						!blk_get_integrity(disk)) ||
						ns->type == NVME_NS_LIGHTNVM)
K
Keith Busch 已提交
2003 2004 2005 2006
		set_capacity(disk, 0);
	else
		set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));

2007
	if (dev->ctrl.oncs & NVME_CTRL_ONCS_DSM)
K
Keith Busch 已提交
2008
		nvme_config_discard(ns);
2009
	blk_mq_unfreeze_queue(disk->queue);
K
Keith Busch 已提交
2010

2011
	kfree(id);
K
Keith Busch 已提交
2012 2013 2014
	return 0;
}

K
Keith Busch 已提交
2015 2016 2017 2018 2019 2020 2021 2022 2023 2024 2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046
static char nvme_pr_type(enum pr_type type)
{
	switch (type) {
	case PR_WRITE_EXCLUSIVE:
		return 1;
	case PR_EXCLUSIVE_ACCESS:
		return 2;
	case PR_WRITE_EXCLUSIVE_REG_ONLY:
		return 3;
	case PR_EXCLUSIVE_ACCESS_REG_ONLY:
		return 4;
	case PR_WRITE_EXCLUSIVE_ALL_REGS:
		return 5;
	case PR_EXCLUSIVE_ACCESS_ALL_REGS:
		return 6;
	default:
		return 0;
	}
};

static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
				u64 key, u64 sa_key, u8 op)
{
	struct nvme_ns *ns = bdev->bd_disk->private_data;
	struct nvme_command c;
	u8 data[16] = { 0, };

	put_unaligned_le64(key, &data[0]);
	put_unaligned_le64(sa_key, &data[8]);

	memset(&c, 0, sizeof(c));
	c.common.opcode = op;
2047 2048
	c.common.nsid = cpu_to_le32(ns->ns_id);
	c.common.cdw10[0] = cpu_to_le32(cdw10);
K
Keith Busch 已提交
2049 2050 2051 2052 2053 2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085 2086 2087 2088

	return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
}

static int nvme_pr_register(struct block_device *bdev, u64 old,
		u64 new, unsigned flags)
{
	u32 cdw10;

	if (flags & ~PR_FL_IGNORE_KEY)
		return -EOPNOTSUPP;

	cdw10 = old ? 2 : 0;
	cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
	cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
}

static int nvme_pr_reserve(struct block_device *bdev, u64 key,
		enum pr_type type, unsigned flags)
{
	u32 cdw10;

	if (flags & ~PR_FL_IGNORE_KEY)
		return -EOPNOTSUPP;

	cdw10 = nvme_pr_type(type) << 8;
	cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
}

static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
		enum pr_type type, bool abort)
{
	u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
}

static int nvme_pr_clear(struct block_device *bdev, u64 key)
{
2089
	u32 cdw10 = 1 | (key ? 1 << 3 : 0);
K
Keith Busch 已提交
2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
}

static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
{
	u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
}

static const struct pr_ops nvme_pr_ops = {
	.pr_register	= nvme_pr_register,
	.pr_reserve	= nvme_pr_reserve,
	.pr_release	= nvme_pr_release,
	.pr_preempt	= nvme_pr_preempt,
	.pr_clear	= nvme_pr_clear,
};

M
Matthew Wilcox 已提交
2107 2108 2109
static const struct block_device_operations nvme_fops = {
	.owner		= THIS_MODULE,
	.ioctl		= nvme_ioctl,
K
Keith Busch 已提交
2110
	.compat_ioctl	= nvme_compat_ioctl,
2111 2112
	.open		= nvme_open,
	.release	= nvme_release,
K
Keith Busch 已提交
2113
	.getgeo		= nvme_getgeo,
K
Keith Busch 已提交
2114
	.revalidate_disk= nvme_revalidate_disk,
K
Keith Busch 已提交
2115
	.pr_ops		= &nvme_pr_ops,
M
Matthew Wilcox 已提交
2116 2117
};

2118 2119
static int nvme_kthread(void *data)
{
2120
	struct nvme_dev *dev, *next;
2121 2122

	while (!kthread_should_stop()) {
2123
		set_current_state(TASK_INTERRUPTIBLE);
2124
		spin_lock(&dev_list_lock);
2125
		list_for_each_entry_safe(dev, next, &dev_list, node) {
2126
			int i;
2127
			u32 csts = readl(dev->bar + NVME_REG_CSTS);
2128 2129 2130

			if ((dev->subsystem && (csts & NVME_CSTS_NSSRO)) ||
							csts & NVME_CSTS_CFS) {
2131 2132 2133
				if (!__nvme_reset(dev)) {
					dev_warn(dev->dev,
						"Failed status: %x, reset controller\n",
2134
						readl(dev->bar + NVME_REG_CSTS));
2135
				}
2136 2137
				continue;
			}
2138
			for (i = 0; i < dev->queue_count; i++) {
M
Matias Bjørling 已提交
2139
				struct nvme_queue *nvmeq = dev->queues[i];
2140 2141
				if (!nvmeq)
					continue;
2142
				spin_lock_irq(&nvmeq->q_lock);
2143
				nvme_process_cq(nvmeq);
K
Keith Busch 已提交
2144

2145
				while (i == 0 && dev->ctrl.event_limit > 0) {
M
Matias Bjørling 已提交
2146
					if (nvme_submit_async_admin_req(dev))
K
Keith Busch 已提交
2147
						break;
2148
					dev->ctrl.event_limit--;
K
Keith Busch 已提交
2149
				}
2150 2151 2152 2153
				spin_unlock_irq(&nvmeq->q_lock);
			}
		}
		spin_unlock(&dev_list_lock);
2154
		schedule_timeout(round_jiffies_relative(HZ));
2155 2156 2157 2158
	}
	return 0;
}

K
Keith Busch 已提交
2159
static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
M
Matthew Wilcox 已提交
2160 2161 2162
{
	struct nvme_ns *ns;
	struct gendisk *disk;
2163
	int node = dev_to_node(dev->dev);
M
Matthew Wilcox 已提交
2164

M
Matias Bjørling 已提交
2165
	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2166
	if (!ns)
K
Keith Busch 已提交
2167 2168
		return;

M
Matias Bjørling 已提交
2169
	ns->queue = blk_mq_init_queue(&dev->tagset);
2170
	if (IS_ERR(ns->queue))
M
Matthew Wilcox 已提交
2171
		goto out_free_ns;
M
Matthew Wilcox 已提交
2172 2173
	queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
2174
	ns->ctrl = &dev->ctrl;
M
Matthew Wilcox 已提交
2175 2176
	ns->queue->queuedata = ns;

M
Matias Bjørling 已提交
2177
	disk = alloc_disk_node(0, node);
M
Matthew Wilcox 已提交
2178 2179
	if (!disk)
		goto out_free_queue;
M
Matias Bjørling 已提交
2180

2181
	kref_init(&ns->kref);
2182
	ns->ns_id = nsid;
M
Matthew Wilcox 已提交
2183
	ns->disk = disk;
K
Keith Busch 已提交
2184 2185 2186
	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
	list_add_tail(&ns->list, &dev->namespaces);

2187
	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
K
Keith Busch 已提交
2188
	if (dev->max_hw_sectors) {
K
Keith Busch 已提交
2189
		blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
K
Keith Busch 已提交
2190
		blk_queue_max_segments(ns->queue,
2191
			(dev->max_hw_sectors / (dev->page_size >> 9)) + 1);
K
Keith Busch 已提交
2192
	}
M
Matias Bjørling 已提交
2193 2194
	if (dev->stripe_size)
		blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
2195
	if (dev->ctrl.vwc & NVME_CTRL_VWC_PRESENT)
2196
		blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
2197
	blk_queue_virt_boundary(ns->queue, dev->page_size - 1);
M
Matthew Wilcox 已提交
2198 2199

	disk->major = nvme_major;
2200
	disk->first_minor = 0;
M
Matthew Wilcox 已提交
2201 2202 2203
	disk->fops = &nvme_fops;
	disk->private_data = ns;
	disk->queue = ns->queue;
2204
	disk->driverfs_dev = dev->device;
2205
	disk->flags = GENHD_FL_EXT_DEVT;
2206
	sprintf(disk->disk_name, "nvme%dn%d", dev->ctrl.instance, nsid);
M
Matthew Wilcox 已提交
2207

K
Keith Busch 已提交
2208 2209 2210 2211 2212 2213 2214
	/*
	 * Initialize capacity to 0 until we establish the namespace format and
	 * setup integrity extentions if necessary. The revalidate_disk after
	 * add_disk allows the driver to register with integrity if the format
	 * requires it.
	 */
	set_capacity(disk, 0);
K
Keith Busch 已提交
2215 2216 2217
	if (nvme_revalidate_disk(ns->disk))
		goto out_free_disk;

2218
	kref_get(&dev->kref);
M
Matias Bjørling 已提交
2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230
	if (ns->type != NVME_NS_LIGHTNVM) {
		add_disk(ns->disk);
		if (ns->ms) {
			struct block_device *bd = bdget_disk(ns->disk, 0);
			if (!bd)
				return;
			if (blkdev_get(bd, FMODE_READ, NULL)) {
				bdput(bd);
				return;
			}
			blkdev_reread_part(bd);
			blkdev_put(bd, FMODE_READ);
2231 2232
		}
	}
K
Keith Busch 已提交
2233
	return;
K
Keith Busch 已提交
2234 2235 2236
 out_free_disk:
	kfree(disk);
	list_del(&ns->list);
M
Matthew Wilcox 已提交
2237 2238 2239 2240 2241 2242
 out_free_queue:
	blk_cleanup_queue(ns->queue);
 out_free_ns:
	kfree(ns);
}

2243 2244 2245 2246 2247 2248 2249
/*
 * Create I/O queues.  Failing to create an I/O queue is not an issue,
 * we can continue with less than the desired amount of queues, and
 * even a controller without I/O queues an still be used to issue
 * admin commands.  This might be useful to upgrade a buggy firmware
 * for example.
 */
K
Keith Busch 已提交
2250 2251
static void nvme_create_io_queues(struct nvme_dev *dev)
{
M
Matias Bjørling 已提交
2252
	unsigned i;
K
Keith Busch 已提交
2253

M
Matias Bjørling 已提交
2254
	for (i = dev->queue_count; i <= dev->max_qid; i++)
K
Keith Busch 已提交
2255
		if (!nvme_alloc_queue(dev, i, dev->q_depth))
K
Keith Busch 已提交
2256 2257
			break;

M
Matias Bjørling 已提交
2258
	for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
2259 2260
		if (nvme_create_queue(dev->queues[i], i)) {
			nvme_free_queues(dev, i);
K
Keith Busch 已提交
2261
			break;
2262
		}
K
Keith Busch 已提交
2263 2264
}

2265
static int set_queue_count(struct nvme_dev *dev, int count)
M
Matthew Wilcox 已提交
2266 2267 2268
{
	int status;
	u32 result;
2269
	u32 q_count = (count - 1) | ((count - 1) << 16);
M
Matthew Wilcox 已提交
2270

2271
	status = nvme_set_features(&dev->ctrl, NVME_FEAT_NUM_QUEUES, q_count, 0,
2272
								&result);
M
Matthew Wilcox 已提交
2273 2274 2275
	if (status < 0)
		return status;
	if (status > 0) {
2276
		dev_err(dev->dev, "Could not set queue count (%d)\n", status);
2277
		return 0;
M
Matthew Wilcox 已提交
2278
	}
M
Matthew Wilcox 已提交
2279 2280 2281
	return min(result & 0xffff, result >> 16) + 1;
}

2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293
static void __iomem *nvme_map_cmb(struct nvme_dev *dev)
{
	u64 szu, size, offset;
	u32 cmbloc;
	resource_size_t bar_size;
	struct pci_dev *pdev = to_pci_dev(dev->dev);
	void __iomem *cmb;
	dma_addr_t dma_addr;

	if (!use_cmb_sqes)
		return NULL;

2294
	dev->cmbsz = readl(dev->bar + NVME_REG_CMBSZ);
2295 2296 2297
	if (!(NVME_CMB_SZ(dev->cmbsz)))
		return NULL;

2298
	cmbloc = readl(dev->bar + NVME_REG_CMBLOC);
2299 2300 2301 2302 2303 2304 2305 2306 2307 2308 2309 2310 2311 2312 2313 2314 2315 2316 2317 2318 2319 2320 2321 2322 2323 2324 2325 2326 2327 2328 2329 2330 2331 2332 2333

	szu = (u64)1 << (12 + 4 * NVME_CMB_SZU(dev->cmbsz));
	size = szu * NVME_CMB_SZ(dev->cmbsz);
	offset = szu * NVME_CMB_OFST(cmbloc);
	bar_size = pci_resource_len(pdev, NVME_CMB_BIR(cmbloc));

	if (offset > bar_size)
		return NULL;

	/*
	 * Controllers may support a CMB size larger than their BAR,
	 * for example, due to being behind a bridge. Reduce the CMB to
	 * the reported size of the BAR
	 */
	if (size > bar_size - offset)
		size = bar_size - offset;

	dma_addr = pci_resource_start(pdev, NVME_CMB_BIR(cmbloc)) + offset;
	cmb = ioremap_wc(dma_addr, size);
	if (!cmb)
		return NULL;

	dev->cmb_dma_addr = dma_addr;
	dev->cmb_size = size;
	return cmb;
}

static inline void nvme_release_cmb(struct nvme_dev *dev)
{
	if (dev->cmb) {
		iounmap(dev->cmb);
		dev->cmb = NULL;
	}
}

K
Keith Busch 已提交
2334 2335
static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
{
2336
	return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
K
Keith Busch 已提交
2337 2338
}

2339
static int nvme_setup_io_queues(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2340
{
M
Matias Bjørling 已提交
2341
	struct nvme_queue *adminq = dev->queues[0];
2342
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
2343
	int result, i, vecs, nr_io_queues, size;
M
Matthew Wilcox 已提交
2344

K
Keith Busch 已提交
2345
	nr_io_queues = num_possible_cpus();
2346
	result = set_queue_count(dev, nr_io_queues);
2347
	if (result <= 0)
M
Matthew Wilcox 已提交
2348
		return result;
2349 2350
	if (result < nr_io_queues)
		nr_io_queues = result;
M
Matthew Wilcox 已提交
2351

2352 2353 2354 2355 2356 2357 2358 2359 2360
	if (dev->cmb && NVME_CMB_SQS(dev->cmbsz)) {
		result = nvme_cmb_qdepth(dev, nr_io_queues,
				sizeof(struct nvme_command));
		if (result > 0)
			dev->q_depth = result;
		else
			nvme_release_cmb(dev);
	}

K
Keith Busch 已提交
2361 2362
	size = db_bar_size(dev, nr_io_queues);
	if (size > 8192) {
2363
		iounmap(dev->bar);
K
Keith Busch 已提交
2364 2365 2366 2367 2368 2369 2370 2371
		do {
			dev->bar = ioremap(pci_resource_start(pdev, 0), size);
			if (dev->bar)
				break;
			if (!--nr_io_queues)
				return -ENOMEM;
			size = db_bar_size(dev, nr_io_queues);
		} while (1);
2372
		dev->dbs = dev->bar + 4096;
2373
		adminq->q_db = dev->dbs;
2374 2375
	}

K
Keith Busch 已提交
2376
	/* Deregister the admin queue's interrupt */
2377
	free_irq(dev->entry[0].vector, adminq);
K
Keith Busch 已提交
2378

2379 2380 2381 2382 2383 2384 2385
	/*
	 * If we enable msix early due to not intx, disable it again before
	 * setting up the full range we need.
	 */
	if (!pdev->irq)
		pci_disable_msix(pdev);

2386
	for (i = 0; i < nr_io_queues; i++)
M
Matthew Wilcox 已提交
2387
		dev->entry[i].entry = i;
2388 2389 2390 2391 2392 2393 2394 2395
	vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
	if (vecs < 0) {
		vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
		if (vecs < 0) {
			vecs = 1;
		} else {
			for (i = 0; i < vecs; i++)
				dev->entry[i].vector = i + pdev->irq;
R
Ramachandra Rao Gajula 已提交
2396 2397 2398
		}
	}

2399 2400 2401 2402 2403 2404 2405
	/*
	 * Should investigate if there's a performance win from allocating
	 * more queues than interrupt vectors; it might allow the submission
	 * path to scale better, even if the receive path is limited by the
	 * number of interrupts.
	 */
	nr_io_queues = vecs;
K
Keith Busch 已提交
2406
	dev->max_qid = nr_io_queues;
2407

2408
	result = queue_request_irq(dev, adminq, adminq->irqname);
2409 2410
	if (result) {
		adminq->cq_vector = -1;
2411
		goto free_queues;
2412
	}
M
Matthew Wilcox 已提交
2413

2414
	/* Free previously allocated queues that are no longer usable */
K
Keith Busch 已提交
2415
	nvme_free_queues(dev, nr_io_queues + 1);
M
Matias Bjørling 已提交
2416
	nvme_create_io_queues(dev);
M
Matthew Wilcox 已提交
2417

2418
	return 0;
M
Matthew Wilcox 已提交
2419

2420
 free_queues:
2421
	nvme_free_queues(dev, 1);
2422
	return result;
M
Matthew Wilcox 已提交
2423 2424
}

K
Keith Busch 已提交
2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447
static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
	struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);

	return nsa->ns_id - nsb->ns_id;
}

static struct nvme_ns *nvme_find_ns(struct nvme_dev *dev, unsigned nsid)
{
	struct nvme_ns *ns;

	list_for_each_entry(ns, &dev->namespaces, list) {
		if (ns->ns_id == nsid)
			return ns;
		if (ns->ns_id > nsid)
			break;
	}
	return NULL;
}

static inline bool nvme_io_incapable(struct nvme_dev *dev)
{
2448 2449 2450
	return (!dev->bar ||
		readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_CFS ||
		dev->online_queues < 2);
K
Keith Busch 已提交
2451 2452 2453 2454
}

static void nvme_ns_remove(struct nvme_ns *ns)
{
2455 2456
	bool kill = nvme_io_incapable(to_nvme_dev(ns->ctrl)) &&
			!blk_queue_dying(ns->queue);
K
Keith Busch 已提交
2457 2458 2459

	if (kill)
		blk_set_queue_dying(ns->queue);
2460
	if (ns->disk->flags & GENHD_FL_UP)
K
Keith Busch 已提交
2461 2462 2463 2464
		del_gendisk(ns->disk);
	if (kill || !blk_queue_dying(ns->queue)) {
		blk_mq_abort_requeue_list(ns->queue);
		blk_cleanup_queue(ns->queue);
2465 2466 2467
	}
	list_del_init(&ns->list);
	kref_put(&ns->kref, nvme_free_ns);
K
Keith Busch 已提交
2468 2469 2470 2471 2472 2473 2474 2475 2476 2477
}

static void nvme_scan_namespaces(struct nvme_dev *dev, unsigned nn)
{
	struct nvme_ns *ns, *next;
	unsigned i;

	for (i = 1; i <= nn; i++) {
		ns = nvme_find_ns(dev, i);
		if (ns) {
2478
			if (revalidate_disk(ns->disk))
K
Keith Busch 已提交
2479 2480 2481 2482 2483
				nvme_ns_remove(ns);
		} else
			nvme_alloc_ns(dev, i);
	}
	list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
2484
		if (ns->ns_id > nn)
K
Keith Busch 已提交
2485 2486 2487 2488 2489
			nvme_ns_remove(ns);
	}
	list_sort(NULL, &dev->namespaces, ns_cmp);
}

2490 2491 2492 2493 2494 2495 2496 2497 2498 2499 2500 2501 2502 2503 2504 2505
static void nvme_set_irq_hints(struct nvme_dev *dev)
{
	struct nvme_queue *nvmeq;
	int i;

	for (i = 0; i < dev->online_queues; i++) {
		nvmeq = dev->queues[i];

		if (!nvmeq->tags || !(*nvmeq->tags))
			continue;

		irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
					blk_mq_tags_cpumask(*nvmeq->tags));
	}
}

K
Keith Busch 已提交
2506 2507 2508 2509 2510 2511 2512
static void nvme_dev_scan(struct work_struct *work)
{
	struct nvme_dev *dev = container_of(work, struct nvme_dev, scan_work);
	struct nvme_id_ctrl *ctrl;

	if (!dev->tagset.tags)
		return;
2513
	if (nvme_identify_ctrl(&dev->ctrl, &ctrl))
K
Keith Busch 已提交
2514 2515 2516
		return;
	nvme_scan_namespaces(dev, le32_to_cpup(&ctrl->nn));
	kfree(ctrl);
2517
	nvme_set_irq_hints(dev);
K
Keith Busch 已提交
2518 2519
}

2520 2521 2522 2523 2524 2525
/*
 * Return: error value if an error occurred setting up the queues or calling
 * Identify Device.  0 if these succeeded, even if adding some of the
 * namespaces failed.  At the moment, these failures are silent.  TBD which
 * failures should be reported.
 */
2526
static int nvme_dev_add(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2527
{
2528
	struct pci_dev *pdev = to_pci_dev(dev->dev);
2529
	int res;
2530
	struct nvme_id_ctrl *ctrl;
2531
	int shift = NVME_CAP_MPSMIN(lo_hi_readq(dev->bar + NVME_REG_CAP)) + 12;
M
Matthew Wilcox 已提交
2532

2533
	res = nvme_identify_ctrl(&dev->ctrl, &ctrl);
M
Matthew Wilcox 已提交
2534
	if (res) {
2535
		dev_err(dev->dev, "Identify Controller failed (%d)\n", res);
K
Keith Busch 已提交
2536
		return -EIO;
M
Matthew Wilcox 已提交
2537 2538
	}

2539 2540 2541 2542 2543 2544
	dev->ctrl.oncs = le16_to_cpup(&ctrl->oncs);
	dev->ctrl.abort_limit = ctrl->acl + 1;
	dev->ctrl.vwc = ctrl->vwc;
	memcpy(dev->ctrl.serial, ctrl->sn, sizeof(ctrl->sn));
	memcpy(dev->ctrl.model, ctrl->mn, sizeof(ctrl->mn));
	memcpy(dev->ctrl.firmware_rev, ctrl->fr, sizeof(ctrl->fr));
2545
	if (ctrl->mdts)
K
Keith Busch 已提交
2546
		dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
2547 2548
	else
		dev->max_hw_sectors = UINT_MAX;
2549
	if ((pdev->vendor == PCI_VENDOR_ID_INTEL) &&
M
Matias Bjørling 已提交
2550 2551 2552
			(pdev->device == 0x0953) && ctrl->vs[3]) {
		unsigned int max_hw_sectors;

2553
		dev->stripe_size = 1 << (ctrl->vs[3] + shift);
M
Matias Bjørling 已提交
2554 2555 2556 2557 2558 2559 2560
		max_hw_sectors = dev->stripe_size >> (shift - 9);
		if (dev->max_hw_sectors) {
			dev->max_hw_sectors = min(max_hw_sectors,
							dev->max_hw_sectors);
		} else
			dev->max_hw_sectors = max_hw_sectors;
	}
2561
	kfree(ctrl);
M
Matias Bjørling 已提交
2562

2563 2564 2565 2566 2567 2568
	if (!dev->tagset.tags) {
		dev->tagset.ops = &nvme_mq_ops;
		dev->tagset.nr_hw_queues = dev->online_queues - 1;
		dev->tagset.timeout = NVME_IO_TIMEOUT;
		dev->tagset.numa_node = dev_to_node(dev->dev);
		dev->tagset.queue_depth =
M
Matias Bjørling 已提交
2569
				min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
2570 2571 2572
		dev->tagset.cmd_size = nvme_cmd_size(dev);
		dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
		dev->tagset.driver_data = dev;
M
Matthew Wilcox 已提交
2573

2574 2575 2576
		if (blk_mq_alloc_tag_set(&dev->tagset))
			return 0;
	}
K
Keith Busch 已提交
2577
	schedule_work(&dev->scan_work);
K
Keith Busch 已提交
2578
	return 0;
M
Matthew Wilcox 已提交
2579 2580
}

2581 2582
static int nvme_dev_map(struct nvme_dev *dev)
{
K
Keith Busch 已提交
2583
	u64 cap;
2584
	int bars, result = -ENOMEM;
2585
	struct pci_dev *pdev = to_pci_dev(dev->dev);
2586 2587 2588 2589 2590 2591 2592

	if (pci_enable_device_mem(pdev))
		return result;

	dev->entry[0].vector = pdev->irq;
	pci_set_master(pdev);
	bars = pci_select_bars(pdev, IORESOURCE_MEM);
2593 2594 2595
	if (!bars)
		goto disable_pci;

2596 2597 2598
	if (pci_request_selected_regions(pdev, bars, "nvme"))
		goto disable_pci;

2599 2600
	if (dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(64)) &&
	    dma_set_mask_and_coherent(dev->dev, DMA_BIT_MASK(32)))
2601
		goto disable;
2602 2603 2604 2605

	dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
	if (!dev->bar)
		goto disable;
2606

2607
	if (readl(dev->bar + NVME_REG_CSTS) == -1) {
K
Keith Busch 已提交
2608 2609 2610
		result = -ENODEV;
		goto unmap;
	}
2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621

	/*
	 * Some devices don't advertse INTx interrupts, pre-enable a single
	 * MSIX vec for setup. We'll adjust this later.
	 */
	if (!pdev->irq) {
		result = pci_enable_msix(pdev, dev->entry, 1);
		if (result < 0)
			goto unmap;
	}

2622 2623
	cap = lo_hi_readq(dev->bar + NVME_REG_CAP);

K
Keith Busch 已提交
2624 2625
	dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
	dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
2626 2627
	dev->dbs = dev->bar + 4096;
	if (readl(dev->bar + NVME_REG_VS) >= NVME_VS(1, 2))
2628
		dev->cmb = nvme_map_cmb(dev);
2629 2630 2631

	return 0;

K
Keith Busch 已提交
2632 2633 2634
 unmap:
	iounmap(dev->bar);
	dev->bar = NULL;
2635 2636 2637 2638 2639 2640 2641 2642 2643
 disable:
	pci_release_regions(pdev);
 disable_pci:
	pci_disable_device(pdev);
	return result;
}

static void nvme_dev_unmap(struct nvme_dev *dev)
{
2644 2645 2646 2647 2648 2649
	struct pci_dev *pdev = to_pci_dev(dev->dev);

	if (pdev->msi_enabled)
		pci_disable_msi(pdev);
	else if (pdev->msix_enabled)
		pci_disable_msix(pdev);
2650 2651 2652 2653

	if (dev->bar) {
		iounmap(dev->bar);
		dev->bar = NULL;
2654
		pci_release_regions(pdev);
2655 2656
	}

2657 2658
	if (pci_is_enabled(pdev))
		pci_disable_device(pdev);
2659 2660
}

K
Keith Busch 已提交
2661 2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676 2677
struct nvme_delq_ctx {
	struct task_struct *waiter;
	struct kthread_worker *worker;
	atomic_t refcount;
};

static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
{
	dq->waiter = current;
	mb();

	for (;;) {
		set_current_state(TASK_KILLABLE);
		if (!atomic_read(&dq->refcount))
			break;
		if (!schedule_timeout(ADMIN_TIMEOUT) ||
					fatal_signal_pending(current)) {
K
Keith Busch 已提交
2678 2679 2680 2681 2682 2683 2684
			/*
			 * Disable the controller first since we can't trust it
			 * at this point, but leave the admin queue enabled
			 * until all queue deletion requests are flushed.
			 * FIXME: This may take a while if there are more h/w
			 * queues than admin tags.
			 */
K
Keith Busch 已提交
2685
			set_current_state(TASK_RUNNING);
2686 2687
			nvme_disable_ctrl(dev,
				lo_hi_readq(dev->bar + NVME_REG_CAP));
K
Keith Busch 已提交
2688
			nvme_clear_queue(dev->queues[0]);
K
Keith Busch 已提交
2689
			flush_kthread_worker(dq->worker);
K
Keith Busch 已提交
2690
			nvme_disable_queue(dev, 0);
K
Keith Busch 已提交
2691 2692 2693 2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709 2710 2711 2712 2713
			return;
		}
	}
	set_current_state(TASK_RUNNING);
}

static void nvme_put_dq(struct nvme_delq_ctx *dq)
{
	atomic_dec(&dq->refcount);
	if (dq->waiter)
		wake_up_process(dq->waiter);
}

static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq)
{
	atomic_inc(&dq->refcount);
	return dq;
}

static void nvme_del_queue_end(struct nvme_queue *nvmeq)
{
	struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx;
	nvme_put_dq(dq);
2714 2715 2716 2717

	spin_lock_irq(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
K
Keith Busch 已提交
2718 2719 2720 2721 2722 2723 2724 2725 2726 2727 2728 2729
}

static int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode,
						kthread_work_func_t fn)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(nvmeq->qid);

	init_kthread_work(&nvmeq->cmdinfo.work, fn);
M
Matias Bjørling 已提交
2730 2731
	return nvme_submit_admin_async_cmd(nvmeq->dev, &c, &nvmeq->cmdinfo,
								ADMIN_TIMEOUT);
K
Keith Busch 已提交
2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775 2776 2777 2778
}

static void nvme_del_cq_work_handler(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	nvme_del_queue_end(nvmeq);
}

static int nvme_delete_cq(struct nvme_queue *nvmeq)
{
	return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq,
						nvme_del_cq_work_handler);
}

static void nvme_del_sq_work_handler(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	int status = nvmeq->cmdinfo.status;

	if (!status)
		status = nvme_delete_cq(nvmeq);
	if (status)
		nvme_del_queue_end(nvmeq);
}

static int nvme_delete_sq(struct nvme_queue *nvmeq)
{
	return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq,
						nvme_del_sq_work_handler);
}

static void nvme_del_queue_start(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	if (nvme_delete_sq(nvmeq))
		nvme_del_queue_end(nvmeq);
}

static void nvme_disable_io_queues(struct nvme_dev *dev)
{
	int i;
	DEFINE_KTHREAD_WORKER_ONSTACK(worker);
	struct nvme_delq_ctx dq;
	struct task_struct *kworker_task = kthread_run(kthread_worker_fn,
2779
					&worker, "nvme%d", dev->ctrl.instance);
K
Keith Busch 已提交
2780 2781

	if (IS_ERR(kworker_task)) {
2782
		dev_err(dev->dev,
K
Keith Busch 已提交
2783 2784 2785 2786 2787 2788 2789 2790 2791 2792
			"Failed to create queue del task\n");
		for (i = dev->queue_count - 1; i > 0; i--)
			nvme_disable_queue(dev, i);
		return;
	}

	dq.waiter = NULL;
	atomic_set(&dq.refcount, 0);
	dq.worker = &worker;
	for (i = dev->queue_count - 1; i > 0; i--) {
M
Matias Bjørling 已提交
2793
		struct nvme_queue *nvmeq = dev->queues[i];
K
Keith Busch 已提交
2794 2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805

		if (nvme_suspend_queue(nvmeq))
			continue;
		nvmeq->cmdinfo.ctx = nvme_get_dq(&dq);
		nvmeq->cmdinfo.worker = dq.worker;
		init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start);
		queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work);
	}
	nvme_wait_dq(&dq, dev);
	kthread_stop(kworker_task);
}

2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825
/*
* Remove the node from the device list and check
* for whether or not we need to stop the nvme_thread.
*/
static void nvme_dev_list_remove(struct nvme_dev *dev)
{
	struct task_struct *tmp = NULL;

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) {
		tmp = nvme_thread;
		nvme_thread = NULL;
	}
	spin_unlock(&dev_list_lock);

	if (tmp)
		kthread_stop(tmp);
}

2826 2827 2828 2829 2830 2831 2832
static void nvme_freeze_queues(struct nvme_dev *dev)
{
	struct nvme_ns *ns;

	list_for_each_entry(ns, &dev->namespaces, list) {
		blk_mq_freeze_queue_start(ns->queue);

2833
		spin_lock_irq(ns->queue->queue_lock);
2834
		queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
2835
		spin_unlock_irq(ns->queue->queue_lock);
2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848 2849 2850 2851 2852 2853

		blk_mq_cancel_requeue_work(ns->queue);
		blk_mq_stop_hw_queues(ns->queue);
	}
}

static void nvme_unfreeze_queues(struct nvme_dev *dev)
{
	struct nvme_ns *ns;

	list_for_each_entry(ns, &dev->namespaces, list) {
		queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
		blk_mq_unfreeze_queue(ns->queue);
		blk_mq_start_stopped_hw_queues(ns->queue, true);
		blk_mq_kick_requeue_list(ns->queue);
	}
}

2854
static void nvme_dev_shutdown(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2855
{
2856
	int i;
2857
	u32 csts = -1;
2858

2859
	nvme_dev_list_remove(dev);
2860

2861 2862
	if (dev->bar) {
		nvme_freeze_queues(dev);
2863
		csts = readl(dev->bar + NVME_REG_CSTS);
2864
	}
2865
	if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
K
Keith Busch 已提交
2866
		for (i = dev->queue_count - 1; i >= 0; i--) {
M
Matias Bjørling 已提交
2867
			struct nvme_queue *nvmeq = dev->queues[i];
K
Keith Busch 已提交
2868 2869 2870 2871
			nvme_suspend_queue(nvmeq);
		}
	} else {
		nvme_disable_io_queues(dev);
K
Keith Busch 已提交
2872
		nvme_shutdown_ctrl(dev);
K
Keith Busch 已提交
2873 2874
		nvme_disable_queue(dev, 0);
	}
2875
	nvme_dev_unmap(dev);
2876 2877 2878

	for (i = dev->queue_count - 1; i >= 0; i--)
		nvme_clear_queue(dev->queues[i]);
2879 2880 2881 2882
}

static void nvme_dev_remove(struct nvme_dev *dev)
{
2883
	struct nvme_ns *ns, *next;
2884

2885
	list_for_each_entry_safe(ns, next, &dev->namespaces, list)
K
Keith Busch 已提交
2886
		nvme_ns_remove(ns);
M
Matthew Wilcox 已提交
2887 2888
}

M
Matthew Wilcox 已提交
2889 2890
static int nvme_setup_prp_pools(struct nvme_dev *dev)
{
2891
	dev->prp_page_pool = dma_pool_create("prp list page", dev->dev,
M
Matthew Wilcox 已提交
2892 2893 2894 2895
						PAGE_SIZE, PAGE_SIZE, 0);
	if (!dev->prp_page_pool)
		return -ENOMEM;

2896
	/* Optimisation for I/Os between 4k and 128k */
2897
	dev->prp_small_pool = dma_pool_create("prp list 256", dev->dev,
2898 2899 2900 2901 2902
						256, 256, 0);
	if (!dev->prp_small_pool) {
		dma_pool_destroy(dev->prp_page_pool);
		return -ENOMEM;
	}
M
Matthew Wilcox 已提交
2903 2904 2905 2906 2907 2908
	return 0;
}

static void nvme_release_prp_pools(struct nvme_dev *dev)
{
	dma_pool_destroy(dev->prp_page_pool);
2909
	dma_pool_destroy(dev->prp_small_pool);
M
Matthew Wilcox 已提交
2910 2911
}

2912 2913 2914
static DEFINE_IDA(nvme_instance_ida);

static int nvme_set_instance(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2915
{
2916 2917 2918 2919 2920 2921 2922 2923 2924 2925 2926 2927 2928 2929
	int instance, error;

	do {
		if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
			return -ENODEV;

		spin_lock(&dev_list_lock);
		error = ida_get_new(&nvme_instance_ida, &instance);
		spin_unlock(&dev_list_lock);
	} while (error == -EAGAIN);

	if (error)
		return -ENODEV;

2930
	dev->ctrl.instance = instance;
2931
	return 0;
M
Matthew Wilcox 已提交
2932 2933 2934 2935
}

static void nvme_release_instance(struct nvme_dev *dev)
{
2936
	spin_lock(&dev_list_lock);
2937
	ida_remove(&nvme_instance_ida, dev->ctrl.instance);
2938
	spin_unlock(&dev_list_lock);
M
Matthew Wilcox 已提交
2939 2940
}

2941 2942 2943
static void nvme_free_dev(struct kref *kref)
{
	struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
2944

2945
	put_device(dev->dev);
2946
	put_device(dev->device);
2947
	nvme_release_instance(dev);
2948 2949
	if (dev->tagset.tags)
		blk_mq_free_tag_set(&dev->tagset);
2950 2951
	if (dev->ctrl.admin_q)
		blk_put_queue(dev->ctrl.admin_q);
2952 2953 2954 2955 2956 2957 2958
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
}

static int nvme_dev_open(struct inode *inode, struct file *f)
{
2959 2960 2961 2962 2963 2964
	struct nvme_dev *dev;
	int instance = iminor(inode);
	int ret = -ENODEV;

	spin_lock(&dev_list_lock);
	list_for_each_entry(dev, &dev_list, node) {
2965 2966
		if (dev->ctrl.instance == instance) {
			if (!dev->ctrl.admin_q) {
2967 2968 2969
				ret = -EWOULDBLOCK;
				break;
			}
2970 2971 2972 2973 2974 2975 2976 2977 2978 2979
			if (!kref_get_unless_zero(&dev->kref))
				break;
			f->private_data = dev;
			ret = 0;
			break;
		}
	}
	spin_unlock(&dev_list_lock);

	return ret;
2980 2981 2982 2983 2984 2985 2986 2987 2988 2989 2990 2991
}

static int nvme_dev_release(struct inode *inode, struct file *f)
{
	struct nvme_dev *dev = f->private_data;
	kref_put(&dev->kref, nvme_free_dev);
	return 0;
}

static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
{
	struct nvme_dev *dev = f->private_data;
M
Matias Bjørling 已提交
2992 2993
	struct nvme_ns *ns;

2994 2995
	switch (cmd) {
	case NVME_IOCTL_ADMIN_CMD:
2996
		return nvme_user_cmd(&dev->ctrl, NULL, (void __user *)arg);
2997
	case NVME_IOCTL_IO_CMD:
M
Matias Bjørling 已提交
2998 2999 3000
		if (list_empty(&dev->namespaces))
			return -ENOTTY;
		ns = list_first_entry(&dev->namespaces, struct nvme_ns, list);
3001
		return nvme_user_cmd(&dev->ctrl, ns, (void __user *)arg);
3002 3003 3004
	case NVME_IOCTL_RESET:
		dev_warn(dev->dev, "resetting controller\n");
		return nvme_reset(dev);
3005 3006
	case NVME_IOCTL_SUBSYS_RESET:
		return nvme_subsys_reset(dev);
3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019
	default:
		return -ENOTTY;
	}
}

static const struct file_operations nvme_dev_fops = {
	.owner		= THIS_MODULE,
	.open		= nvme_dev_open,
	.release	= nvme_dev_release,
	.unlocked_ioctl	= nvme_dev_ioctl,
	.compat_ioctl	= nvme_dev_ioctl,
};

3020
static void nvme_probe_work(struct work_struct *work)
3021
{
3022
	struct nvme_dev *dev = container_of(work, struct nvme_dev, probe_work);
3023
	bool start_thread = false;
3024
	int result;
3025 3026 3027

	result = nvme_dev_map(dev);
	if (result)
3028
		goto out;
3029 3030 3031 3032 3033 3034

	result = nvme_configure_admin_queue(dev);
	if (result)
		goto unmap;

	spin_lock(&dev_list_lock);
3035 3036 3037 3038
	if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
		start_thread = true;
		nvme_thread = NULL;
	}
3039 3040 3041
	list_add(&dev->node, &dev_list);
	spin_unlock(&dev_list_lock);

3042 3043
	if (start_thread) {
		nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
3044
		wake_up_all(&nvme_kthread_wait);
3045 3046 3047 3048 3049 3050 3051
	} else
		wait_event_killable(nvme_kthread_wait, nvme_thread);

	if (IS_ERR_OR_NULL(nvme_thread)) {
		result = nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;
		goto disable;
	}
M
Matias Bjørling 已提交
3052 3053

	nvme_init_queue(dev->queues[0], 0);
K
Keith Busch 已提交
3054 3055 3056
	result = nvme_alloc_admin_tags(dev);
	if (result)
		goto disable;
3057

3058
	result = nvme_setup_io_queues(dev);
3059
	if (result)
K
Keith Busch 已提交
3060
		goto free_tags;
3061

3062
	dev->ctrl.event_limit = 1;
3063

3064 3065 3066 3067
	/*
	 * Keep the controller around but remove all namespaces if we don't have
	 * any working I/O queue.
	 */
3068 3069 3070 3071 3072 3073 3074 3075 3076
	if (dev->online_queues < 2) {
		dev_warn(dev->dev, "IO queues not created\n");
		nvme_dev_remove(dev);
	} else {
		nvme_unfreeze_queues(dev);
		nvme_dev_add(dev);
	}

	return;
3077

K
Keith Busch 已提交
3078 3079
 free_tags:
	nvme_dev_remove_admin(dev);
3080 3081
	blk_put_queue(dev->ctrl.admin_q);
	dev->ctrl.admin_q = NULL;
3082
	dev->queues[0]->tags = NULL;
3083
 disable:
3084
	nvme_disable_queue(dev, 0);
3085
	nvme_dev_list_remove(dev);
3086 3087
 unmap:
	nvme_dev_unmap(dev);
3088 3089 3090
 out:
	if (!work_busy(&dev->reset_work))
		nvme_dead_ctrl(dev);
3091 3092
}

K
Keith Busch 已提交
3093 3094 3095
static int nvme_remove_dead_ctrl(void *arg)
{
	struct nvme_dev *dev = (struct nvme_dev *)arg;
3096
	struct pci_dev *pdev = to_pci_dev(dev->dev);
K
Keith Busch 已提交
3097 3098

	if (pci_get_drvdata(pdev))
3099
		pci_stop_and_remove_bus_device_locked(pdev);
K
Keith Busch 已提交
3100 3101 3102 3103
	kref_put(&dev->kref, nvme_free_dev);
	return 0;
}

3104 3105 3106 3107 3108
static void nvme_dead_ctrl(struct nvme_dev *dev)
{
	dev_warn(dev->dev, "Device failed to resume\n");
	kref_get(&dev->kref);
	if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d",
3109
						dev->ctrl.instance))) {
3110 3111 3112 3113 3114 3115
		dev_err(dev->dev,
			"Failed to start controller remove task\n");
		kref_put(&dev->kref, nvme_free_dev);
	}
}

3116
static void nvme_reset_work(struct work_struct *ws)
K
Keith Busch 已提交
3117
{
3118
	struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
3119 3120
	bool in_probe = work_busy(&dev->probe_work);

K
Keith Busch 已提交
3121
	nvme_dev_shutdown(dev);
3122 3123 3124 3125 3126 3127 3128 3129

	/* Synchronize with device probe so that work will see failure status
	 * and exit gracefully without trying to schedule another reset */
	flush_work(&dev->probe_work);

	/* Fail this device if reset occured during probe to avoid
	 * infinite initialization loops. */
	if (in_probe) {
3130
		nvme_dead_ctrl(dev);
3131
		return;
K
Keith Busch 已提交
3132
	}
3133 3134 3135
	/* Schedule device resume asynchronously so the reset work is available
	 * to cleanup errors that may occur during reinitialization */
	schedule_work(&dev->probe_work);
K
Keith Busch 已提交
3136 3137
}

3138
static int __nvme_reset(struct nvme_dev *dev)
T
Tejun Heo 已提交
3139
{
3140 3141 3142 3143 3144
	if (work_pending(&dev->reset_work))
		return -EBUSY;
	list_del_init(&dev->node);
	queue_work(nvme_workq, &dev->reset_work);
	return 0;
T
Tejun Heo 已提交
3145 3146
}

3147 3148
static int nvme_reset(struct nvme_dev *dev)
{
3149
	int ret;
3150

3151
	if (!dev->ctrl.admin_q || blk_queue_dying(dev->ctrl.admin_q))
3152 3153 3154
		return -ENODEV;

	spin_lock(&dev_list_lock);
3155
	ret = __nvme_reset(dev);
3156 3157 3158 3159
	spin_unlock(&dev_list_lock);

	if (!ret) {
		flush_work(&dev->reset_work);
3160
		flush_work(&dev->probe_work);
3161 3162 3163 3164 3165 3166 3167 3168 3169 3170 3171 3172 3173 3174 3175 3176 3177 3178 3179 3180 3181
		return 0;
	}

	return ret;
}

static ssize_t nvme_sysfs_reset(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_dev *ndev = dev_get_drvdata(dev);
	int ret;

	ret = nvme_reset(ndev);
	if (ret < 0)
		return ret;

	return count;
}
static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);

3182 3183 3184 3185 3186 3187 3188 3189 3190 3191
static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val)
{
	*val = readl(to_nvme_dev(ctrl)->bar + off);
	return 0;
}

static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = {
	.reg_read32		= nvme_pci_reg_read32,
};

3192
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
M
Matthew Wilcox 已提交
3193
{
M
Matias Bjørling 已提交
3194
	int node, result = -ENOMEM;
M
Matthew Wilcox 已提交
3195 3196
	struct nvme_dev *dev;

M
Matias Bjørling 已提交
3197 3198 3199 3200 3201
	node = dev_to_node(&pdev->dev);
	if (node == NUMA_NO_NODE)
		set_dev_node(&pdev->dev, 0);

	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
3202 3203
	if (!dev)
		return -ENOMEM;
M
Matias Bjørling 已提交
3204 3205
	dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
3206 3207
	if (!dev->entry)
		goto free;
M
Matias Bjørling 已提交
3208 3209
	dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
3210 3211 3212 3213
	if (!dev->queues)
		goto free;

	INIT_LIST_HEAD(&dev->namespaces);
3214
	INIT_WORK(&dev->reset_work, nvme_reset_work);
3215
	dev->dev = get_device(&pdev->dev);
K
Keith Busch 已提交
3216
	pci_set_drvdata(pdev, dev);
3217 3218 3219 3220

	dev->ctrl.ops = &nvme_pci_ctrl_ops;
	dev->ctrl.dev = dev->dev;

3221 3222
	result = nvme_set_instance(dev);
	if (result)
K
Keith Busch 已提交
3223
		goto put_pci;
M
Matthew Wilcox 已提交
3224

M
Matthew Wilcox 已提交
3225 3226
	result = nvme_setup_prp_pools(dev);
	if (result)
3227
		goto release;
M
Matthew Wilcox 已提交
3228

3229
	kref_init(&dev->kref);
3230
	dev->device = device_create(nvme_class, &pdev->dev,
3231 3232
				MKDEV(nvme_char_major, dev->ctrl.instance),
				dev, "nvme%d", dev->ctrl.instance);
3233 3234
	if (IS_ERR(dev->device)) {
		result = PTR_ERR(dev->device);
3235
		goto release_pools;
3236 3237
	}
	get_device(dev->device);
3238 3239 3240 3241 3242
	dev_set_drvdata(dev->device, dev);

	result = device_create_file(dev->device, &dev_attr_reset_controller);
	if (result)
		goto put_dev;
3243

3244
	INIT_LIST_HEAD(&dev->node);
K
Keith Busch 已提交
3245
	INIT_WORK(&dev->scan_work, nvme_dev_scan);
3246
	INIT_WORK(&dev->probe_work, nvme_probe_work);
3247
	schedule_work(&dev->probe_work);
M
Matthew Wilcox 已提交
3248 3249
	return 0;

3250
 put_dev:
3251
	device_destroy(nvme_class, MKDEV(nvme_char_major, dev->ctrl.instance));
3252
	put_device(dev->device);
3253
 release_pools:
M
Matthew Wilcox 已提交
3254
	nvme_release_prp_pools(dev);
3255 3256
 release:
	nvme_release_instance(dev);
K
Keith Busch 已提交
3257
 put_pci:
3258
	put_device(dev->dev);
M
Matthew Wilcox 已提交
3259 3260 3261 3262 3263 3264 3265
 free:
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
	return result;
}

3266 3267
static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
{
K
Keith Busch 已提交
3268
	struct nvme_dev *dev = pci_get_drvdata(pdev);
3269

K
Keith Busch 已提交
3270 3271 3272
	if (prepare)
		nvme_dev_shutdown(dev);
	else
3273
		schedule_work(&dev->probe_work);
3274 3275
}

3276 3277 3278 3279 3280 3281
static void nvme_shutdown(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
	nvme_dev_shutdown(dev);
}

3282
static void nvme_remove(struct pci_dev *pdev)
M
Matthew Wilcox 已提交
3283 3284
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
K
Keith Busch 已提交
3285 3286 3287 3288 3289 3290

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	spin_unlock(&dev_list_lock);

	pci_set_drvdata(pdev, NULL);
3291
	flush_work(&dev->probe_work);
K
Keith Busch 已提交
3292
	flush_work(&dev->reset_work);
K
Keith Busch 已提交
3293
	flush_work(&dev->scan_work);
3294
	device_remove_file(dev->device, &dev_attr_reset_controller);
3295
	nvme_dev_remove(dev);
3296
	nvme_dev_shutdown(dev);
M
Matias Bjørling 已提交
3297
	nvme_dev_remove_admin(dev);
3298
	device_destroy(nvme_class, MKDEV(nvme_char_major, dev->ctrl.instance));
3299
	nvme_free_queues(dev, 0);
3300
	nvme_release_cmb(dev);
K
Keith Busch 已提交
3301
	nvme_release_prp_pools(dev);
3302
	kref_put(&dev->kref, nvme_free_dev);
M
Matthew Wilcox 已提交
3303 3304 3305 3306 3307 3308 3309 3310
}

/* These functions are yet to be implemented */
#define nvme_error_detected NULL
#define nvme_dump_registers NULL
#define nvme_link_reset NULL
#define nvme_slot_reset NULL
#define nvme_error_resume NULL
3311

3312
#ifdef CONFIG_PM_SLEEP
3313 3314 3315 3316 3317 3318 3319 3320 3321 3322 3323 3324 3325 3326
static int nvme_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

	nvme_dev_shutdown(ndev);
	return 0;
}

static int nvme_resume(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

3327
	schedule_work(&ndev->probe_work);
K
Keith Busch 已提交
3328
	return 0;
3329
}
3330
#endif
3331 3332

static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
M
Matthew Wilcox 已提交
3333

3334
static const struct pci_error_handlers nvme_err_handler = {
M
Matthew Wilcox 已提交
3335 3336 3337 3338 3339
	.error_detected	= nvme_error_detected,
	.mmio_enabled	= nvme_dump_registers,
	.link_reset	= nvme_link_reset,
	.slot_reset	= nvme_slot_reset,
	.resume		= nvme_error_resume,
3340
	.reset_notify	= nvme_reset_notify,
M
Matthew Wilcox 已提交
3341 3342 3343 3344 3345
};

/* Move to pci_ids.h later */
#define PCI_CLASS_STORAGE_EXPRESS	0x010802

3346
static const struct pci_device_id nvme_id_table[] = {
M
Matthew Wilcox 已提交
3347
	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
3348
	{ PCI_DEVICE(PCI_VENDOR_ID_APPLE, 0x2001) },
M
Matthew Wilcox 已提交
3349 3350 3351 3352 3353 3354 3355 3356
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, nvme_id_table);

static struct pci_driver nvme_driver = {
	.name		= "nvme",
	.id_table	= nvme_id_table,
	.probe		= nvme_probe,
3357
	.remove		= nvme_remove,
3358
	.shutdown	= nvme_shutdown,
3359 3360 3361
	.driver		= {
		.pm	= &nvme_dev_pm_ops,
	},
M
Matthew Wilcox 已提交
3362 3363 3364 3365 3366
	.err_handler	= &nvme_err_handler,
};

static int __init nvme_init(void)
{
3367
	int result;
3368

3369
	init_waitqueue_head(&nvme_kthread_wait);
M
Matthew Wilcox 已提交
3370

K
Keith Busch 已提交
3371 3372
	nvme_workq = create_singlethread_workqueue("nvme");
	if (!nvme_workq)
3373
		return -ENOMEM;
K
Keith Busch 已提交
3374

3375 3376
	result = register_blkdev(nvme_major, "nvme");
	if (result < 0)
K
Keith Busch 已提交
3377
		goto kill_workq;
3378
	else if (result > 0)
3379
		nvme_major = result;
M
Matthew Wilcox 已提交
3380

3381 3382 3383 3384 3385 3386 3387 3388
	result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
							&nvme_dev_fops);
	if (result < 0)
		goto unregister_blkdev;
	else if (result > 0)
		nvme_char_major = result;

	nvme_class = class_create(THIS_MODULE, "nvme");
3389 3390
	if (IS_ERR(nvme_class)) {
		result = PTR_ERR(nvme_class);
3391
		goto unregister_chrdev;
3392
	}
3393

3394 3395
	result = pci_register_driver(&nvme_driver);
	if (result)
3396
		goto destroy_class;
3397
	return 0;
M
Matthew Wilcox 已提交
3398

3399 3400 3401 3402
 destroy_class:
	class_destroy(nvme_class);
 unregister_chrdev:
	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
3403
 unregister_blkdev:
M
Matthew Wilcox 已提交
3404
	unregister_blkdev(nvme_major, "nvme");
K
Keith Busch 已提交
3405 3406
 kill_workq:
	destroy_workqueue(nvme_workq);
M
Matthew Wilcox 已提交
3407 3408 3409 3410 3411 3412 3413
	return result;
}

static void __exit nvme_exit(void)
{
	pci_unregister_driver(&nvme_driver);
	unregister_blkdev(nvme_major, "nvme");
K
Keith Busch 已提交
3414
	destroy_workqueue(nvme_workq);
3415 3416
	class_destroy(nvme_class);
	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
3417
	BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
3418
	_nvme_check_size();
M
Matthew Wilcox 已提交
3419 3420 3421 3422
}

MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
MODULE_LICENSE("GPL");
3423
MODULE_VERSION("1.0");
M
Matthew Wilcox 已提交
3424 3425
module_init(nvme_init);
module_exit(nvme_exit);