nvme-core.c 78.9 KB
Newer Older
M
Matthew Wilcox 已提交
1 2
/*
 * NVM Express device driver
3
 * Copyright (c) 2011-2014, Intel Corporation.
M
Matthew Wilcox 已提交
4 5 6 7 8 9 10 11 12 13 14 15
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/nvme.h>
16
#include <linux/bitops.h>
M
Matthew Wilcox 已提交
17
#include <linux/blkdev.h>
18
#include <linux/blk-mq.h>
K
Keith Busch 已提交
19
#include <linux/cpu.h>
20
#include <linux/delay.h>
M
Matthew Wilcox 已提交
21 22 23
#include <linux/errno.h>
#include <linux/fs.h>
#include <linux/genhd.h>
24
#include <linux/hdreg.h>
25
#include <linux/idr.h>
M
Matthew Wilcox 已提交
26 27 28 29
#include <linux/init.h>
#include <linux/interrupt.h>
#include <linux/io.h>
#include <linux/kdev_t.h>
30
#include <linux/kthread.h>
M
Matthew Wilcox 已提交
31 32 33 34 35
#include <linux/kernel.h>
#include <linux/mm.h>
#include <linux/module.h>
#include <linux/moduleparam.h>
#include <linux/pci.h>
36
#include <linux/poison.h>
37
#include <linux/ptrace.h>
M
Matthew Wilcox 已提交
38 39
#include <linux/sched.h>
#include <linux/slab.h>
40
#include <linux/t10-pi.h>
M
Matthew Wilcox 已提交
41
#include <linux/types.h>
V
Vishal Verma 已提交
42
#include <scsi/sg.h>
43 44
#include <asm-generic/io-64-nonatomic-lo-hi.h>

45
#define NVME_MINORS		(1U << MINORBITS)
46
#define NVME_Q_DEPTH		1024
47
#define NVME_AQ_DEPTH		64
M
Matthew Wilcox 已提交
48 49
#define SQ_SIZE(depth)		(depth * sizeof(struct nvme_command))
#define CQ_SIZE(depth)		(depth * sizeof(struct nvme_completion))
50
#define ADMIN_TIMEOUT		(admin_timeout * HZ)
51
#define SHUTDOWN_TIMEOUT	(shutdown_timeout * HZ)
52 53 54 55

static unsigned char admin_timeout = 60;
module_param(admin_timeout, byte, 0644);
MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
M
Matthew Wilcox 已提交
56

57 58
unsigned char nvme_io_timeout = 30;
module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
59
MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
M
Matthew Wilcox 已提交
60

61 62 63 64
static unsigned char shutdown_timeout = 5;
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");

M
Matthew Wilcox 已提交
65 66 67
static int nvme_major;
module_param(nvme_major, int, 0);

68 69 70
static int nvme_char_major;
module_param(nvme_char_major, int, 0);

71 72 73
static int use_threaded_interrupts;
module_param(use_threaded_interrupts, int, 0);

74 75 76
static DEFINE_SPINLOCK(dev_list_lock);
static LIST_HEAD(dev_list);
static struct task_struct *nvme_thread;
77
static struct workqueue_struct *nvme_workq;
78
static wait_queue_head_t nvme_kthread_wait;
79

80 81
static struct class *nvme_class;

82
static void nvme_reset_failed_dev(struct work_struct *ws);
83
static int nvme_process_cq(struct nvme_queue *nvmeq);
84

85 86 87
struct async_cmd_info {
	struct kthread_work work;
	struct kthread_worker *worker;
88
	struct request *req;
89 90 91 92
	u32 result;
	int status;
	void *ctx;
};
93

M
Matthew Wilcox 已提交
94 95 96 97 98 99
/*
 * An NVM Express queue.  Each device has at least two (one for admin
 * commands and one for I/O commands).
 */
struct nvme_queue {
	struct device *q_dmadev;
100
	struct nvme_dev *dev;
101
	char irqname[24];	/* nvme4294967295-65535\0 */
M
Matthew Wilcox 已提交
102 103 104 105 106 107 108
	spinlock_t q_lock;
	struct nvme_command *sq_cmds;
	volatile struct nvme_completion *cqes;
	dma_addr_t sq_dma_addr;
	dma_addr_t cq_dma_addr;
	u32 __iomem *q_db;
	u16 q_depth;
109
	s16 cq_vector;
M
Matthew Wilcox 已提交
110 111 112
	u16 sq_head;
	u16 sq_tail;
	u16 cq_head;
113
	u16 qid;
114 115
	u8 cq_phase;
	u8 cqe_seen;
116
	struct async_cmd_info cmdinfo;
117
	struct blk_mq_hw_ctx *hctx;
M
Matthew Wilcox 已提交
118 119 120 121 122 123 124 125 126 127 128 129
};

/*
 * Check we didin't inadvertently grow the command struct
 */
static inline void _nvme_check_size(void)
{
	BUILD_BUG_ON(sizeof(struct nvme_rw_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_cq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_create_sq) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_delete_queue) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_features) != 64);
130
	BUILD_BUG_ON(sizeof(struct nvme_format_cmd) != 64);
131
	BUILD_BUG_ON(sizeof(struct nvme_abort_cmd) != 64);
M
Matthew Wilcox 已提交
132 133 134 135
	BUILD_BUG_ON(sizeof(struct nvme_command) != 64);
	BUILD_BUG_ON(sizeof(struct nvme_id_ctrl) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_id_ns) != 4096);
	BUILD_BUG_ON(sizeof(struct nvme_lba_range_type) != 64);
K
Keith Busch 已提交
136
	BUILD_BUG_ON(sizeof(struct nvme_smart_log) != 512);
M
Matthew Wilcox 已提交
137 138
}

139
typedef void (*nvme_completion_fn)(struct nvme_queue *, void *,
140 141
						struct nvme_completion *);

142
struct nvme_cmd_info {
143 144
	nvme_completion_fn fn;
	void *ctx;
145
	int aborted;
146
	struct nvme_queue *nvmeq;
147
	struct nvme_iod iod[0];
148 149
};

150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177
/*
 * Max size of iod being embedded in the request payload
 */
#define NVME_INT_PAGES		2
#define NVME_INT_BYTES(dev)	(NVME_INT_PAGES * (dev)->page_size)

/*
 * Will slightly overestimate the number of pages needed.  This is OK
 * as it only leads to a small amount of wasted memory for the lifetime of
 * the I/O.
 */
static int nvme_npages(unsigned size, struct nvme_dev *dev)
{
	unsigned nprps = DIV_ROUND_UP(size + dev->page_size, dev->page_size);
	return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8);
}

static unsigned int nvme_cmd_size(struct nvme_dev *dev)
{
	unsigned int ret = sizeof(struct nvme_cmd_info);

	ret += sizeof(struct nvme_iod);
	ret += sizeof(__le64 *) * nvme_npages(NVME_INT_BYTES(dev), dev);
	ret += sizeof(struct scatterlist) * NVME_INT_PAGES;

	return ret;
}

178 179
static int nvme_admin_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
				unsigned int hctx_idx)
180
{
181 182 183 184 185 186 187
	struct nvme_dev *dev = data;
	struct nvme_queue *nvmeq = dev->queues[0];

	WARN_ON(nvmeq->hctx);
	nvmeq->hctx = hctx;
	hctx->driver_data = nvmeq;
	return 0;
188 189
}

190 191 192
static int nvme_admin_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
193
{
194 195 196 197 198 199 200
	struct nvme_dev *dev = data;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = dev->queues[0];

	BUG_ON(!nvmeq);
	cmd->nvmeq = nvmeq;
	return 0;
201 202
}

J
Jens Axboe 已提交
203 204 205 206 207 208 209
static void nvme_exit_hctx(struct blk_mq_hw_ctx *hctx, unsigned int hctx_idx)
{
	struct nvme_queue *nvmeq = hctx->driver_data;

	nvmeq->hctx = NULL;
}

210 211
static int nvme_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
			  unsigned int hctx_idx)
M
Matthew Wilcox 已提交
212
{
213 214 215
	struct nvme_dev *dev = data;
	struct nvme_queue *nvmeq = dev->queues[
					(hctx_idx % dev->queue_count) + 1];
M
Matthew Wilcox 已提交
216

217 218 219 220 221 222
	if (!nvmeq->hctx)
		nvmeq->hctx = hctx;

	/* nvmeq queues are shared between namespaces. We assume here that
	 * blk-mq map the tags so they match up with the nvme queue tags. */
	WARN_ON(nvmeq->hctx->tags != hctx->tags);
M
Matthew Wilcox 已提交
223

224 225
	hctx->driver_data = nvmeq;
	return 0;
M
Matthew Wilcox 已提交
226 227
}

228 229 230
static int nvme_init_request(void *data, struct request *req,
				unsigned int hctx_idx, unsigned int rq_idx,
				unsigned int numa_node)
M
Matthew Wilcox 已提交
231
{
232 233 234 235 236 237 238 239 240 241 242 243 244 245 246
	struct nvme_dev *dev = data;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = dev->queues[hctx_idx + 1];

	BUG_ON(!nvmeq);
	cmd->nvmeq = nvmeq;
	return 0;
}

static void nvme_set_info(struct nvme_cmd_info *cmd, void *ctx,
				nvme_completion_fn handler)
{
	cmd->fn = handler;
	cmd->ctx = ctx;
	cmd->aborted = 0;
K
Keith Busch 已提交
247
	blk_mq_start_request(blk_mq_rq_from_pdu(cmd));
M
Matthew Wilcox 已提交
248 249
}

250 251 252 253 254 255 256 257 258 259 260 261 262
static void *iod_get_private(struct nvme_iod *iod)
{
	return (void *) (iod->private & ~0x1UL);
}

/*
 * If bit 0 is set, the iod is embedded in the request payload.
 */
static bool iod_should_kfree(struct nvme_iod *iod)
{
	return (iod->private & 0x01) == 0;
}

263 264
/* Special values must be less than 0x1000 */
#define CMD_CTX_BASE		((void *)POISON_POINTER_DELTA)
265 266 267
#define CMD_CTX_CANCELLED	(0x30C + CMD_CTX_BASE)
#define CMD_CTX_COMPLETED	(0x310 + CMD_CTX_BASE)
#define CMD_CTX_INVALID		(0x314 + CMD_CTX_BASE)
268

269
static void special_completion(struct nvme_queue *nvmeq, void *ctx,
270 271 272 273 274
						struct nvme_completion *cqe)
{
	if (ctx == CMD_CTX_CANCELLED)
		return;
	if (ctx == CMD_CTX_COMPLETED) {
275
		dev_warn(nvmeq->q_dmadev,
276 277 278 279 280
				"completed id %d twice on queue %d\n",
				cqe->command_id, le16_to_cpup(&cqe->sq_id));
		return;
	}
	if (ctx == CMD_CTX_INVALID) {
281
		dev_warn(nvmeq->q_dmadev,
282 283 284 285
				"invalid id %d completed on queue %d\n",
				cqe->command_id, le16_to_cpup(&cqe->sq_id));
		return;
	}
286
	dev_warn(nvmeq->q_dmadev, "Unknown special completion %p\n", ctx);
287 288
}

289
static void *cancel_cmd_info(struct nvme_cmd_info *cmd, nvme_completion_fn *fn)
M
Matthew Wilcox 已提交
290
{
291
	void *ctx;
M
Matthew Wilcox 已提交
292

293
	if (fn)
294 295 296 297
		*fn = cmd->fn;
	ctx = cmd->ctx;
	cmd->fn = special_completion;
	cmd->ctx = CMD_CTX_CANCELLED;
298
	return ctx;
M
Matthew Wilcox 已提交
299 300
}

301 302
static void async_req_completion(struct nvme_queue *nvmeq, void *ctx,
						struct nvme_completion *cqe)
303
{
304
	struct request *req = ctx;
305

306 307 308 309 310 311 312 313 314
	u32 result = le32_to_cpup(&cqe->result);
	u16 status = le16_to_cpup(&cqe->status) >> 1;

	if (status == NVME_SC_SUCCESS || status == NVME_SC_ABORT_REQ)
		++nvmeq->dev->event_limit;
	if (status == NVME_SC_SUCCESS)
		dev_warn(nvmeq->q_dmadev,
			"async event result %08x\n", result);

315
	blk_mq_free_hctx_request(nvmeq->hctx, req);
M
Matthew Wilcox 已提交
316 317
}

318 319
static void abort_completion(struct nvme_queue *nvmeq, void *ctx,
						struct nvme_completion *cqe)
320
{
321 322 323 324
	struct request *req = ctx;

	u16 status = le16_to_cpup(&cqe->status) >> 1;
	u32 result = le32_to_cpup(&cqe->result);
325

326
	blk_mq_free_hctx_request(nvmeq->hctx, req);
327

328 329
	dev_warn(nvmeq->q_dmadev, "Abort status:%x result:%x", status, result);
	++nvmeq->dev->abort_limit;
330 331
}

332 333
static void async_completion(struct nvme_queue *nvmeq, void *ctx,
						struct nvme_completion *cqe)
M
Matthew Wilcox 已提交
334
{
335 336 337 338
	struct async_cmd_info *cmdinfo = ctx;
	cmdinfo->result = le32_to_cpup(&cqe->result);
	cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
	queue_kthread_work(cmdinfo->worker, &cmdinfo->work);
339
	blk_mq_free_hctx_request(nvmeq->hctx, cmdinfo->req);
M
Matthew Wilcox 已提交
340 341
}

342 343
static inline struct nvme_cmd_info *get_cmd_from_tag(struct nvme_queue *nvmeq,
				  unsigned int tag)
M
Matthew Wilcox 已提交
344
{
345 346
	struct blk_mq_hw_ctx *hctx = nvmeq->hctx;
	struct request *req = blk_mq_tag_to_rq(hctx->tags, tag);
347

348
	return blk_mq_rq_to_pdu(req);
349 350
}

351 352 353 354 355
/*
 * Called with local interrupts disabled and the q_lock held.  May not sleep.
 */
static void *nvme_finish_cmd(struct nvme_queue *nvmeq, int tag,
						nvme_completion_fn *fn)
356
{
357 358 359 360 361 362 363 364 365 366 367 368
	struct nvme_cmd_info *cmd = get_cmd_from_tag(nvmeq, tag);
	void *ctx;
	if (tag >= nvmeq->q_depth) {
		*fn = special_completion;
		return CMD_CTX_INVALID;
	}
	if (fn)
		*fn = cmd->fn;
	ctx = cmd->ctx;
	cmd->fn = special_completion;
	cmd->ctx = CMD_CTX_COMPLETED;
	return ctx;
M
Matthew Wilcox 已提交
369 370 371
}

/**
372
 * nvme_submit_cmd() - Copy a command into a queue and ring the doorbell
M
Matthew Wilcox 已提交
373 374 375 376 377
 * @nvmeq: The queue to use
 * @cmd: The command to send
 *
 * Safe to use from interrupt context
 */
378
static int __nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
M
Matthew Wilcox 已提交
379
{
380 381
	u16 tail = nvmeq->sq_tail;

M
Matthew Wilcox 已提交
382 383 384
	memcpy(&nvmeq->sq_cmds[tail], cmd, sizeof(*cmd));
	if (++tail == nvmeq->q_depth)
		tail = 0;
385
	writel(tail, nvmeq->q_db);
M
Matthew Wilcox 已提交
386 387 388 389 390
	nvmeq->sq_tail = tail;

	return 0;
}

391 392 393 394 395 396 397 398 399 400
static int nvme_submit_cmd(struct nvme_queue *nvmeq, struct nvme_command *cmd)
{
	unsigned long flags;
	int ret;
	spin_lock_irqsave(&nvmeq->q_lock, flags);
	ret = __nvme_submit_cmd(nvmeq, cmd);
	spin_unlock_irqrestore(&nvmeq->q_lock, flags);
	return ret;
}

401
static __le64 **iod_list(struct nvme_iod *iod)
402
{
403
	return ((void *)iod) + iod->offset;
404 405
}

406 407
static inline void iod_init(struct nvme_iod *iod, unsigned nbytes,
			    unsigned nseg, unsigned long private)
408
{
409 410 411 412 413
	iod->private = private;
	iod->offset = offsetof(struct nvme_iod, sg[nseg]);
	iod->npages = -1;
	iod->length = nbytes;
	iod->nents = 0;
414
}
M
Matthew Wilcox 已提交
415

416
static struct nvme_iod *
417 418
__nvme_alloc_iod(unsigned nseg, unsigned bytes, struct nvme_dev *dev,
		 unsigned long priv, gfp_t gfp)
M
Matthew Wilcox 已提交
419
{
420
	struct nvme_iod *iod = kmalloc(sizeof(struct nvme_iod) +
421
				sizeof(__le64 *) * nvme_npages(bytes, dev) +
422 423
				sizeof(struct scatterlist) * nseg, gfp);

424 425
	if (iod)
		iod_init(iod, bytes, nseg, priv);
426 427

	return iod;
M
Matthew Wilcox 已提交
428 429
}

430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452
static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev,
			               gfp_t gfp)
{
	unsigned size = !(rq->cmd_flags & REQ_DISCARD) ? blk_rq_bytes(rq) :
                                                sizeof(struct nvme_dsm_range);
	unsigned long mask = 0;
	struct nvme_iod *iod;

	if (rq->nr_phys_segments <= NVME_INT_PAGES &&
	    size <= NVME_INT_BYTES(dev)) {
		struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(rq);

		iod = cmd->iod;
		mask = 0x01;
		iod_init(iod, size, rq->nr_phys_segments,
				(unsigned long) rq | 0x01);
		return iod;
	}

	return __nvme_alloc_iod(rq->nr_phys_segments, size, dev,
				(unsigned long) rq, gfp);
}

V
Vishal Verma 已提交
453
void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod)
M
Matthew Wilcox 已提交
454
{
455
	const int last_prp = dev->page_size / 8 - 1;
456 457 458 459 460 461 462 463 464 465 466 467
	int i;
	__le64 **list = iod_list(iod);
	dma_addr_t prp_dma = iod->first_dma;

	if (iod->npages == 0)
		dma_pool_free(dev->prp_small_pool, list[0], prp_dma);
	for (i = 0; i < iod->npages; i++) {
		__le64 *prp_list = list[i];
		dma_addr_t next_prp_dma = le64_to_cpu(prp_list[last_prp]);
		dma_pool_free(dev->prp_page_pool, prp_list, prp_dma);
		prp_dma = next_prp_dma;
	}
468 469 470

	if (iod_should_kfree(iod))
		kfree(iod);
M
Matthew Wilcox 已提交
471 472
}

473 474 475 476 477 478 479 480 481 482 483 484
static int nvme_error_status(u16 status)
{
	switch (status & 0x7ff) {
	case NVME_SC_SUCCESS:
		return 0;
	case NVME_SC_CAP_EXCEEDED:
		return -ENOSPC;
	default:
		return -EIO;
	}
}

485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540
static void nvme_dif_prep(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == v)
		pi->ref_tag = cpu_to_be32(p);
}

static void nvme_dif_complete(u32 p, u32 v, struct t10_pi_tuple *pi)
{
	if (be32_to_cpu(pi->ref_tag) == p)
		pi->ref_tag = cpu_to_be32(v);
}

/**
 * nvme_dif_remap - remaps ref tags to bip seed and physical lba
 *
 * The virtual start sector is the one that was originally submitted by the
 * block layer.	Due to partitioning, MD/DM cloning, etc. the actual physical
 * start sector may be different. Remap protection information to match the
 * physical LBA on writes, and back to the original seed on reads.
 *
 * Type 0 and 3 do not have a ref tag, so no remapping required.
 */
static void nvme_dif_remap(struct request *req,
			void (*dif_swap)(u32 p, u32 v, struct t10_pi_tuple *pi))
{
	struct nvme_ns *ns = req->rq_disk->private_data;
	struct bio_integrity_payload *bip;
	struct t10_pi_tuple *pi;
	void *p, *pmap;
	u32 i, nlb, ts, phys, virt;

	if (!ns->pi_type || ns->pi_type == NVME_NS_DPS_PI_TYPE3)
		return;

	bip = bio_integrity(req->bio);
	if (!bip)
		return;

	pmap = kmap_atomic(bip->bip_vec->bv_page) + bip->bip_vec->bv_offset;
	if (!pmap)
		return;

	p = pmap;
	virt = bip_get_seed(bip);
	phys = nvme_block_nr(ns, blk_rq_pos(req));
	nlb = (blk_rq_bytes(req) >> ns->lba_shift);
	ts = ns->disk->integrity->tuple_size;

	for (i = 0; i < nlb; i++, virt++, phys++) {
		pi = (struct t10_pi_tuple *)p;
		dif_swap(phys, virt, pi);
		p += ts;
	}
	kunmap_atomic(pmap);
}

541
static void req_completion(struct nvme_queue *nvmeq, void *ctx,
M
Matthew Wilcox 已提交
542 543
						struct nvme_completion *cqe)
{
544
	struct nvme_iod *iod = ctx;
545
	struct request *req = iod_get_private(iod);
546 547
	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);

M
Matthew Wilcox 已提交
548 549
	u16 status = le16_to_cpup(&cqe->status) >> 1;

550
	if (unlikely(status)) {
551 552
		if (!(status & NVME_SC_DNR || blk_noretry_request(req))
		    && (jiffies - req->start_time) < req->timeout) {
553 554
			unsigned long flags;

555
			blk_mq_requeue_request(req);
556 557 558 559
			spin_lock_irqsave(req->q->queue_lock, flags);
			if (!blk_queue_stopped(req->q))
				blk_mq_kick_requeue_list(req->q);
			spin_unlock_irqrestore(req->q->queue_lock, flags);
560 561
			return;
		}
562 563 564 565 566 567 568 569 570
		req->errors = nvme_error_status(status);
	} else
		req->errors = 0;

	if (cmd_rq->aborted)
		dev_warn(&nvmeq->dev->pci_dev->dev,
			"completing aborted command with status:%04x\n",
			status);

571
	if (iod->nents) {
572 573
		dma_unmap_sg(&nvmeq->dev->pci_dev->dev, iod->sg, iod->nents,
			rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
574 575 576 577 578 579 580
		if (blk_integrity_rq(req)) {
			if (!rq_data_dir(req))
				nvme_dif_remap(req, nvme_dif_complete);
			dma_unmap_sg(&nvmeq->dev->pci_dev->dev, iod->meta_sg, 1,
				rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
		}
	}
581
	nvme_free_iod(nvmeq->dev, iod);
K
Keith Busch 已提交
582

583
	blk_mq_complete_request(req);
M
Matthew Wilcox 已提交
584 585
}

586
/* length is in bytes.  gfp flags indicates whether we may sleep. */
587 588
int nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod, int total_len,
								gfp_t gfp)
589
{
590
	struct dma_pool *pool;
591 592
	int length = total_len;
	struct scatterlist *sg = iod->sg;
593 594 595
	int dma_len = sg_dma_len(sg);
	u64 dma_addr = sg_dma_address(sg);
	int offset = offset_in_page(dma_addr);
596
	__le64 *prp_list;
597
	__le64 **list = iod_list(iod);
598
	dma_addr_t prp_dma;
599
	int nprps, i;
600
	u32 page_size = dev->page_size;
601

602
	length -= (page_size - offset);
603
	if (length <= 0)
604
		return total_len;
605

606
	dma_len -= (page_size - offset);
607
	if (dma_len) {
608
		dma_addr += (page_size - offset);
609 610 611 612 613 614
	} else {
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
	}

615
	if (length <= page_size) {
616
		iod->first_dma = dma_addr;
617
		return total_len;
618 619
	}

620
	nprps = DIV_ROUND_UP(length, page_size);
621 622
	if (nprps <= (256 / 8)) {
		pool = dev->prp_small_pool;
623
		iod->npages = 0;
624 625
	} else {
		pool = dev->prp_page_pool;
626
		iod->npages = 1;
627 628
	}

629 630
	prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
	if (!prp_list) {
631
		iod->first_dma = dma_addr;
632
		iod->npages = -1;
633
		return (total_len - length) + page_size;
634
	}
635 636
	list[0] = prp_list;
	iod->first_dma = prp_dma;
637 638
	i = 0;
	for (;;) {
639
		if (i == page_size >> 3) {
640
			__le64 *old_prp_list = prp_list;
641
			prp_list = dma_pool_alloc(pool, gfp, &prp_dma);
642 643 644
			if (!prp_list)
				return total_len - length;
			list[iod->npages++] = prp_list;
645 646 647
			prp_list[0] = old_prp_list[i - 1];
			old_prp_list[i - 1] = cpu_to_le64(prp_dma);
			i = 1;
648 649
		}
		prp_list[i++] = cpu_to_le64(dma_addr);
650 651 652
		dma_len -= page_size;
		dma_addr += page_size;
		length -= page_size;
653 654 655 656 657 658 659 660
		if (length <= 0)
			break;
		if (dma_len > 0)
			continue;
		BUG_ON(dma_len < 0);
		sg = sg_next(sg);
		dma_addr = sg_dma_address(sg);
		dma_len = sg_dma_len(sg);
661 662
	}

663
	return total_len;
664 665
}

666 667 668 669 670 671 672
/*
 * We reuse the small pool to allocate the 16-byte range here as it is not
 * worth having a special pool for these or additional cases to handle freeing
 * the iod.
 */
static void nvme_submit_discard(struct nvme_queue *nvmeq, struct nvme_ns *ns,
		struct request *req, struct nvme_iod *iod)
673
{
674 675
	struct nvme_dsm_range *range =
				(struct nvme_dsm_range *)iod_list(iod)[0];
676 677 678
	struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];

	range->cattr = cpu_to_le32(0);
679 680
	range->nlb = cpu_to_le32(blk_rq_bytes(req) >> ns->lba_shift);
	range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
681 682 683

	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->dsm.opcode = nvme_cmd_dsm;
684
	cmnd->dsm.command_id = req->tag;
685 686 687 688 689 690 691 692 693 694
	cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
	cmnd->dsm.prp1 = cpu_to_le64(iod->first_dma);
	cmnd->dsm.nr = 0;
	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);

	if (++nvmeq->sq_tail == nvmeq->q_depth)
		nvmeq->sq_tail = 0;
	writel(nvmeq->sq_tail, nvmeq->q_db);
}

695
static void nvme_submit_flush(struct nvme_queue *nvmeq, struct nvme_ns *ns,
M
Matthew Wilcox 已提交
696 697 698 699 700 701 702 703 704 705 706 707 708 709
								int cmdid)
{
	struct nvme_command *cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];

	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->common.opcode = nvme_cmd_flush;
	cmnd->common.command_id = cmdid;
	cmnd->common.nsid = cpu_to_le32(ns->ns_id);

	if (++nvmeq->sq_tail == nvmeq->q_depth)
		nvmeq->sq_tail = 0;
	writel(nvmeq->sq_tail, nvmeq->q_db);
}

710 711
static int nvme_submit_iod(struct nvme_queue *nvmeq, struct nvme_iod *iod,
							struct nvme_ns *ns)
M
Matthew Wilcox 已提交
712
{
713
	struct request *req = iod_get_private(iod);
714
	struct nvme_command *cmnd;
715 716
	u16 control = 0;
	u32 dsmgmt = 0;
M
Matthew Wilcox 已提交
717

718
	if (req->cmd_flags & REQ_FUA)
M
Matthew Wilcox 已提交
719
		control |= NVME_RW_FUA;
720
	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
M
Matthew Wilcox 已提交
721 722
		control |= NVME_RW_LR;

723
	if (req->cmd_flags & REQ_RAHEAD)
M
Matthew Wilcox 已提交
724 725
		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;

726
	cmnd = &nvmeq->sq_cmds[nvmeq->sq_tail];
727
	memset(cmnd, 0, sizeof(*cmnd));
M
Matthew Wilcox 已提交
728

729 730
	cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
	cmnd->rw.command_id = req->tag;
731
	cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
732 733
	cmnd->rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
	cmnd->rw.prp2 = cpu_to_le64(iod->first_dma);
734 735
	cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);
736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753

	if (blk_integrity_rq(req)) {
		cmnd->rw.metadata = cpu_to_le64(sg_dma_address(iod->meta_sg));
		switch (ns->pi_type) {
		case NVME_NS_DPS_PI_TYPE3:
			control |= NVME_RW_PRINFO_PRCHK_GUARD;
			break;
		case NVME_NS_DPS_PI_TYPE1:
		case NVME_NS_DPS_PI_TYPE2:
			control |= NVME_RW_PRINFO_PRCHK_GUARD |
					NVME_RW_PRINFO_PRCHK_REF;
			cmnd->rw.reftag = cpu_to_le32(
					nvme_block_nr(ns, blk_rq_pos(req)));
			break;
		}
	} else if (ns->ms)
		control |= NVME_RW_PRINFO_PRACT;

754 755
	cmnd->rw.control = cpu_to_le16(control);
	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
M
Matthew Wilcox 已提交
756 757 758

	if (++nvmeq->sq_tail == nvmeq->q_depth)
		nvmeq->sq_tail = 0;
759
	writel(nvmeq->sq_tail, nvmeq->q_db);
M
Matthew Wilcox 已提交
760

761
	return 0;
762 763
}

764 765
static int nvme_queue_rq(struct blk_mq_hw_ctx *hctx,
			 const struct blk_mq_queue_data *bd)
766
{
767 768 769 770
	struct nvme_ns *ns = hctx->queue->queuedata;
	struct nvme_queue *nvmeq = hctx->driver_data;
	struct request *req = bd->rq;
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
771
	struct nvme_iod *iod;
772
	enum dma_data_direction dma_dir;
773

774 775 776 777 778 779 780 781 782 783 784 785 786
	/*
	 * If formated with metadata, require the block layer provide a buffer
	 * unless this namespace is formated such that the metadata can be
	 * stripped/generated by the controller with PRACT=1.
	 */
	if (ns->ms && !blk_integrity_rq(req)) {
		if (!(ns->pi_type && ns->ms == 8)) {
			req->errors = -EFAULT;
			blk_mq_complete_request(req);
			return BLK_MQ_RQ_QUEUE_OK;
		}
	}

787
	iod = nvme_alloc_iod(req, ns->dev, GFP_ATOMIC);
788
	if (!iod)
789
		return BLK_MQ_RQ_QUEUE_BUSY;
790 791

	if (req->cmd_flags & REQ_DISCARD) {
792 793 794 795 796 797 798 799 800
		void *range;
		/*
		 * We reuse the small pool to allocate the 16-byte range here
		 * as it is not worth having a special pool for these or
		 * additional cases to handle freeing the iod.
		 */
		range = dma_pool_alloc(nvmeq->dev->prp_small_pool,
						GFP_ATOMIC,
						&iod->first_dma);
801
		if (!range)
802
			goto retry_cmd;
803 804
		iod_list(iod)[0] = (__le64 *)range;
		iod->npages = 0;
805
	} else if (req->nr_phys_segments) {
806 807
		dma_dir = rq_data_dir(req) ? DMA_TO_DEVICE : DMA_FROM_DEVICE;

808
		sg_init_table(iod->sg, req->nr_phys_segments);
809
		iod->nents = blk_rq_map_sg(req->q, req, iod->sg);
810 811
		if (!iod->nents)
			goto error_cmd;
812 813

		if (!dma_map_sg(nvmeq->q_dmadev, iod->sg, iod->nents, dma_dir))
814
			goto retry_cmd;
815

816 817 818 819 820 821
		if (blk_rq_bytes(req) !=
                    nvme_setup_prps(nvmeq->dev, iod, blk_rq_bytes(req), GFP_ATOMIC)) {
			dma_unmap_sg(&nvmeq->dev->pci_dev->dev, iod->sg,
					iod->nents, dma_dir);
			goto retry_cmd;
		}
822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
		if (blk_integrity_rq(req)) {
			if (blk_rq_count_integrity_sg(req->q, req->bio) != 1)
				goto error_cmd;

			sg_init_table(iod->meta_sg, 1);
			if (blk_rq_map_integrity_sg(
					req->q, req->bio, iod->meta_sg) != 1)
				goto error_cmd;

			if (rq_data_dir(req))
				nvme_dif_remap(req, nvme_dif_prep);

			if (!dma_map_sg(nvmeq->q_dmadev, iod->meta_sg, 1, dma_dir))
				goto error_cmd;
		}
837
	}
838

839
	nvme_set_info(cmd, iod, req_completion);
840 841 842 843 844 845 846 847 848 849 850 851
	spin_lock_irq(&nvmeq->q_lock);
	if (req->cmd_flags & REQ_DISCARD)
		nvme_submit_discard(nvmeq, ns, req, iod);
	else if (req->cmd_flags & REQ_FLUSH)
		nvme_submit_flush(nvmeq, ns, req->tag);
	else
		nvme_submit_iod(nvmeq, iod, ns);

	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
	return BLK_MQ_RQ_QUEUE_OK;

852 853 854 855
 error_cmd:
	nvme_free_iod(nvmeq->dev, iod);
	return BLK_MQ_RQ_QUEUE_ERROR;
 retry_cmd:
856
	nvme_free_iod(nvmeq->dev, iod);
857
	return BLK_MQ_RQ_QUEUE_BUSY;
M
Matthew Wilcox 已提交
858 859
}

860
static int nvme_process_cq(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
861
{
862
	u16 head, phase;
M
Matthew Wilcox 已提交
863 864

	head = nvmeq->cq_head;
865
	phase = nvmeq->cq_phase;
M
Matthew Wilcox 已提交
866 867

	for (;;) {
868 869
		void *ctx;
		nvme_completion_fn fn;
M
Matthew Wilcox 已提交
870
		struct nvme_completion cqe = nvmeq->cqes[head];
871
		if ((le16_to_cpu(cqe.status) & 1) != phase)
M
Matthew Wilcox 已提交
872 873 874 875
			break;
		nvmeq->sq_head = le16_to_cpu(cqe.sq_head);
		if (++head == nvmeq->q_depth) {
			head = 0;
876
			phase = !phase;
M
Matthew Wilcox 已提交
877
		}
878
		ctx = nvme_finish_cmd(nvmeq, cqe.command_id, &fn);
879
		fn(nvmeq, ctx, &cqe);
M
Matthew Wilcox 已提交
880 881 882 883 884 885 886 887
	}

	/* If the controller ignores the cq head doorbell and continuously
	 * writes to the queue, it is theoretically possible to wrap around
	 * the queue twice and mistakenly return IRQ_NONE.  Linux only
	 * requires that 0.1% of your interrupts are handled, so this isn't
	 * a big problem.
	 */
888
	if (head == nvmeq->cq_head && phase == nvmeq->cq_phase)
889
		return 0;
M
Matthew Wilcox 已提交
890

891
	writel(head, nvmeq->q_db + nvmeq->dev->db_stride);
M
Matthew Wilcox 已提交
892
	nvmeq->cq_head = head;
893
	nvmeq->cq_phase = phase;
M
Matthew Wilcox 已提交
894

895 896
	nvmeq->cqe_seen = 1;
	return 1;
M
Matthew Wilcox 已提交
897 898
}

899 900 901 902
/* Admin queue isn't initialized as a request queue. If at some point this
 * happens anyway, make sure to notify the user */
static int nvme_admin_queue_rq(struct blk_mq_hw_ctx *hctx,
			       const struct blk_mq_queue_data *bd)
903
{
904 905
	WARN_ON_ONCE(1);
	return BLK_MQ_RQ_QUEUE_ERROR;
906 907
}

M
Matthew Wilcox 已提交
908
static irqreturn_t nvme_irq(int irq, void *data)
909 910 911 912
{
	irqreturn_t result;
	struct nvme_queue *nvmeq = data;
	spin_lock(&nvmeq->q_lock);
913 914 915
	nvme_process_cq(nvmeq);
	result = nvmeq->cqe_seen ? IRQ_HANDLED : IRQ_NONE;
	nvmeq->cqe_seen = 0;
916 917 918 919 920 921 922 923 924 925 926 927 928
	spin_unlock(&nvmeq->q_lock);
	return result;
}

static irqreturn_t nvme_irq_check(int irq, void *data)
{
	struct nvme_queue *nvmeq = data;
	struct nvme_completion cqe = nvmeq->cqes[nvmeq->cq_head];
	if ((le16_to_cpu(cqe.status) & 1) != nvmeq->cq_phase)
		return IRQ_NONE;
	return IRQ_WAKE_THREAD;
}

929 930 931 932 933 934
struct sync_cmd_info {
	struct task_struct *task;
	u32 result;
	int status;
};

935
static void sync_completion(struct nvme_queue *nvmeq, void *ctx,
936 937 938 939 940 941 942 943
						struct nvme_completion *cqe)
{
	struct sync_cmd_info *cmdinfo = ctx;
	cmdinfo->result = le32_to_cpup(&cqe->result);
	cmdinfo->status = le16_to_cpup(&cqe->status) >> 1;
	wake_up_process(cmdinfo->task);
}

M
Matthew Wilcox 已提交
944 945 946 947
/*
 * Returns 0 on success.  If the result is negative, it's a Linux error code;
 * if the result is positive, it's an NVM Express status code
 */
948
static int nvme_submit_sync_cmd(struct request *req, struct nvme_command *cmd,
V
Vishal Verma 已提交
949
						u32 *result, unsigned timeout)
M
Matthew Wilcox 已提交
950 951
{
	struct sync_cmd_info cmdinfo;
952 953
	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd_rq->nvmeq;
M
Matthew Wilcox 已提交
954 955 956 957

	cmdinfo.task = current;
	cmdinfo.status = -EINTR;

958 959 960
	cmd->common.command_id = req->tag;

	nvme_set_info(cmd_rq, &cmdinfo, sync_completion);
M
Matthew Wilcox 已提交
961

962 963 964
	set_current_state(TASK_UNINTERRUPTIBLE);
	nvme_submit_cmd(nvmeq, cmd);
	schedule();
965

M
Matthew Wilcox 已提交
966 967 968 969 970
	if (result)
		*result = cmdinfo.result;
	return cmdinfo.status;
}

971 972 973 974 975 976 977
static int nvme_submit_async_admin_req(struct nvme_dev *dev)
{
	struct nvme_queue *nvmeq = dev->queues[0];
	struct nvme_command c;
	struct nvme_cmd_info *cmd_info;
	struct request *req;

978
	req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC, false);
979 980
	if (IS_ERR(req))
		return PTR_ERR(req);
981

K
Keith Busch 已提交
982
	req->cmd_flags |= REQ_NO_TIMEOUT;
983 984 985 986 987 988 989 990 991 992 993
	cmd_info = blk_mq_rq_to_pdu(req);
	nvme_set_info(cmd_info, req, async_req_completion);

	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_async_event;
	c.common.command_id = req->tag;

	return __nvme_submit_cmd(nvmeq, &c);
}

static int nvme_submit_admin_async_cmd(struct nvme_dev *dev,
994 995 996
			struct nvme_command *cmd,
			struct async_cmd_info *cmdinfo, unsigned timeout)
{
997 998 999
	struct nvme_queue *nvmeq = dev->queues[0];
	struct request *req;
	struct nvme_cmd_info *cmd_rq;
1000

1001
	req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_KERNEL, false);
1002 1003
	if (IS_ERR(req))
		return PTR_ERR(req);
1004 1005 1006 1007 1008

	req->timeout = timeout;
	cmd_rq = blk_mq_rq_to_pdu(req);
	cmdinfo->req = req;
	nvme_set_info(cmd_rq, cmdinfo, async_completion);
1009
	cmdinfo->status = -EINTR;
1010 1011 1012

	cmd->common.command_id = req->tag;

1013
	return nvme_submit_cmd(nvmeq, cmd);
1014 1015
}

1016
static int __nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
1017
						u32 *result, unsigned timeout)
M
Matthew Wilcox 已提交
1018
{
1019 1020 1021 1022
	int res;
	struct request *req;

	req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_KERNEL, false);
1023 1024
	if (IS_ERR(req))
		return PTR_ERR(req);
1025
	res = nvme_submit_sync_cmd(req, cmd, result, timeout);
1026
	blk_mq_free_request(req);
1027
	return res;
1028 1029
}

1030
int nvme_submit_admin_cmd(struct nvme_dev *dev, struct nvme_command *cmd,
1031 1032
								u32 *result)
{
1033
	return __nvme_submit_admin_cmd(dev, cmd, result, ADMIN_TIMEOUT);
M
Matthew Wilcox 已提交
1034 1035
}

1036 1037
int nvme_submit_io_cmd(struct nvme_dev *dev, struct nvme_ns *ns,
					struct nvme_command *cmd, u32 *result)
1038
{
1039 1040 1041 1042 1043
	int res;
	struct request *req;

	req = blk_mq_alloc_request(ns->queue, WRITE, (GFP_KERNEL|__GFP_WAIT),
									false);
1044 1045
	if (IS_ERR(req))
		return PTR_ERR(req);
1046
	res = nvme_submit_sync_cmd(req, cmd, result, NVME_IO_TIMEOUT);
1047
	blk_mq_free_request(req);
1048
	return res;
1049 1050
}

M
Matthew Wilcox 已提交
1051 1052 1053 1054 1055 1056 1057 1058
static int adapter_delete_queue(struct nvme_dev *dev, u8 opcode, u16 id)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(id);

1059
	return nvme_submit_admin_cmd(dev, &c, NULL);
M
Matthew Wilcox 已提交
1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075
}

static int adapter_alloc_cq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_CQ_IRQ_ENABLED;

	memset(&c, 0, sizeof(c));
	c.create_cq.opcode = nvme_admin_create_cq;
	c.create_cq.prp1 = cpu_to_le64(nvmeq->cq_dma_addr);
	c.create_cq.cqid = cpu_to_le16(qid);
	c.create_cq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_cq.cq_flags = cpu_to_le16(flags);
	c.create_cq.irq_vector = cpu_to_le16(nvmeq->cq_vector);

1076
	return nvme_submit_admin_cmd(dev, &c, NULL);
M
Matthew Wilcox 已提交
1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092
}

static int adapter_alloc_sq(struct nvme_dev *dev, u16 qid,
						struct nvme_queue *nvmeq)
{
	struct nvme_command c;
	int flags = NVME_QUEUE_PHYS_CONTIG | NVME_SQ_PRIO_MEDIUM;

	memset(&c, 0, sizeof(c));
	c.create_sq.opcode = nvme_admin_create_sq;
	c.create_sq.prp1 = cpu_to_le64(nvmeq->sq_dma_addr);
	c.create_sq.sqid = cpu_to_le16(qid);
	c.create_sq.qsize = cpu_to_le16(nvmeq->q_depth - 1);
	c.create_sq.sq_flags = cpu_to_le16(flags);
	c.create_sq.cqid = cpu_to_le16(qid);

1093
	return nvme_submit_admin_cmd(dev, &c, NULL);
M
Matthew Wilcox 已提交
1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105
}

static int adapter_delete_cq(struct nvme_dev *dev, u16 cqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_cq, cqid);
}

static int adapter_delete_sq(struct nvme_dev *dev, u16 sqid)
{
	return adapter_delete_queue(dev, nvme_admin_delete_sq, sqid);
}

V
Vishal Verma 已提交
1106
int nvme_identify(struct nvme_dev *dev, unsigned nsid, unsigned cns,
1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119
							dma_addr_t dma_addr)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.identify.opcode = nvme_admin_identify;
	c.identify.nsid = cpu_to_le32(nsid);
	c.identify.prp1 = cpu_to_le64(dma_addr);
	c.identify.cns = cpu_to_le32(cns);

	return nvme_submit_admin_cmd(dev, &c, NULL);
}

V
Vishal Verma 已提交
1120
int nvme_get_features(struct nvme_dev *dev, unsigned fid, unsigned nsid,
1121
					dma_addr_t dma_addr, u32 *result)
1122 1123 1124 1125 1126
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.features.opcode = nvme_admin_get_features;
1127
	c.features.nsid = cpu_to_le32(nsid);
1128 1129 1130
	c.features.prp1 = cpu_to_le64(dma_addr);
	c.features.fid = cpu_to_le32(fid);

1131
	return nvme_submit_admin_cmd(dev, &c, result);
1132 1133
}

V
Vishal Verma 已提交
1134 1135
int nvme_set_features(struct nvme_dev *dev, unsigned fid, unsigned dword11,
					dma_addr_t dma_addr, u32 *result)
1136 1137 1138 1139 1140 1141 1142 1143 1144
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.features.opcode = nvme_admin_set_features;
	c.features.prp1 = cpu_to_le64(dma_addr);
	c.features.fid = cpu_to_le32(fid);
	c.features.dword11 = cpu_to_le32(dword11);

1145 1146 1147
	return nvme_submit_admin_cmd(dev, &c, result);
}

1148
/**
1149
 * nvme_abort_req - Attempt aborting a request
1150 1151 1152 1153
 *
 * Schedule controller reset if the command was already aborted once before and
 * still hasn't been returned to the driver, or if this is the admin queue.
 */
1154
static void nvme_abort_req(struct request *req)
1155
{
1156 1157
	struct nvme_cmd_info *cmd_rq = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd_rq->nvmeq;
1158
	struct nvme_dev *dev = nvmeq->dev;
1159 1160 1161
	struct request *abort_req;
	struct nvme_cmd_info *abort_cmd;
	struct nvme_command cmd;
1162

1163
	if (!nvmeq->qid || cmd_rq->aborted) {
1164 1165 1166
		unsigned long flags;

		spin_lock_irqsave(&dev_list_lock, flags);
1167
		if (work_busy(&dev->reset_work))
1168
			goto out;
1169 1170
		list_del_init(&dev->node);
		dev_warn(&dev->pci_dev->dev,
1171 1172
			"I/O %d QID %d timeout, reset controller\n",
							req->tag, nvmeq->qid);
T
Tejun Heo 已提交
1173
		dev->reset_workfn = nvme_reset_failed_dev;
1174
		queue_work(nvme_workq, &dev->reset_work);
1175 1176
 out:
		spin_unlock_irqrestore(&dev_list_lock, flags);
1177 1178 1179 1180 1181 1182
		return;
	}

	if (!dev->abort_limit)
		return;

1183 1184
	abort_req = blk_mq_alloc_request(dev->admin_q, WRITE, GFP_ATOMIC,
									false);
1185
	if (IS_ERR(abort_req))
1186 1187
		return;

1188 1189 1190
	abort_cmd = blk_mq_rq_to_pdu(abort_req);
	nvme_set_info(abort_cmd, abort_req, abort_completion);

1191 1192
	memset(&cmd, 0, sizeof(cmd));
	cmd.abort.opcode = nvme_admin_abort_cmd;
1193
	cmd.abort.cid = req->tag;
1194
	cmd.abort.sqid = cpu_to_le16(nvmeq->qid);
1195
	cmd.abort.command_id = abort_req->tag;
1196 1197

	--dev->abort_limit;
1198
	cmd_rq->aborted = 1;
1199

1200
	dev_warn(nvmeq->q_dmadev, "Aborting I/O %d QID %d\n", req->tag,
1201
							nvmeq->qid);
1202 1203 1204 1205
	if (nvme_submit_cmd(dev->queues[0], &cmd) < 0) {
		dev_warn(nvmeq->q_dmadev,
				"Could not abort I/O %d QID %d",
				req->tag, nvmeq->qid);
1206
		blk_mq_free_request(abort_req);
1207
	}
1208 1209
}

1210 1211
static void nvme_cancel_queue_ios(struct blk_mq_hw_ctx *hctx,
				struct request *req, void *data, bool reserved)
1212
{
1213 1214 1215 1216
	struct nvme_queue *nvmeq = data;
	void *ctx;
	nvme_completion_fn fn;
	struct nvme_cmd_info *cmd;
1217 1218 1219 1220
	struct nvme_completion cqe;

	if (!blk_mq_request_started(req))
		return;
1221

1222
	cmd = blk_mq_rq_to_pdu(req);
1223

1224 1225 1226
	if (cmd->ctx == CMD_CTX_CANCELLED)
		return;

1227 1228 1229 1230 1231 1232
	if (blk_queue_dying(req->q))
		cqe.status = cpu_to_le16((NVME_SC_ABORT_REQ | NVME_SC_DNR) << 1);
	else
		cqe.status = cpu_to_le16(NVME_SC_ABORT_REQ << 1);


1233 1234 1235 1236
	dev_warn(nvmeq->q_dmadev, "Cancelling I/O %d QID %d\n",
						req->tag, nvmeq->qid);
	ctx = cancel_cmd_info(cmd, &fn);
	fn(nvmeq, ctx, &cqe);
1237 1238
}

1239
static enum blk_eh_timer_return nvme_timeout(struct request *req, bool reserved)
1240
{
1241 1242 1243 1244 1245
	struct nvme_cmd_info *cmd = blk_mq_rq_to_pdu(req);
	struct nvme_queue *nvmeq = cmd->nvmeq;

	dev_warn(nvmeq->q_dmadev, "Timeout I/O %d QID %d\n", req->tag,
							nvmeq->qid);
1246
	spin_lock_irq(&nvmeq->q_lock);
1247
	nvme_abort_req(req);
1248
	spin_unlock_irq(&nvmeq->q_lock);
1249

1250 1251 1252 1253 1254 1255
	/*
	 * The aborted req will be completed on receiving the abort req.
	 * We enable the timer again. If hit twice, it'll cause a device reset,
	 * as the device then is in a faulty state.
	 */
	return BLK_EH_RESET_TIMER;
1256
}
1257

1258 1259
static void nvme_free_queue(struct nvme_queue *nvmeq)
{
1260 1261 1262 1263 1264 1265 1266
	dma_free_coherent(nvmeq->q_dmadev, CQ_SIZE(nvmeq->q_depth),
				(void *)nvmeq->cqes, nvmeq->cq_dma_addr);
	dma_free_coherent(nvmeq->q_dmadev, SQ_SIZE(nvmeq->q_depth),
					nvmeq->sq_cmds, nvmeq->sq_dma_addr);
	kfree(nvmeq);
}

1267
static void nvme_free_queues(struct nvme_dev *dev, int lowest)
1268 1269 1270
{
	int i;

1271
	for (i = dev->queue_count - 1; i >= lowest; i--) {
1272
		struct nvme_queue *nvmeq = dev->queues[i];
1273
		dev->queue_count--;
1274
		dev->queues[i] = NULL;
1275
		nvme_free_queue(nvmeq);
1276
	}
1277 1278
}

1279 1280 1281 1282 1283
/**
 * nvme_suspend_queue - put queue into suspended state
 * @nvmeq - queue to suspend
 */
static int nvme_suspend_queue(struct nvme_queue *nvmeq)
M
Matthew Wilcox 已提交
1284
{
K
Keith Busch 已提交
1285
	int vector;
M
Matthew Wilcox 已提交
1286

1287
	spin_lock_irq(&nvmeq->q_lock);
K
Keith Busch 已提交
1288 1289 1290 1291 1292
	if (nvmeq->cq_vector == -1) {
		spin_unlock_irq(&nvmeq->q_lock);
		return 1;
	}
	vector = nvmeq->dev->entry[nvmeq->cq_vector].vector;
K
Keith Busch 已提交
1293
	nvmeq->dev->online_queues--;
K
Keith Busch 已提交
1294
	nvmeq->cq_vector = -1;
1295 1296
	spin_unlock_irq(&nvmeq->q_lock);

1297 1298
	irq_set_affinity_hint(vector, NULL);
	free_irq(vector, nvmeq);
M
Matthew Wilcox 已提交
1299

1300 1301
	return 0;
}
M
Matthew Wilcox 已提交
1302

1303 1304
static void nvme_clear_queue(struct nvme_queue *nvmeq)
{
1305 1306
	struct blk_mq_hw_ctx *hctx = nvmeq->hctx;

1307
	spin_lock_irq(&nvmeq->q_lock);
1308 1309
	if (hctx && hctx->tags)
		blk_mq_tag_busy_iter(hctx, nvme_cancel_queue_ios, nvmeq);
1310
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1311 1312
}

1313 1314
static void nvme_disable_queue(struct nvme_dev *dev, int qid)
{
1315
	struct nvme_queue *nvmeq = dev->queues[qid];
1316 1317 1318 1319 1320 1321

	if (!nvmeq)
		return;
	if (nvme_suspend_queue(nvmeq))
		return;

1322 1323 1324
	/* Don't tell the adapter to delete the admin queue.
	 * Don't tell a removed adapter to delete IO queues. */
	if (qid && readl(&dev->bar->csts) != -1) {
M
Matthew Wilcox 已提交
1325 1326 1327
		adapter_delete_sq(dev, qid);
		adapter_delete_cq(dev, qid);
	}
1328 1329
	if (!qid && dev->admin_q)
		blk_mq_freeze_queue_start(dev->admin_q);
1330 1331 1332 1333

	spin_lock_irq(&nvmeq->q_lock);
	nvme_process_cq(nvmeq);
	spin_unlock_irq(&nvmeq->q_lock);
M
Matthew Wilcox 已提交
1334 1335 1336
}

static struct nvme_queue *nvme_alloc_queue(struct nvme_dev *dev, int qid,
K
Keith Busch 已提交
1337
							int depth)
M
Matthew Wilcox 已提交
1338 1339
{
	struct device *dmadev = &dev->pci_dev->dev;
1340
	struct nvme_queue *nvmeq = kzalloc(sizeof(*nvmeq), GFP_KERNEL);
M
Matthew Wilcox 已提交
1341 1342 1343
	if (!nvmeq)
		return NULL;

1344 1345
	nvmeq->cqes = dma_zalloc_coherent(dmadev, CQ_SIZE(depth),
					  &nvmeq->cq_dma_addr, GFP_KERNEL);
M
Matthew Wilcox 已提交
1346 1347 1348 1349 1350 1351 1352 1353 1354
	if (!nvmeq->cqes)
		goto free_nvmeq;

	nvmeq->sq_cmds = dma_alloc_coherent(dmadev, SQ_SIZE(depth),
					&nvmeq->sq_dma_addr, GFP_KERNEL);
	if (!nvmeq->sq_cmds)
		goto free_cqdma;

	nvmeq->q_dmadev = dmadev;
1355
	nvmeq->dev = dev;
1356 1357
	snprintf(nvmeq->irqname, sizeof(nvmeq->irqname), "nvme%dq%d",
			dev->instance, qid);
M
Matthew Wilcox 已提交
1358 1359
	spin_lock_init(&nvmeq->q_lock);
	nvmeq->cq_head = 0;
1360
	nvmeq->cq_phase = 1;
1361
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
M
Matthew Wilcox 已提交
1362
	nvmeq->q_depth = depth;
1363
	nvmeq->qid = qid;
1364
	dev->queue_count++;
1365
	dev->queues[qid] = nvmeq;
M
Matthew Wilcox 已提交
1366 1367 1368 1369

	return nvmeq;

 free_cqdma:
1370
	dma_free_coherent(dmadev, CQ_SIZE(depth), (void *)nvmeq->cqes,
M
Matthew Wilcox 已提交
1371 1372 1373 1374 1375 1376
							nvmeq->cq_dma_addr);
 free_nvmeq:
	kfree(nvmeq);
	return NULL;
}

1377 1378 1379
static int queue_request_irq(struct nvme_dev *dev, struct nvme_queue *nvmeq,
							const char *name)
{
1380 1381
	if (use_threaded_interrupts)
		return request_threaded_irq(dev->entry[nvmeq->cq_vector].vector,
1382
					nvme_irq_check, nvme_irq, IRQF_SHARED,
1383
					name, nvmeq);
1384
	return request_irq(dev->entry[nvmeq->cq_vector].vector, nvme_irq,
1385
				IRQF_SHARED, name, nvmeq);
1386 1387
}

1388
static void nvme_init_queue(struct nvme_queue *nvmeq, u16 qid)
M
Matthew Wilcox 已提交
1389
{
1390
	struct nvme_dev *dev = nvmeq->dev;
M
Matthew Wilcox 已提交
1391

1392
	spin_lock_irq(&nvmeq->q_lock);
1393 1394 1395
	nvmeq->sq_tail = 0;
	nvmeq->cq_head = 0;
	nvmeq->cq_phase = 1;
1396
	nvmeq->q_db = &dev->dbs[qid * 2 * dev->db_stride];
1397
	memset((void *)nvmeq->cqes, 0, CQ_SIZE(nvmeq->q_depth));
K
Keith Busch 已提交
1398
	dev->online_queues++;
1399
	spin_unlock_irq(&nvmeq->q_lock);
1400 1401 1402 1403 1404 1405
}

static int nvme_create_queue(struct nvme_queue *nvmeq, int qid)
{
	struct nvme_dev *dev = nvmeq->dev;
	int result;
1406

K
Keith Busch 已提交
1407
	nvmeq->cq_vector = qid - 1;
M
Matthew Wilcox 已提交
1408 1409
	result = adapter_alloc_cq(dev, qid, nvmeq);
	if (result < 0)
1410
		return result;
M
Matthew Wilcox 已提交
1411 1412 1413 1414 1415

	result = adapter_alloc_sq(dev, qid, nvmeq);
	if (result < 0)
		goto release_cq;

1416
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
M
Matthew Wilcox 已提交
1417 1418 1419
	if (result < 0)
		goto release_sq;

1420 1421
	nvme_init_queue(nvmeq, qid);
	return result;
M
Matthew Wilcox 已提交
1422 1423 1424 1425 1426

 release_sq:
	adapter_delete_sq(dev, qid);
 release_cq:
	adapter_delete_cq(dev, qid);
1427
	return result;
M
Matthew Wilcox 已提交
1428 1429
}

1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442
static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled)
{
	unsigned long timeout;
	u32 bit = enabled ? NVME_CSTS_RDY : 0;

	timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;

	while ((readl(&dev->bar->csts) & NVME_CSTS_RDY) != bit) {
		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
			dev_err(&dev->pci_dev->dev,
1443 1444
				"Device not ready; aborting %s\n", enabled ?
						"initialisation" : "reset");
1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
			return -ENODEV;
		}
	}

	return 0;
}

/*
 * If the device has been passed off to us in an enabled state, just clear
 * the enabled bit.  The spec says we should set the 'shutdown notification
 * bits', but doing so may cause the device to complete commands to the
 * admin queue ... and we don't know what memory that might be pointing at!
 */
static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap)
{
1460 1461 1462
	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
	dev->ctrl_config &= ~NVME_CC_ENABLE;
	writel(dev->ctrl_config, &dev->bar->cc);
1463

1464 1465 1466 1467 1468
	return nvme_wait_ready(dev, cap, false);
}

static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap)
{
1469 1470 1471 1472
	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
	dev->ctrl_config |= NVME_CC_ENABLE;
	writel(dev->ctrl_config, &dev->bar->cc);

1473 1474 1475
	return nvme_wait_ready(dev, cap, true);
}

K
Keith Busch 已提交
1476 1477 1478 1479
static int nvme_shutdown_ctrl(struct nvme_dev *dev)
{
	unsigned long timeout;

1480 1481 1482 1483
	dev->ctrl_config &= ~NVME_CC_SHN_MASK;
	dev->ctrl_config |= NVME_CC_SHN_NORMAL;

	writel(dev->ctrl_config, &dev->bar->cc);
K
Keith Busch 已提交
1484

1485
	timeout = SHUTDOWN_TIMEOUT + jiffies;
K
Keith Busch 已提交
1486 1487 1488 1489 1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500
	while ((readl(&dev->bar->csts) & NVME_CSTS_SHST_MASK) !=
							NVME_CSTS_SHST_CMPLT) {
		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
			dev_err(&dev->pci_dev->dev,
				"Device shutdown incomplete; abort shutdown\n");
			return -ENODEV;
		}
	}

	return 0;
}

1501 1502 1503 1504
static struct blk_mq_ops nvme_mq_admin_ops = {
	.queue_rq	= nvme_admin_queue_rq,
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_admin_init_hctx,
J
Jens Axboe 已提交
1505
	.exit_hctx	= nvme_exit_hctx,
1506 1507 1508 1509 1510 1511 1512 1513
	.init_request	= nvme_admin_init_request,
	.timeout	= nvme_timeout,
};

static struct blk_mq_ops nvme_mq_ops = {
	.queue_rq	= nvme_queue_rq,
	.map_queue	= blk_mq_map_queue,
	.init_hctx	= nvme_init_hctx,
J
Jens Axboe 已提交
1514
	.exit_hctx	= nvme_exit_hctx,
1515 1516 1517 1518
	.init_request	= nvme_init_request,
	.timeout	= nvme_timeout,
};

1519 1520 1521 1522 1523 1524 1525 1526
static void nvme_dev_remove_admin(struct nvme_dev *dev)
{
	if (dev->admin_q && !blk_queue_dying(dev->admin_q)) {
		blk_cleanup_queue(dev->admin_q);
		blk_mq_free_tag_set(&dev->admin_tagset);
	}
}

1527 1528 1529 1530 1531 1532 1533 1534
static int nvme_alloc_admin_tags(struct nvme_dev *dev)
{
	if (!dev->admin_q) {
		dev->admin_tagset.ops = &nvme_mq_admin_ops;
		dev->admin_tagset.nr_hw_queues = 1;
		dev->admin_tagset.queue_depth = NVME_AQ_DEPTH - 1;
		dev->admin_tagset.timeout = ADMIN_TIMEOUT;
		dev->admin_tagset.numa_node = dev_to_node(&dev->pci_dev->dev);
1535
		dev->admin_tagset.cmd_size = nvme_cmd_size(dev);
1536 1537 1538 1539 1540 1541
		dev->admin_tagset.driver_data = dev;

		if (blk_mq_alloc_tag_set(&dev->admin_tagset))
			return -ENOMEM;

		dev->admin_q = blk_mq_init_queue(&dev->admin_tagset);
1542
		if (IS_ERR(dev->admin_q)) {
1543 1544 1545
			blk_mq_free_tag_set(&dev->admin_tagset);
			return -ENOMEM;
		}
1546 1547 1548 1549
		if (!blk_get_queue(dev->admin_q)) {
			nvme_dev_remove_admin(dev);
			return -ENODEV;
		}
1550 1551
	} else
		blk_mq_unfreeze_queue(dev->admin_q);
1552 1553 1554 1555

	return 0;
}

1556
static int nvme_configure_admin_queue(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
1557
{
1558
	int result;
M
Matthew Wilcox 已提交
1559
	u32 aqa;
1560
	u64 cap = readq(&dev->bar->cap);
M
Matthew Wilcox 已提交
1561
	struct nvme_queue *nvmeq;
1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579
	unsigned page_shift = PAGE_SHIFT;
	unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12;
	unsigned dev_page_max = NVME_CAP_MPSMAX(cap) + 12;

	if (page_shift < dev_page_min) {
		dev_err(&dev->pci_dev->dev,
				"Minimum device page size (%u) too large for "
				"host (%u)\n", 1 << dev_page_min,
				1 << page_shift);
		return -ENODEV;
	}
	if (page_shift > dev_page_max) {
		dev_info(&dev->pci_dev->dev,
				"Device maximum page size (%u) smaller than "
				"host (%u); enabling work-around\n",
				1 << dev_page_max, 1 << page_shift);
		page_shift = dev_page_max;
	}
M
Matthew Wilcox 已提交
1580

1581 1582 1583
	result = nvme_disable_ctrl(dev, cap);
	if (result < 0)
		return result;
M
Matthew Wilcox 已提交
1584

1585
	nvmeq = dev->queues[0];
1586
	if (!nvmeq) {
K
Keith Busch 已提交
1587
		nvmeq = nvme_alloc_queue(dev, 0, NVME_AQ_DEPTH);
1588 1589 1590
		if (!nvmeq)
			return -ENOMEM;
	}
M
Matthew Wilcox 已提交
1591 1592 1593 1594

	aqa = nvmeq->q_depth - 1;
	aqa |= aqa << 16;

1595 1596
	dev->page_size = 1 << page_shift;

1597
	dev->ctrl_config = NVME_CC_CSS_NVM;
1598
	dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
M
Matthew Wilcox 已提交
1599
	dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
1600
	dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
M
Matthew Wilcox 已提交
1601 1602 1603 1604 1605

	writel(aqa, &dev->bar->aqa);
	writeq(nvmeq->sq_dma_addr, &dev->bar->asq);
	writeq(nvmeq->cq_dma_addr, &dev->bar->acq);

1606
	result = nvme_enable_ctrl(dev, cap);
1607
	if (result)
1608 1609
		goto free_nvmeq;

K
Keith Busch 已提交
1610
	nvmeq->cq_vector = 0;
1611
	result = queue_request_irq(dev, nvmeq, nvmeq->irqname);
1612
	if (result)
1613
		goto free_nvmeq;
1614

M
Matthew Wilcox 已提交
1615
	return result;
1616 1617 1618 1619

 free_nvmeq:
	nvme_free_queues(dev, 0);
	return result;
M
Matthew Wilcox 已提交
1620 1621
}

V
Vishal Verma 已提交
1622
struct nvme_iod *nvme_map_user_pages(struct nvme_dev *dev, int write,
1623
				unsigned long addr, unsigned length)
M
Matthew Wilcox 已提交
1624
{
1625
	int i, err, count, nents, offset;
1626 1627
	struct scatterlist *sg;
	struct page **pages;
1628
	struct nvme_iod *iod;
1629 1630

	if (addr & 3)
1631
		return ERR_PTR(-EINVAL);
1632
	if (!length || length > INT_MAX - PAGE_SIZE)
1633
		return ERR_PTR(-EINVAL);
1634

1635
	offset = offset_in_page(addr);
1636 1637
	count = DIV_ROUND_UP(offset + length, PAGE_SIZE);
	pages = kcalloc(count, sizeof(*pages), GFP_KERNEL);
1638 1639
	if (!pages)
		return ERR_PTR(-ENOMEM);
1640 1641 1642 1643 1644 1645 1646

	err = get_user_pages_fast(addr, count, 1, pages);
	if (err < count) {
		count = err;
		err = -EFAULT;
		goto put_pages;
	}
1647

1648
	err = -ENOMEM;
1649
	iod = __nvme_alloc_iod(count, length, dev, 0, GFP_KERNEL);
1650 1651 1652
	if (!iod)
		goto put_pages;

1653
	sg = iod->sg;
1654
	sg_init_table(sg, count);
1655 1656
	for (i = 0; i < count; i++) {
		sg_set_page(&sg[i], pages[i],
1657 1658
			    min_t(unsigned, length, PAGE_SIZE - offset),
			    offset);
1659 1660
		length -= (PAGE_SIZE - offset);
		offset = 0;
1661
	}
1662
	sg_mark_end(&sg[i - 1]);
1663
	iod->nents = count;
1664 1665 1666

	nents = dma_map_sg(&dev->pci_dev->dev, sg, count,
				write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1667
	if (!nents)
1668
		goto free_iod;
M
Matthew Wilcox 已提交
1669

1670
	kfree(pages);
1671
	return iod;
M
Matthew Wilcox 已提交
1672

1673 1674
 free_iod:
	kfree(iod);
1675 1676 1677 1678
 put_pages:
	for (i = 0; i < count; i++)
		put_page(pages[i]);
	kfree(pages);
1679
	return ERR_PTR(err);
1680
}
M
Matthew Wilcox 已提交
1681

V
Vishal Verma 已提交
1682
void nvme_unmap_user_pages(struct nvme_dev *dev, int write,
1683
			struct nvme_iod *iod)
1684
{
1685
	int i;
M
Matthew Wilcox 已提交
1686

1687 1688
	dma_unmap_sg(&dev->pci_dev->dev, iod->sg, iod->nents,
				write ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1689

1690 1691
	for (i = 0; i < iod->nents; i++)
		put_page(sg_page(&iod->sg[i]));
1692
}
M
Matthew Wilcox 已提交
1693

1694 1695 1696 1697 1698
static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
{
	struct nvme_dev *dev = ns->dev;
	struct nvme_user_io io;
	struct nvme_command c;
1699 1700 1701 1702 1703
	unsigned length, meta_len;
	int status, i;
	struct nvme_iod *iod, *meta_iod = NULL;
	dma_addr_t meta_dma_addr;
	void *meta, *uninitialized_var(meta_mem);
1704 1705 1706

	if (copy_from_user(&io, uio, sizeof(io)))
		return -EFAULT;
1707
	length = (io.nblocks + 1) << ns->lba_shift;
1708 1709 1710 1711
	meta_len = (io.nblocks + 1) * ns->ms;

	if (meta_len && ((io.metadata & 3) || !io.metadata))
		return -EINVAL;
1712 1713 1714 1715

	switch (io.opcode) {
	case nvme_cmd_write:
	case nvme_cmd_read:
M
Matthew Wilcox 已提交
1716
	case nvme_cmd_compare:
1717
		iod = nvme_map_user_pages(dev, io.opcode & 1, io.addr, length);
1718
		break;
1719
	default:
M
Matthew Wilcox 已提交
1720
		return -EINVAL;
1721 1722
	}

1723 1724
	if (IS_ERR(iod))
		return PTR_ERR(iod);
1725 1726 1727 1728

	memset(&c, 0, sizeof(c));
	c.rw.opcode = io.opcode;
	c.rw.flags = io.flags;
1729
	c.rw.nsid = cpu_to_le32(ns->ns_id);
1730
	c.rw.slba = cpu_to_le64(io.slba);
1731
	c.rw.length = cpu_to_le16(io.nblocks);
1732
	c.rw.control = cpu_to_le16(io.control);
1733 1734 1735 1736
	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
	c.rw.reftag = cpu_to_le32(io.reftag);
	c.rw.apptag = cpu_to_le16(io.apptag);
	c.rw.appmask = cpu_to_le16(io.appmask);
1737 1738

	if (meta_len) {
K
Keith Busch 已提交
1739 1740
		meta_iod = nvme_map_user_pages(dev, io.opcode & 1, io.metadata,
								meta_len);
1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769
		if (IS_ERR(meta_iod)) {
			status = PTR_ERR(meta_iod);
			meta_iod = NULL;
			goto unmap;
		}

		meta_mem = dma_alloc_coherent(&dev->pci_dev->dev, meta_len,
						&meta_dma_addr, GFP_KERNEL);
		if (!meta_mem) {
			status = -ENOMEM;
			goto unmap;
		}

		if (io.opcode & 1) {
			int meta_offset = 0;

			for (i = 0; i < meta_iod->nents; i++) {
				meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
						meta_iod->sg[i].offset;
				memcpy(meta_mem + meta_offset, meta,
						meta_iod->sg[i].length);
				kunmap_atomic(meta);
				meta_offset += meta_iod->sg[i].length;
			}
		}

		c.rw.metadata = cpu_to_le64(meta_dma_addr);
	}

1770 1771 1772
	length = nvme_setup_prps(dev, iod, length, GFP_KERNEL);
	c.rw.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
	c.rw.prp2 = cpu_to_le64(iod->first_dma);
1773

1774 1775 1776
	if (length != (io.nblocks + 1) << ns->lba_shift)
		status = -ENOMEM;
	else
1777
		status = nvme_submit_io_cmd(dev, ns, &c, NULL);
1778

1779 1780 1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795 1796 1797
	if (meta_len) {
		if (status == NVME_SC_SUCCESS && !(io.opcode & 1)) {
			int meta_offset = 0;

			for (i = 0; i < meta_iod->nents; i++) {
				meta = kmap_atomic(sg_page(&meta_iod->sg[i])) +
						meta_iod->sg[i].offset;
				memcpy(meta, meta_mem + meta_offset,
						meta_iod->sg[i].length);
				kunmap_atomic(meta);
				meta_offset += meta_iod->sg[i].length;
			}
		}

		dma_free_coherent(&dev->pci_dev->dev, meta_len, meta_mem,
								meta_dma_addr);
	}

 unmap:
1798
	nvme_unmap_user_pages(dev, io.opcode & 1, iod);
1799
	nvme_free_iod(dev, iod);
1800 1801 1802 1803 1804 1805

	if (meta_iod) {
		nvme_unmap_user_pages(dev, io.opcode & 1, meta_iod);
		nvme_free_iod(dev, meta_iod);
	}

1806 1807 1808
	return status;
}

1809 1810
static int nvme_user_cmd(struct nvme_dev *dev, struct nvme_ns *ns,
			struct nvme_passthru_cmd __user *ucmd)
1811
{
1812
	struct nvme_passthru_cmd cmd;
1813
	struct nvme_command c;
1814
	int status, length;
1815
	struct nvme_iod *uninitialized_var(iod);
1816
	unsigned timeout;
1817

M
Matthew Wilcox 已提交
1818 1819 1820
	if (!capable(CAP_SYS_ADMIN))
		return -EACCES;
	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
1821 1822 1823
		return -EFAULT;

	memset(&c, 0, sizeof(c));
M
Matthew Wilcox 已提交
1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837
	c.common.opcode = cmd.opcode;
	c.common.flags = cmd.flags;
	c.common.nsid = cpu_to_le32(cmd.nsid);
	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
	c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
	c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
	c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
	c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
	c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
	c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);

	length = cmd.data_len;
	if (cmd.data_len) {
1838 1839
		iod = nvme_map_user_pages(dev, cmd.opcode & 1, cmd.addr,
								length);
1840 1841
		if (IS_ERR(iod))
			return PTR_ERR(iod);
1842 1843 1844
		length = nvme_setup_prps(dev, iod, length, GFP_KERNEL);
		c.common.prp1 = cpu_to_le64(sg_dma_address(iod->sg));
		c.common.prp2 = cpu_to_le64(iod->first_dma);
M
Matthew Wilcox 已提交
1845 1846
	}

1847 1848
	timeout = cmd.timeout_ms ? msecs_to_jiffies(cmd.timeout_ms) :
								ADMIN_TIMEOUT;
1849

M
Matthew Wilcox 已提交
1850
	if (length != cmd.data_len)
1851
		status = -ENOMEM;
1852 1853 1854 1855 1856
	else if (ns) {
		struct request *req;

		req = blk_mq_alloc_request(ns->queue, WRITE,
						(GFP_KERNEL|__GFP_WAIT), false);
1857 1858
		if (IS_ERR(req))
			status = PTR_ERR(req);
1859 1860 1861
		else {
			status = nvme_submit_sync_cmd(req, &c, &cmd.result,
								timeout);
1862
			blk_mq_free_request(req);
1863 1864 1865
		}
	} else
		status = __nvme_submit_admin_cmd(dev, &c, &cmd.result, timeout);
1866

M
Matthew Wilcox 已提交
1867
	if (cmd.data_len) {
1868
		nvme_unmap_user_pages(dev, cmd.opcode & 1, iod);
1869
		nvme_free_iod(dev, iod);
M
Matthew Wilcox 已提交
1870
	}
1871

1872
	if ((status >= 0) && copy_to_user(&ucmd->result, &cmd.result,
1873 1874 1875
							sizeof(cmd.result)))
		status = -EFAULT;

1876 1877 1878
	return status;
}

M
Matthew Wilcox 已提交
1879 1880 1881 1882 1883 1884
static int nvme_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
							unsigned long arg)
{
	struct nvme_ns *ns = bdev->bd_disk->private_data;

	switch (cmd) {
M
Matthew Wilcox 已提交
1885
	case NVME_IOCTL_ID:
1886
		force_successful_syscall_return();
M
Matthew Wilcox 已提交
1887 1888
		return ns->ns_id;
	case NVME_IOCTL_ADMIN_CMD:
1889
		return nvme_user_cmd(ns->dev, NULL, (void __user *)arg);
1890
	case NVME_IOCTL_IO_CMD:
1891
		return nvme_user_cmd(ns->dev, ns, (void __user *)arg);
1892 1893
	case NVME_IOCTL_SUBMIT_IO:
		return nvme_submit_io(ns, (void __user *)arg);
V
Vishal Verma 已提交
1894 1895 1896 1897
	case SG_GET_VERSION_NUM:
		return nvme_sg_get_version_num((void __user *)arg);
	case SG_IO:
		return nvme_sg_io(ns, (void __user *)arg);
M
Matthew Wilcox 已提交
1898 1899 1900 1901 1902
	default:
		return -ENOTTY;
	}
}

K
Keith Busch 已提交
1903 1904 1905 1906 1907 1908
#ifdef CONFIG_COMPAT
static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
					unsigned int cmd, unsigned long arg)
{
	switch (cmd) {
	case SG_IO:
1909
		return -ENOIOCTLCMD;
K
Keith Busch 已提交
1910 1911 1912 1913 1914 1915 1916
	}
	return nvme_ioctl(bdev, mode, cmd, arg);
}
#else
#define nvme_compat_ioctl	NULL
#endif

1917 1918
static int nvme_open(struct block_device *bdev, fmode_t mode)
{
1919 1920
	int ret = 0;
	struct nvme_ns *ns;
1921

1922 1923 1924 1925 1926 1927 1928 1929 1930
	spin_lock(&dev_list_lock);
	ns = bdev->bd_disk->private_data;
	if (!ns)
		ret = -ENXIO;
	else if (!kref_get_unless_zero(&ns->dev->kref))
		ret = -ENXIO;
	spin_unlock(&dev_list_lock);

	return ret;
1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942
}

static void nvme_free_dev(struct kref *kref);

static void nvme_release(struct gendisk *disk, fmode_t mode)
{
	struct nvme_ns *ns = disk->private_data;
	struct nvme_dev *dev = ns->dev;

	kref_put(&dev->kref, nvme_free_dev);
}

1943 1944 1945 1946 1947 1948 1949 1950 1951
static int nvme_getgeo(struct block_device *bd, struct hd_geometry *geo)
{
	/* some standard values */
	geo->heads = 1 << 6;
	geo->sectors = 1 << 5;
	geo->cylinders = get_capacity(bd->bd_disk) >> 11;
	return 0;
}

1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975 1976 1977 1978 1979 1980 1981 1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998
static void nvme_config_discard(struct nvme_ns *ns)
{
	u32 logical_block_size = queue_logical_block_size(ns->queue);
	ns->queue->limits.discard_zeroes_data = 0;
	ns->queue->limits.discard_alignment = logical_block_size;
	ns->queue->limits.discard_granularity = logical_block_size;
	ns->queue->limits.max_discard_sectors = 0xffffffff;
	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
}

static int nvme_noop_verify(struct blk_integrity_iter *iter)
{
	return 0;
}

static int nvme_noop_generate(struct blk_integrity_iter *iter)
{
	return 0;
}

struct blk_integrity nvme_meta_noop = {
	.name			= "NVME_META_NOOP",
	.generate_fn		= nvme_noop_generate,
	.verify_fn		= nvme_noop_verify,
};

static void nvme_init_integrity(struct nvme_ns *ns)
{
	struct blk_integrity integrity;

	switch (ns->pi_type) {
	case NVME_NS_DPS_PI_TYPE3:
		integrity = t10_pi_type3_crc;
		break;
	case NVME_NS_DPS_PI_TYPE1:
	case NVME_NS_DPS_PI_TYPE2:
		integrity = t10_pi_type1_crc;
		break;
	default:
		integrity = nvme_meta_noop;
		break;
	}
	integrity.tuple_size = ns->ms;
	blk_integrity_register(ns->disk, &integrity);
	blk_queue_max_integrity_segments(ns->queue, 1);
}

1999 2000 2001 2002 2003 2004
static int nvme_revalidate_disk(struct gendisk *disk)
{
	struct nvme_ns *ns = disk->private_data;
	struct nvme_dev *dev = ns->dev;
	struct nvme_id_ns *id;
	dma_addr_t dma_addr;
2005 2006
	int lbaf, pi_type, old_ms;
	unsigned short bs;
2007 2008 2009 2010 2011 2012 2013 2014

	id = dma_alloc_coherent(&dev->pci_dev->dev, 4096, &dma_addr,
								GFP_KERNEL);
	if (!id) {
		dev_warn(&dev->pci_dev->dev, "%s: Memory alocation failure\n",
								__func__);
		return 0;
	}
2015 2016 2017 2018 2019 2020
	if (nvme_identify(dev, ns->ns_id, 0, dma_addr)) {
		dev_warn(&dev->pci_dev->dev,
			"identify failed ns:%d, setting capacity to 0\n",
			ns->ns_id);
		memset(id, 0, sizeof(*id));
	}
2021

2022 2023
	old_ms = ns->ms;
	lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
2024
	ns->lba_shift = id->lbaf[lbaf].ds;
2025 2026 2027 2028 2029 2030 2031 2032 2033 2034 2035 2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057
	ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);

	/*
	 * If identify namespace failed, use default 512 byte block size so
	 * block layer can use before failing read/write for 0 capacity.
	 */
	if (ns->lba_shift == 0)
		ns->lba_shift = 9;
	bs = 1 << ns->lba_shift;

	/* XXX: PI implementation requires metadata equal t10 pi tuple size */
	pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
					id->dps & NVME_NS_DPS_PI_MASK : 0;

	if (disk->integrity && (ns->pi_type != pi_type || ns->ms != old_ms ||
				bs != queue_logical_block_size(disk->queue) ||
				(ns->ms && id->flbas & NVME_NS_FLBAS_META_EXT)))
		blk_integrity_unregister(disk);

	ns->pi_type = pi_type;
	blk_queue_logical_block_size(ns->queue, bs);

	if (ns->ms && !disk->integrity && (disk->flags & GENHD_FL_UP) &&
				!(id->flbas & NVME_NS_FLBAS_META_EXT))
		nvme_init_integrity(ns);

	if (id->ncap == 0 || (ns->ms && !disk->integrity))
		set_capacity(disk, 0);
	else
		set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));

	if (dev->oncs & NVME_CTRL_ONCS_DSM)
		nvme_config_discard(ns);
2058 2059 2060 2061 2062

	dma_free_coherent(&dev->pci_dev->dev, 4096, id, dma_addr);
	return 0;
}

M
Matthew Wilcox 已提交
2063 2064 2065
static const struct block_device_operations nvme_fops = {
	.owner		= THIS_MODULE,
	.ioctl		= nvme_ioctl,
K
Keith Busch 已提交
2066
	.compat_ioctl	= nvme_compat_ioctl,
2067 2068
	.open		= nvme_open,
	.release	= nvme_release,
2069
	.getgeo		= nvme_getgeo,
2070
	.revalidate_disk= nvme_revalidate_disk,
M
Matthew Wilcox 已提交
2071 2072
};

2073 2074
static int nvme_kthread(void *data)
{
2075
	struct nvme_dev *dev, *next;
2076 2077

	while (!kthread_should_stop()) {
2078
		set_current_state(TASK_INTERRUPTIBLE);
2079
		spin_lock(&dev_list_lock);
2080
		list_for_each_entry_safe(dev, next, &dev_list, node) {
2081
			int i;
2082
			if (readl(&dev->bar->csts) & NVME_CSTS_CFS) {
2083 2084 2085 2086
				if (work_busy(&dev->reset_work))
					continue;
				list_del_init(&dev->node);
				dev_warn(&dev->pci_dev->dev,
2087 2088
					"Failed status: %x, reset controller\n",
					readl(&dev->bar->csts));
T
Tejun Heo 已提交
2089
				dev->reset_workfn = nvme_reset_failed_dev;
2090 2091 2092
				queue_work(nvme_workq, &dev->reset_work);
				continue;
			}
2093
			for (i = 0; i < dev->queue_count; i++) {
2094
				struct nvme_queue *nvmeq = dev->queues[i];
2095 2096
				if (!nvmeq)
					continue;
2097
				spin_lock_irq(&nvmeq->q_lock);
2098
				nvme_process_cq(nvmeq);
K
Keith Busch 已提交
2099 2100

				while ((i == 0) && (dev->event_limit > 0)) {
2101
					if (nvme_submit_async_admin_req(dev))
K
Keith Busch 已提交
2102 2103 2104
						break;
					dev->event_limit--;
				}
2105 2106 2107 2108
				spin_unlock_irq(&nvmeq->q_lock);
			}
		}
		spin_unlock(&dev_list_lock);
2109
		schedule_timeout(round_jiffies_relative(HZ));
2110 2111 2112 2113
	}
	return 0;
}

2114
static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid)
M
Matthew Wilcox 已提交
2115 2116 2117
{
	struct nvme_ns *ns;
	struct gendisk *disk;
2118
	int node = dev_to_node(&dev->pci_dev->dev);
M
Matthew Wilcox 已提交
2119

2120
	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2121
	if (!ns)
2122 2123
		return;

2124
	ns->queue = blk_mq_init_queue(&dev->tagset);
2125
	if (IS_ERR(ns->queue))
M
Matthew Wilcox 已提交
2126
		goto out_free_ns;
2127 2128
	queue_flag_set_unlocked(QUEUE_FLAG_NOMERGES, ns->queue);
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
2129
	queue_flag_set_unlocked(QUEUE_FLAG_SG_GAPS, ns->queue);
M
Matthew Wilcox 已提交
2130 2131 2132
	ns->dev = dev;
	ns->queue->queuedata = ns;

2133
	disk = alloc_disk_node(0, node);
M
Matthew Wilcox 已提交
2134 2135
	if (!disk)
		goto out_free_queue;
2136

2137
	ns->ns_id = nsid;
M
Matthew Wilcox 已提交
2138
	ns->disk = disk;
2139 2140 2141
	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */
	list_add_tail(&ns->list, &dev->namespaces);

2142
	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
2143 2144
	if (dev->max_hw_sectors)
		blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors);
2145 2146
	if (dev->stripe_size)
		blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9);
2147 2148
	if (dev->vwc & NVME_CTRL_VWC_PRESENT)
		blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA);
M
Matthew Wilcox 已提交
2149 2150

	disk->major = nvme_major;
2151
	disk->first_minor = 0;
M
Matthew Wilcox 已提交
2152 2153 2154
	disk->fops = &nvme_fops;
	disk->private_data = ns;
	disk->queue = ns->queue;
2155
	disk->driverfs_dev = dev->device;
2156
	disk->flags = GENHD_FL_EXT_DEVT;
2157
	sprintf(disk->disk_name, "nvme%dn%d", dev->instance, nsid);
M
Matthew Wilcox 已提交
2158

2159 2160 2161 2162 2163 2164 2165 2166 2167 2168 2169 2170
	/*
	 * Initialize capacity to 0 until we establish the namespace format and
	 * setup integrity extentions if necessary. The revalidate_disk after
	 * add_disk allows the driver to register with integrity if the format
	 * requires it.
	 */
	set_capacity(disk, 0);
	nvme_revalidate_disk(ns->disk);
	add_disk(ns->disk);
	if (ns->ms)
		revalidate_disk(ns->disk);
	return;
M
Matthew Wilcox 已提交
2171 2172 2173 2174 2175 2176
 out_free_queue:
	blk_cleanup_queue(ns->queue);
 out_free_ns:
	kfree(ns);
}

K
Keith Busch 已提交
2177 2178
static void nvme_create_io_queues(struct nvme_dev *dev)
{
2179
	unsigned i;
K
Keith Busch 已提交
2180

2181
	for (i = dev->queue_count; i <= dev->max_qid; i++)
K
Keith Busch 已提交
2182
		if (!nvme_alloc_queue(dev, i, dev->q_depth))
K
Keith Busch 已提交
2183 2184
			break;

2185 2186
	for (i = dev->online_queues; i <= dev->queue_count - 1; i++)
		if (nvme_create_queue(dev->queues[i], i))
K
Keith Busch 已提交
2187 2188 2189
			break;
}

2190
static int set_queue_count(struct nvme_dev *dev, int count)
M
Matthew Wilcox 已提交
2191 2192 2193
{
	int status;
	u32 result;
2194
	u32 q_count = (count - 1) | ((count - 1) << 16);
M
Matthew Wilcox 已提交
2195

2196
	status = nvme_set_features(dev, NVME_FEAT_NUM_QUEUES, q_count, 0,
2197
								&result);
2198 2199 2200 2201 2202
	if (status < 0)
		return status;
	if (status > 0) {
		dev_err(&dev->pci_dev->dev, "Could not set queue count (%d)\n",
									status);
2203
		return 0;
2204
	}
M
Matthew Wilcox 已提交
2205 2206 2207
	return min(result & 0xffff, result >> 16) + 1;
}

2208 2209
static size_t db_bar_size(struct nvme_dev *dev, unsigned nr_io_queues)
{
2210
	return 4096 + ((nr_io_queues + 1) * 8 * dev->db_stride);
2211 2212
}

2213
static int nvme_setup_io_queues(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2214
{
2215
	struct nvme_queue *adminq = dev->queues[0];
2216
	struct pci_dev *pdev = dev->pci_dev;
K
Keith Busch 已提交
2217
	int result, i, vecs, nr_io_queues, size;
M
Matthew Wilcox 已提交
2218

K
Keith Busch 已提交
2219
	nr_io_queues = num_possible_cpus();
2220
	result = set_queue_count(dev, nr_io_queues);
2221
	if (result <= 0)
2222
		return result;
2223 2224
	if (result < nr_io_queues)
		nr_io_queues = result;
M
Matthew Wilcox 已提交
2225

2226 2227
	size = db_bar_size(dev, nr_io_queues);
	if (size > 8192) {
2228
		iounmap(dev->bar);
2229 2230 2231 2232 2233 2234 2235 2236
		do {
			dev->bar = ioremap(pci_resource_start(pdev, 0), size);
			if (dev->bar)
				break;
			if (!--nr_io_queues)
				return -ENOMEM;
			size = db_bar_size(dev, nr_io_queues);
		} while (1);
2237
		dev->dbs = ((void __iomem *)dev->bar) + 4096;
2238
		adminq->q_db = dev->dbs;
2239 2240
	}

2241
	/* Deregister the admin queue's interrupt */
2242
	free_irq(dev->entry[0].vector, adminq);
2243

2244 2245 2246 2247 2248 2249 2250
	/*
	 * If we enable msix early due to not intx, disable it again before
	 * setting up the full range we need.
	 */
	if (!pdev->irq)
		pci_disable_msix(pdev);

2251
	for (i = 0; i < nr_io_queues; i++)
2252
		dev->entry[i].entry = i;
2253 2254 2255 2256 2257 2258 2259 2260
	vecs = pci_enable_msix_range(pdev, dev->entry, 1, nr_io_queues);
	if (vecs < 0) {
		vecs = pci_enable_msi_range(pdev, 1, min(nr_io_queues, 32));
		if (vecs < 0) {
			vecs = 1;
		} else {
			for (i = 0; i < vecs; i++)
				dev->entry[i].vector = i + pdev->irq;
2261 2262 2263
		}
	}

2264 2265 2266 2267 2268 2269 2270
	/*
	 * Should investigate if there's a performance win from allocating
	 * more queues than interrupt vectors; it might allow the submission
	 * path to scale better, even if the receive path is limited by the
	 * number of interrupts.
	 */
	nr_io_queues = vecs;
K
Keith Busch 已提交
2271
	dev->max_qid = nr_io_queues;
2272

2273
	result = queue_request_irq(dev, adminq, adminq->irqname);
2274
	if (result)
2275
		goto free_queues;
2276

2277
	/* Free previously allocated queues that are no longer usable */
K
Keith Busch 已提交
2278
	nvme_free_queues(dev, nr_io_queues + 1);
2279
	nvme_create_io_queues(dev);
2280

2281
	return 0;
M
Matthew Wilcox 已提交
2282

2283
 free_queues:
2284
	nvme_free_queues(dev, 1);
2285
	return result;
M
Matthew Wilcox 已提交
2286 2287
}

2288 2289 2290 2291 2292 2293
/*
 * Return: error value if an error occurred setting up the queues or calling
 * Identify Device.  0 if these succeeded, even if adding some of the
 * namespaces failed.  At the moment, these failures are silent.  TBD which
 * failures should be reported.
 */
2294
static int nvme_dev_add(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2295
{
2296
	struct pci_dev *pdev = dev->pci_dev;
2297 2298
	int res;
	unsigned nn, i;
2299
	struct nvme_id_ctrl *ctrl;
2300
	void *mem;
M
Matthew Wilcox 已提交
2301
	dma_addr_t dma_addr;
2302
	int shift = NVME_CAP_MPSMIN(readq(&dev->bar->cap)) + 12;
M
Matthew Wilcox 已提交
2303

2304
	mem = dma_alloc_coherent(&pdev->dev, 4096, &dma_addr, GFP_KERNEL);
2305 2306
	if (!mem)
		return -ENOMEM;
M
Matthew Wilcox 已提交
2307

2308
	res = nvme_identify(dev, 0, 1, dma_addr);
M
Matthew Wilcox 已提交
2309
	if (res) {
2310
		dev_err(&pdev->dev, "Identify Controller failed (%d)\n", res);
2311 2312
		dma_free_coherent(&dev->pci_dev->dev, 4096, mem, dma_addr);
		return -EIO;
M
Matthew Wilcox 已提交
2313 2314
	}

2315
	ctrl = mem;
2316
	nn = le32_to_cpup(&ctrl->nn);
2317
	dev->oncs = le16_to_cpup(&ctrl->oncs);
2318
	dev->abort_limit = ctrl->acl + 1;
2319
	dev->vwc = ctrl->vwc;
K
Keith Busch 已提交
2320
	dev->event_limit = min(ctrl->aerl + 1, 8);
2321 2322 2323
	memcpy(dev->serial, ctrl->sn, sizeof(ctrl->sn));
	memcpy(dev->model, ctrl->mn, sizeof(ctrl->mn));
	memcpy(dev->firmware_rev, ctrl->fr, sizeof(ctrl->fr));
2324
	if (ctrl->mdts)
2325
		dev->max_hw_sectors = 1 << (ctrl->mdts + shift - 9);
2326
	if ((pdev->vendor == PCI_VENDOR_ID_INTEL) &&
2327 2328 2329
			(pdev->device == 0x0953) && ctrl->vs[3]) {
		unsigned int max_hw_sectors;

2330
		dev->stripe_size = 1 << (ctrl->vs[3] + shift);
2331 2332 2333 2334 2335 2336 2337
		max_hw_sectors = dev->stripe_size >> (shift - 9);
		if (dev->max_hw_sectors) {
			dev->max_hw_sectors = min(max_hw_sectors,
							dev->max_hw_sectors);
		} else
			dev->max_hw_sectors = max_hw_sectors;
	}
2338
	dma_free_coherent(&dev->pci_dev->dev, 4096, mem, dma_addr);
2339 2340 2341 2342 2343 2344 2345

	dev->tagset.ops = &nvme_mq_ops;
	dev->tagset.nr_hw_queues = dev->online_queues - 1;
	dev->tagset.timeout = NVME_IO_TIMEOUT;
	dev->tagset.numa_node = dev_to_node(&dev->pci_dev->dev);
	dev->tagset.queue_depth =
				min_t(int, dev->q_depth, BLK_MQ_MAX_DEPTH) - 1;
2346
	dev->tagset.cmd_size = nvme_cmd_size(dev);
2347 2348 2349 2350
	dev->tagset.flags = BLK_MQ_F_SHOULD_MERGE;
	dev->tagset.driver_data = dev;

	if (blk_mq_alloc_tag_set(&dev->tagset))
2351
		return 0;
M
Matthew Wilcox 已提交
2352

2353 2354
	for (i = 1; i <= nn; i++)
		nvme_alloc_ns(dev, i);
M
Matthew Wilcox 已提交
2355

2356
	return 0;
M
Matthew Wilcox 已提交
2357 2358
}

2359 2360
static int nvme_dev_map(struct nvme_dev *dev)
{
K
Keith Busch 已提交
2361
	u64 cap;
2362 2363 2364 2365 2366 2367 2368 2369 2370
	int bars, result = -ENOMEM;
	struct pci_dev *pdev = dev->pci_dev;

	if (pci_enable_device_mem(pdev))
		return result;

	dev->entry[0].vector = pdev->irq;
	pci_set_master(pdev);
	bars = pci_select_bars(pdev, IORESOURCE_MEM);
2371 2372 2373
	if (!bars)
		goto disable_pci;

2374 2375 2376
	if (pci_request_selected_regions(pdev, bars, "nvme"))
		goto disable_pci;

2377 2378 2379
	if (dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)) &&
	    dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32)))
		goto disable;
2380 2381 2382 2383

	dev->bar = ioremap(pci_resource_start(pdev, 0), 8192);
	if (!dev->bar)
		goto disable;
2384

2385 2386 2387 2388
	if (readl(&dev->bar->csts) == -1) {
		result = -ENODEV;
		goto unmap;
	}
2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399

	/*
	 * Some devices don't advertse INTx interrupts, pre-enable a single
	 * MSIX vec for setup. We'll adjust this later.
	 */
	if (!pdev->irq) {
		result = pci_enable_msix(pdev, dev->entry, 1);
		if (result < 0)
			goto unmap;
	}

K
Keith Busch 已提交
2400 2401 2402
	cap = readq(&dev->bar->cap);
	dev->q_depth = min_t(int, NVME_CAP_MQES(cap) + 1, NVME_Q_DEPTH);
	dev->db_stride = 1 << NVME_CAP_STRIDE(cap);
2403 2404 2405 2406
	dev->dbs = ((void __iomem *)dev->bar) + 4096;

	return 0;

2407 2408 2409
 unmap:
	iounmap(dev->bar);
	dev->bar = NULL;
2410 2411 2412 2413 2414 2415 2416 2417 2418 2419 2420 2421 2422 2423 2424 2425 2426
 disable:
	pci_release_regions(pdev);
 disable_pci:
	pci_disable_device(pdev);
	return result;
}

static void nvme_dev_unmap(struct nvme_dev *dev)
{
	if (dev->pci_dev->msi_enabled)
		pci_disable_msi(dev->pci_dev);
	else if (dev->pci_dev->msix_enabled)
		pci_disable_msix(dev->pci_dev);

	if (dev->bar) {
		iounmap(dev->bar);
		dev->bar = NULL;
2427
		pci_release_regions(dev->pci_dev);
2428 2429 2430 2431 2432 2433
	}

	if (pci_is_enabled(dev->pci_dev))
		pci_disable_device(dev->pci_dev);
}

2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450
struct nvme_delq_ctx {
	struct task_struct *waiter;
	struct kthread_worker *worker;
	atomic_t refcount;
};

static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev)
{
	dq->waiter = current;
	mb();

	for (;;) {
		set_current_state(TASK_KILLABLE);
		if (!atomic_read(&dq->refcount))
			break;
		if (!schedule_timeout(ADMIN_TIMEOUT) ||
					fatal_signal_pending(current)) {
2451 2452 2453 2454 2455 2456 2457
			/*
			 * Disable the controller first since we can't trust it
			 * at this point, but leave the admin queue enabled
			 * until all queue deletion requests are flushed.
			 * FIXME: This may take a while if there are more h/w
			 * queues than admin tags.
			 */
2458 2459
			set_current_state(TASK_RUNNING);
			nvme_disable_ctrl(dev, readq(&dev->bar->cap));
2460
			nvme_clear_queue(dev->queues[0]);
2461
			flush_kthread_worker(dq->worker);
2462
			nvme_disable_queue(dev, 0);
2463 2464 2465 2466 2467 2468 2469 2470 2471 2472 2473 2474 2475 2476 2477 2478 2479 2480 2481 2482 2483 2484 2485 2486 2487 2488 2489 2490 2491 2492 2493 2494 2495 2496 2497
			return;
		}
	}
	set_current_state(TASK_RUNNING);
}

static void nvme_put_dq(struct nvme_delq_ctx *dq)
{
	atomic_dec(&dq->refcount);
	if (dq->waiter)
		wake_up_process(dq->waiter);
}

static struct nvme_delq_ctx *nvme_get_dq(struct nvme_delq_ctx *dq)
{
	atomic_inc(&dq->refcount);
	return dq;
}

static void nvme_del_queue_end(struct nvme_queue *nvmeq)
{
	struct nvme_delq_ctx *dq = nvmeq->cmdinfo.ctx;
	nvme_put_dq(dq);
}

static int adapter_async_del_queue(struct nvme_queue *nvmeq, u8 opcode,
						kthread_work_func_t fn)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	c.delete_queue.opcode = opcode;
	c.delete_queue.qid = cpu_to_le16(nvmeq->qid);

	init_kthread_work(&nvmeq->cmdinfo.work, fn);
2498 2499
	return nvme_submit_admin_async_cmd(nvmeq->dev, &c, &nvmeq->cmdinfo,
								ADMIN_TIMEOUT);
2500 2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538 2539 2540 2541 2542 2543 2544 2545 2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556 2557 2558 2559 2560
}

static void nvme_del_cq_work_handler(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	nvme_del_queue_end(nvmeq);
}

static int nvme_delete_cq(struct nvme_queue *nvmeq)
{
	return adapter_async_del_queue(nvmeq, nvme_admin_delete_cq,
						nvme_del_cq_work_handler);
}

static void nvme_del_sq_work_handler(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	int status = nvmeq->cmdinfo.status;

	if (!status)
		status = nvme_delete_cq(nvmeq);
	if (status)
		nvme_del_queue_end(nvmeq);
}

static int nvme_delete_sq(struct nvme_queue *nvmeq)
{
	return adapter_async_del_queue(nvmeq, nvme_admin_delete_sq,
						nvme_del_sq_work_handler);
}

static void nvme_del_queue_start(struct kthread_work *work)
{
	struct nvme_queue *nvmeq = container_of(work, struct nvme_queue,
							cmdinfo.work);
	if (nvme_delete_sq(nvmeq))
		nvme_del_queue_end(nvmeq);
}

static void nvme_disable_io_queues(struct nvme_dev *dev)
{
	int i;
	DEFINE_KTHREAD_WORKER_ONSTACK(worker);
	struct nvme_delq_ctx dq;
	struct task_struct *kworker_task = kthread_run(kthread_worker_fn,
					&worker, "nvme%d", dev->instance);

	if (IS_ERR(kworker_task)) {
		dev_err(&dev->pci_dev->dev,
			"Failed to create queue del task\n");
		for (i = dev->queue_count - 1; i > 0; i--)
			nvme_disable_queue(dev, i);
		return;
	}

	dq.waiter = NULL;
	atomic_set(&dq.refcount, 0);
	dq.worker = &worker;
	for (i = dev->queue_count - 1; i > 0; i--) {
2561
		struct nvme_queue *nvmeq = dev->queues[i];
2562 2563 2564 2565 2566 2567 2568 2569 2570 2571 2572 2573

		if (nvme_suspend_queue(nvmeq))
			continue;
		nvmeq->cmdinfo.ctx = nvme_get_dq(&dq);
		nvmeq->cmdinfo.worker = dq.worker;
		init_kthread_work(&nvmeq->cmdinfo.work, nvme_del_queue_start);
		queue_kthread_work(dq.worker, &nvmeq->cmdinfo.work);
	}
	nvme_wait_dq(&dq, dev);
	kthread_stop(kworker_task);
}

2574 2575 2576 2577 2578 2579 2580 2581 2582 2583 2584 2585 2586 2587 2588 2589 2590 2591 2592 2593
/*
* Remove the node from the device list and check
* for whether or not we need to stop the nvme_thread.
*/
static void nvme_dev_list_remove(struct nvme_dev *dev)
{
	struct task_struct *tmp = NULL;

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	if (list_empty(&dev_list) && !IS_ERR_OR_NULL(nvme_thread)) {
		tmp = nvme_thread;
		nvme_thread = NULL;
	}
	spin_unlock(&dev_list_lock);

	if (tmp)
		kthread_stop(tmp);
}

2594 2595 2596 2597 2598 2599 2600 2601 2602 2603 2604 2605 2606 2607 2608 2609 2610 2611 2612 2613 2614 2615 2616 2617 2618 2619 2620 2621
static void nvme_freeze_queues(struct nvme_dev *dev)
{
	struct nvme_ns *ns;

	list_for_each_entry(ns, &dev->namespaces, list) {
		blk_mq_freeze_queue_start(ns->queue);

		spin_lock(ns->queue->queue_lock);
		queue_flag_set(QUEUE_FLAG_STOPPED, ns->queue);
		spin_unlock(ns->queue->queue_lock);

		blk_mq_cancel_requeue_work(ns->queue);
		blk_mq_stop_hw_queues(ns->queue);
	}
}

static void nvme_unfreeze_queues(struct nvme_dev *dev)
{
	struct nvme_ns *ns;

	list_for_each_entry(ns, &dev->namespaces, list) {
		queue_flag_clear_unlocked(QUEUE_FLAG_STOPPED, ns->queue);
		blk_mq_unfreeze_queue(ns->queue);
		blk_mq_start_stopped_hw_queues(ns->queue, true);
		blk_mq_kick_requeue_list(ns->queue);
	}
}

2622
static void nvme_dev_shutdown(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2623
{
2624
	int i;
2625
	u32 csts = -1;
2626

2627
	nvme_dev_list_remove(dev);
2628

2629 2630
	if (dev->bar) {
		nvme_freeze_queues(dev);
2631
		csts = readl(&dev->bar->csts);
2632
	}
2633
	if (csts & NVME_CSTS_CFS || !(csts & NVME_CSTS_RDY)) {
2634
		for (i = dev->queue_count - 1; i >= 0; i--) {
2635
			struct nvme_queue *nvmeq = dev->queues[i];
2636 2637 2638 2639
			nvme_suspend_queue(nvmeq);
		}
	} else {
		nvme_disable_io_queues(dev);
K
Keith Busch 已提交
2640
		nvme_shutdown_ctrl(dev);
2641 2642
		nvme_disable_queue(dev, 0);
	}
2643
	nvme_dev_unmap(dev);
2644 2645 2646

	for (i = dev->queue_count - 1; i >= 0; i--)
		nvme_clear_queue(dev->queues[i]);
2647 2648 2649 2650
}

static void nvme_dev_remove(struct nvme_dev *dev)
{
2651
	struct nvme_ns *ns;
2652

2653
	list_for_each_entry(ns, &dev->namespaces, list) {
2654 2655 2656
		if (ns->disk->flags & GENHD_FL_UP) {
			if (ns->disk->integrity)
				blk_integrity_unregister(ns->disk);
2657
			del_gendisk(ns->disk);
2658
		}
2659 2660
		if (!blk_queue_dying(ns->queue)) {
			blk_mq_abort_requeue_list(ns->queue);
2661
			blk_cleanup_queue(ns->queue);
2662
		}
M
Matthew Wilcox 已提交
2663 2664 2665
	}
}

2666 2667 2668 2669 2670 2671 2672 2673
static int nvme_setup_prp_pools(struct nvme_dev *dev)
{
	struct device *dmadev = &dev->pci_dev->dev;
	dev->prp_page_pool = dma_pool_create("prp list page", dmadev,
						PAGE_SIZE, PAGE_SIZE, 0);
	if (!dev->prp_page_pool)
		return -ENOMEM;

2674 2675 2676 2677 2678 2679 2680
	/* Optimisation for I/Os between 4k and 128k */
	dev->prp_small_pool = dma_pool_create("prp list 256", dmadev,
						256, 256, 0);
	if (!dev->prp_small_pool) {
		dma_pool_destroy(dev->prp_page_pool);
		return -ENOMEM;
	}
2681 2682 2683 2684 2685 2686
	return 0;
}

static void nvme_release_prp_pools(struct nvme_dev *dev)
{
	dma_pool_destroy(dev->prp_page_pool);
2687
	dma_pool_destroy(dev->prp_small_pool);
2688 2689
}

2690 2691 2692
static DEFINE_IDA(nvme_instance_ida);

static int nvme_set_instance(struct nvme_dev *dev)
M
Matthew Wilcox 已提交
2693
{
2694 2695 2696 2697 2698 2699 2700 2701 2702 2703 2704 2705 2706 2707 2708 2709
	int instance, error;

	do {
		if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
			return -ENODEV;

		spin_lock(&dev_list_lock);
		error = ida_get_new(&nvme_instance_ida, &instance);
		spin_unlock(&dev_list_lock);
	} while (error == -EAGAIN);

	if (error)
		return -ENODEV;

	dev->instance = instance;
	return 0;
M
Matthew Wilcox 已提交
2710 2711 2712 2713
}

static void nvme_release_instance(struct nvme_dev *dev)
{
2714 2715 2716
	spin_lock(&dev_list_lock);
	ida_remove(&nvme_instance_ida, dev->instance);
	spin_unlock(&dev_list_lock);
M
Matthew Wilcox 已提交
2717 2718
}

2719 2720 2721 2722 2723 2724
static void nvme_free_namespaces(struct nvme_dev *dev)
{
	struct nvme_ns *ns, *next;

	list_for_each_entry_safe(ns, next, &dev->namespaces, list) {
		list_del(&ns->list);
2725 2726 2727 2728 2729

		spin_lock(&dev_list_lock);
		ns->disk->private_data = NULL;
		spin_unlock(&dev_list_lock);

2730 2731 2732 2733 2734
		put_disk(ns->disk);
		kfree(ns);
	}
}

2735 2736 2737
static void nvme_free_dev(struct kref *kref)
{
	struct nvme_dev *dev = container_of(kref, struct nvme_dev, kref);
2738

2739
	pci_dev_put(dev->pci_dev);
2740
	put_device(dev->device);
2741
	nvme_free_namespaces(dev);
2742
	nvme_release_instance(dev);
2743
	blk_mq_free_tag_set(&dev->tagset);
2744
	blk_put_queue(dev->admin_q);
2745 2746 2747 2748 2749 2750 2751
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
}

static int nvme_dev_open(struct inode *inode, struct file *f)
{
2752 2753 2754 2755 2756 2757 2758
	struct nvme_dev *dev;
	int instance = iminor(inode);
	int ret = -ENODEV;

	spin_lock(&dev_list_lock);
	list_for_each_entry(dev, &dev_list, node) {
		if (dev->instance == instance) {
2759 2760 2761 2762
			if (!dev->admin_q) {
				ret = -EWOULDBLOCK;
				break;
			}
2763 2764 2765 2766 2767 2768 2769 2770 2771 2772
			if (!kref_get_unless_zero(&dev->kref))
				break;
			f->private_data = dev;
			ret = 0;
			break;
		}
	}
	spin_unlock(&dev_list_lock);

	return ret;
2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
}

static int nvme_dev_release(struct inode *inode, struct file *f)
{
	struct nvme_dev *dev = f->private_data;
	kref_put(&dev->kref, nvme_free_dev);
	return 0;
}

static long nvme_dev_ioctl(struct file *f, unsigned int cmd, unsigned long arg)
{
	struct nvme_dev *dev = f->private_data;
2785 2786
	struct nvme_ns *ns;

2787 2788
	switch (cmd) {
	case NVME_IOCTL_ADMIN_CMD:
2789
		return nvme_user_cmd(dev, NULL, (void __user *)arg);
2790
	case NVME_IOCTL_IO_CMD:
2791 2792 2793 2794
		if (list_empty(&dev->namespaces))
			return -ENOTTY;
		ns = list_first_entry(&dev->namespaces, struct nvme_ns, list);
		return nvme_user_cmd(dev, ns, (void __user *)arg);
2795 2796 2797 2798 2799 2800 2801 2802 2803 2804 2805 2806 2807
	default:
		return -ENOTTY;
	}
}

static const struct file_operations nvme_dev_fops = {
	.owner		= THIS_MODULE,
	.open		= nvme_dev_open,
	.release	= nvme_dev_release,
	.unlocked_ioctl	= nvme_dev_ioctl,
	.compat_ioctl	= nvme_dev_ioctl,
};

2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823
static void nvme_set_irq_hints(struct nvme_dev *dev)
{
	struct nvme_queue *nvmeq;
	int i;

	for (i = 0; i < dev->online_queues; i++) {
		nvmeq = dev->queues[i];

		if (!nvmeq->hctx)
			continue;

		irq_set_affinity_hint(dev->entry[nvmeq->cq_vector].vector,
							nvmeq->hctx->cpumask);
	}
}

2824 2825 2826
static int nvme_dev_start(struct nvme_dev *dev)
{
	int result;
2827
	bool start_thread = false;
2828 2829 2830 2831 2832 2833 2834 2835 2836 2837

	result = nvme_dev_map(dev);
	if (result)
		return result;

	result = nvme_configure_admin_queue(dev);
	if (result)
		goto unmap;

	spin_lock(&dev_list_lock);
2838 2839 2840 2841
	if (list_empty(&dev_list) && IS_ERR_OR_NULL(nvme_thread)) {
		start_thread = true;
		nvme_thread = NULL;
	}
2842 2843 2844
	list_add(&dev->node, &dev_list);
	spin_unlock(&dev_list_lock);

2845 2846
	if (start_thread) {
		nvme_thread = kthread_run(nvme_kthread, NULL, "nvme");
2847
		wake_up_all(&nvme_kthread_wait);
2848 2849 2850 2851 2852 2853 2854
	} else
		wait_event_killable(nvme_kthread_wait, nvme_thread);

	if (IS_ERR_OR_NULL(nvme_thread)) {
		result = nvme_thread ? PTR_ERR(nvme_thread) : -EINTR;
		goto disable;
	}
2855 2856

	nvme_init_queue(dev->queues[0], 0);
2857 2858 2859
	result = nvme_alloc_admin_tags(dev);
	if (result)
		goto disable;
2860

2861
	result = nvme_setup_io_queues(dev);
2862
	if (result)
2863
		goto free_tags;
2864

2865 2866
	nvme_set_irq_hints(dev);

2867
	return result;
2868

2869 2870
 free_tags:
	nvme_dev_remove_admin(dev);
2871
 disable:
2872
	nvme_disable_queue(dev, 0);
2873
	nvme_dev_list_remove(dev);
2874 2875 2876 2877 2878
 unmap:
	nvme_dev_unmap(dev);
	return result;
}

2879 2880 2881 2882 2883 2884
static int nvme_remove_dead_ctrl(void *arg)
{
	struct nvme_dev *dev = (struct nvme_dev *)arg;
	struct pci_dev *pdev = dev->pci_dev;

	if (pci_get_drvdata(pdev))
2885
		pci_stop_and_remove_bus_device_locked(pdev);
2886 2887 2888 2889 2890 2891 2892 2893
	kref_put(&dev->kref, nvme_free_dev);
	return 0;
}

static void nvme_remove_disks(struct work_struct *ws)
{
	struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);

2894
	nvme_free_queues(dev, 1);
2895
	nvme_dev_remove(dev);
2896 2897 2898 2899 2900 2901 2902
}

static int nvme_dev_resume(struct nvme_dev *dev)
{
	int ret;

	ret = nvme_dev_start(dev);
2903
	if (ret)
2904
		return ret;
2905
	if (dev->online_queues < 2) {
2906
		spin_lock(&dev_list_lock);
T
Tejun Heo 已提交
2907
		dev->reset_workfn = nvme_remove_disks;
2908 2909
		queue_work(nvme_workq, &dev->reset_work);
		spin_unlock(&dev_list_lock);
2910 2911 2912
	} else {
		nvme_unfreeze_queues(dev);
		nvme_set_irq_hints(dev);
2913 2914 2915 2916 2917 2918 2919 2920
	}
	return 0;
}

static void nvme_dev_reset(struct nvme_dev *dev)
{
	nvme_dev_shutdown(dev);
	if (nvme_dev_resume(dev)) {
2921
		dev_warn(&dev->pci_dev->dev, "Device failed to resume\n");
2922 2923 2924 2925 2926 2927 2928 2929 2930 2931 2932 2933 2934 2935 2936 2937
		kref_get(&dev->kref);
		if (IS_ERR(kthread_run(nvme_remove_dead_ctrl, dev, "nvme%d",
							dev->instance))) {
			dev_err(&dev->pci_dev->dev,
				"Failed to start controller remove task\n");
			kref_put(&dev->kref, nvme_free_dev);
		}
	}
}

static void nvme_reset_failed_dev(struct work_struct *ws)
{
	struct nvme_dev *dev = container_of(ws, struct nvme_dev, reset_work);
	nvme_dev_reset(dev);
}

T
Tejun Heo 已提交
2938 2939 2940 2941 2942 2943
static void nvme_reset_workfn(struct work_struct *work)
{
	struct nvme_dev *dev = container_of(work, struct nvme_dev, reset_work);
	dev->reset_workfn(work);
}

2944
static void nvme_async_probe(struct work_struct *work);
2945
static int nvme_probe(struct pci_dev *pdev, const struct pci_device_id *id)
M
Matthew Wilcox 已提交
2946
{
2947
	int node, result = -ENOMEM;
M
Matthew Wilcox 已提交
2948 2949
	struct nvme_dev *dev;

2950 2951 2952 2953 2954
	node = dev_to_node(&pdev->dev);
	if (node == NUMA_NO_NODE)
		set_dev_node(&pdev->dev, 0);

	dev = kzalloc_node(sizeof(*dev), GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2955 2956
	if (!dev)
		return -ENOMEM;
2957 2958
	dev->entry = kzalloc_node(num_possible_cpus() * sizeof(*dev->entry),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2959 2960
	if (!dev->entry)
		goto free;
2961 2962
	dev->queues = kzalloc_node((num_possible_cpus() + 1) * sizeof(void *),
							GFP_KERNEL, node);
M
Matthew Wilcox 已提交
2963 2964 2965 2966
	if (!dev->queues)
		goto free;

	INIT_LIST_HEAD(&dev->namespaces);
T
Tejun Heo 已提交
2967 2968
	dev->reset_workfn = nvme_reset_failed_dev;
	INIT_WORK(&dev->reset_work, nvme_reset_workfn);
2969
	dev->pci_dev = pci_dev_get(pdev);
2970
	pci_set_drvdata(pdev, dev);
2971 2972
	result = nvme_set_instance(dev);
	if (result)
2973
		goto put_pci;
M
Matthew Wilcox 已提交
2974

2975 2976
	result = nvme_setup_prp_pools(dev);
	if (result)
2977
		goto release;
2978

2979
	kref_init(&dev->kref);
2980 2981 2982 2983 2984
	dev->device = device_create(nvme_class, &pdev->dev,
				MKDEV(nvme_char_major, dev->instance),
				dev, "nvme%d", dev->instance);
	if (IS_ERR(dev->device)) {
		result = PTR_ERR(dev->device);
2985
		goto release_pools;
2986 2987
	}
	get_device(dev->device);
2988

2989 2990
	INIT_WORK(&dev->probe_work, nvme_async_probe);
	schedule_work(&dev->probe_work);
M
Matthew Wilcox 已提交
2991 2992
	return 0;

2993
 release_pools:
2994
	nvme_release_prp_pools(dev);
2995 2996
 release:
	nvme_release_instance(dev);
2997 2998
 put_pci:
	pci_dev_put(dev->pci_dev);
M
Matthew Wilcox 已提交
2999 3000 3001 3002 3003 3004 3005
 free:
	kfree(dev->queues);
	kfree(dev->entry);
	kfree(dev);
	return result;
}

3006 3007 3008 3009 3010 3011 3012 3013 3014 3015 3016 3017 3018 3019 3020 3021 3022
static void nvme_async_probe(struct work_struct *work)
{
	struct nvme_dev *dev = container_of(work, struct nvme_dev, probe_work);
	int result;

	result = nvme_dev_start(dev);
	if (result)
		goto reset;

	if (dev->online_queues > 1)
		result = nvme_dev_add(dev);
	if (result)
		goto reset;

	nvme_set_irq_hints(dev);
	return;
 reset:
3023 3024 3025 3026
	if (!work_busy(&dev->reset_work)) {
		dev->reset_workfn = nvme_reset_failed_dev;
		queue_work(nvme_workq, &dev->reset_work);
	}
3027 3028
}

3029 3030
static void nvme_reset_notify(struct pci_dev *pdev, bool prepare)
{
K
Keith Busch 已提交
3031
	struct nvme_dev *dev = pci_get_drvdata(pdev);
3032

K
Keith Busch 已提交
3033 3034 3035 3036
	if (prepare)
		nvme_dev_shutdown(dev);
	else
		nvme_dev_resume(dev);
3037 3038
}

3039 3040 3041 3042 3043 3044
static void nvme_shutdown(struct pci_dev *pdev)
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
	nvme_dev_shutdown(dev);
}

3045
static void nvme_remove(struct pci_dev *pdev)
M
Matthew Wilcox 已提交
3046 3047
{
	struct nvme_dev *dev = pci_get_drvdata(pdev);
3048 3049 3050 3051 3052 3053

	spin_lock(&dev_list_lock);
	list_del_init(&dev->node);
	spin_unlock(&dev_list_lock);

	pci_set_drvdata(pdev, NULL);
3054
	flush_work(&dev->probe_work);
3055 3056
	flush_work(&dev->reset_work);
	nvme_dev_shutdown(dev);
3057
	nvme_dev_remove(dev);
3058
	nvme_dev_remove_admin(dev);
3059
	device_destroy(nvme_class, MKDEV(nvme_char_major, dev->instance));
3060
	nvme_free_queues(dev, 0);
3061
	nvme_release_prp_pools(dev);
3062
	kref_put(&dev->kref, nvme_free_dev);
M
Matthew Wilcox 已提交
3063 3064 3065 3066 3067 3068 3069 3070
}

/* These functions are yet to be implemented */
#define nvme_error_detected NULL
#define nvme_dump_registers NULL
#define nvme_link_reset NULL
#define nvme_slot_reset NULL
#define nvme_error_resume NULL
3071

3072
#ifdef CONFIG_PM_SLEEP
3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086
static int nvme_suspend(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

	nvme_dev_shutdown(ndev);
	return 0;
}

static int nvme_resume(struct device *dev)
{
	struct pci_dev *pdev = to_pci_dev(dev);
	struct nvme_dev *ndev = pci_get_drvdata(pdev);

3087
	if (nvme_dev_resume(ndev) && !work_busy(&ndev->reset_work)) {
T
Tejun Heo 已提交
3088
		ndev->reset_workfn = nvme_reset_failed_dev;
3089 3090 3091
		queue_work(nvme_workq, &ndev->reset_work);
	}
	return 0;
3092
}
3093
#endif
3094 3095

static SIMPLE_DEV_PM_OPS(nvme_dev_pm_ops, nvme_suspend, nvme_resume);
M
Matthew Wilcox 已提交
3096

3097
static const struct pci_error_handlers nvme_err_handler = {
M
Matthew Wilcox 已提交
3098 3099 3100 3101 3102
	.error_detected	= nvme_error_detected,
	.mmio_enabled	= nvme_dump_registers,
	.link_reset	= nvme_link_reset,
	.slot_reset	= nvme_slot_reset,
	.resume		= nvme_error_resume,
3103
	.reset_notify	= nvme_reset_notify,
M
Matthew Wilcox 已提交
3104 3105 3106 3107 3108
};

/* Move to pci_ids.h later */
#define PCI_CLASS_STORAGE_EXPRESS	0x010802

3109
static const struct pci_device_id nvme_id_table[] = {
M
Matthew Wilcox 已提交
3110 3111 3112 3113 3114 3115 3116 3117 3118
	{ PCI_DEVICE_CLASS(PCI_CLASS_STORAGE_EXPRESS, 0xffffff) },
	{ 0, }
};
MODULE_DEVICE_TABLE(pci, nvme_id_table);

static struct pci_driver nvme_driver = {
	.name		= "nvme",
	.id_table	= nvme_id_table,
	.probe		= nvme_probe,
3119
	.remove		= nvme_remove,
3120
	.shutdown	= nvme_shutdown,
3121 3122 3123
	.driver		= {
		.pm	= &nvme_dev_pm_ops,
	},
M
Matthew Wilcox 已提交
3124 3125 3126 3127 3128
	.err_handler	= &nvme_err_handler,
};

static int __init nvme_init(void)
{
3129
	int result;
3130

3131
	init_waitqueue_head(&nvme_kthread_wait);
M
Matthew Wilcox 已提交
3132

3133 3134
	nvme_workq = create_singlethread_workqueue("nvme");
	if (!nvme_workq)
3135
		return -ENOMEM;
3136

3137 3138
	result = register_blkdev(nvme_major, "nvme");
	if (result < 0)
3139
		goto kill_workq;
3140
	else if (result > 0)
3141
		nvme_major = result;
M
Matthew Wilcox 已提交
3142

3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153
	result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
							&nvme_dev_fops);
	if (result < 0)
		goto unregister_blkdev;
	else if (result > 0)
		nvme_char_major = result;

	nvme_class = class_create(THIS_MODULE, "nvme");
	if (!nvme_class)
		goto unregister_chrdev;

3154 3155
	result = pci_register_driver(&nvme_driver);
	if (result)
3156
		goto destroy_class;
3157
	return 0;
M
Matthew Wilcox 已提交
3158

3159 3160 3161 3162
 destroy_class:
	class_destroy(nvme_class);
 unregister_chrdev:
	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
3163
 unregister_blkdev:
M
Matthew Wilcox 已提交
3164
	unregister_blkdev(nvme_major, "nvme");
3165 3166
 kill_workq:
	destroy_workqueue(nvme_workq);
M
Matthew Wilcox 已提交
3167 3168 3169 3170 3171 3172 3173
	return result;
}

static void __exit nvme_exit(void)
{
	pci_unregister_driver(&nvme_driver);
	unregister_blkdev(nvme_major, "nvme");
3174
	destroy_workqueue(nvme_workq);
3175 3176
	class_destroy(nvme_class);
	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
3177
	BUG_ON(nvme_thread && !IS_ERR(nvme_thread));
3178
	_nvme_check_size();
M
Matthew Wilcox 已提交
3179 3180 3181 3182
}

MODULE_AUTHOR("Matthew Wilcox <willy@linux.intel.com>");
MODULE_LICENSE("GPL");
3183
MODULE_VERSION("1.0");
M
Matthew Wilcox 已提交
3184 3185
module_init(nvme_init);
module_exit(nvme_exit);