core.c 52.5 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * NVM Express device driver
 * Copyright (c) 2011-2014, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/blkdev.h>
#include <linux/blk-mq.h>
17
#include <linux/delay.h>
18
#include <linux/errno.h>
19
#include <linux/hdreg.h>
20
#include <linux/kernel.h>
21 22
#include <linux/module.h>
#include <linux/list_sort.h>
23 24
#include <linux/slab.h>
#include <linux/types.h>
25 26 27 28 29 30
#include <linux/pr.h>
#include <linux/ptrace.h>
#include <linux/nvme_ioctl.h>
#include <linux/t10-pi.h>
#include <scsi/sg.h>
#include <asm/unaligned.h>
31 32

#include "nvme.h"
S
Sagi Grimberg 已提交
33
#include "fabrics.h"
34

35 36
#define NVME_MINORS		(1U << MINORBITS)

37 38 39
unsigned char admin_timeout = 60;
module_param(admin_timeout, byte, 0644);
MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
40
EXPORT_SYMBOL_GPL(admin_timeout);
41 42 43 44

unsigned char nvme_io_timeout = 30;
module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
45
EXPORT_SYMBOL_GPL(nvme_io_timeout);
46 47 48 49 50

unsigned char shutdown_timeout = 5;
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");

K
Keith Busch 已提交
51 52 53 54
unsigned int nvme_max_retries = 5;
module_param_named(max_retries, nvme_max_retries, uint, 0644);
MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
EXPORT_SYMBOL_GPL(nvme_max_retries);
55

56 57 58 59
static int nvme_char_major;
module_param(nvme_char_major, int, 0);

static LIST_HEAD(nvme_ctrl_list);
M
Ming Lin 已提交
60
static DEFINE_SPINLOCK(dev_list_lock);
61

62 63
static struct class *nvme_class;

64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80
void nvme_cancel_request(struct request *req, void *data, bool reserved)
{
	int status;

	if (!blk_mq_request_started(req))
		return;

	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
				"Cancelling I/O %d", req->tag);

	status = NVME_SC_ABORT_REQ;
	if (blk_queue_dying(req->q))
		status |= NVME_SC_DNR;
	blk_mq_complete_request(req, status);
}
EXPORT_SYMBOL_GPL(nvme_cancel_request);

81 82 83
bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
		enum nvme_ctrl_state new_state)
{
84
	enum nvme_ctrl_state old_state;
85 86 87
	bool changed = false;

	spin_lock_irq(&ctrl->lock);
88 89

	old_state = ctrl->state;
90 91 92
	switch (new_state) {
	case NVME_CTRL_LIVE:
		switch (old_state) {
93
		case NVME_CTRL_NEW:
94
		case NVME_CTRL_RESETTING:
95
		case NVME_CTRL_RECONNECTING:
96 97 98 99 100 101 102 103 104
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
	case NVME_CTRL_RESETTING:
		switch (old_state) {
		case NVME_CTRL_NEW:
105 106 107 108 109 110 111 112 113 114
		case NVME_CTRL_LIVE:
		case NVME_CTRL_RECONNECTING:
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
	case NVME_CTRL_RECONNECTING:
		switch (old_state) {
115 116 117 118 119 120 121 122 123 124 125
		case NVME_CTRL_LIVE:
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
	case NVME_CTRL_DELETING:
		switch (old_state) {
		case NVME_CTRL_LIVE:
		case NVME_CTRL_RESETTING:
126
		case NVME_CTRL_RECONNECTING:
127 128 129 130 131 132
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
133 134 135 136 137 138 139 140 141
	case NVME_CTRL_DEAD:
		switch (old_state) {
		case NVME_CTRL_DELETING:
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
142 143 144 145 146 147 148
	default:
		break;
	}

	if (changed)
		ctrl->state = new_state;

149 150
	spin_unlock_irq(&ctrl->lock);

151 152 153 154
	return changed;
}
EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);

155 156 157 158
static void nvme_free_ns(struct kref *kref)
{
	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);

159 160
	if (ns->ndev)
		nvme_nvm_unregister(ns);
161

162 163 164 165 166
	if (ns->disk) {
		spin_lock(&dev_list_lock);
		ns->disk->private_data = NULL;
		spin_unlock(&dev_list_lock);
	}
167 168

	put_disk(ns->disk);
169 170
	ida_simple_remove(&ns->ctrl->ns_ida, ns->instance);
	nvme_put_ctrl(ns->ctrl);
171 172 173
	kfree(ns);
}

174
static void nvme_put_ns(struct nvme_ns *ns)
175 176 177 178 179 180 181 182 183 184
{
	kref_put(&ns->kref, nvme_free_ns);
}

static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk)
{
	struct nvme_ns *ns;

	spin_lock(&dev_list_lock);
	ns = disk->private_data;
185 186 187 188 189 190
	if (ns) {
		if (!kref_get_unless_zero(&ns->kref))
			goto fail;
		if (!try_module_get(ns->ctrl->ops->module))
			goto fail_put_ns;
	}
191 192 193
	spin_unlock(&dev_list_lock);

	return ns;
194 195 196 197 198 199

fail_put_ns:
	kref_put(&ns->kref, nvme_free_ns);
fail:
	spin_unlock(&dev_list_lock);
	return NULL;
200 201
}

202 203
void nvme_requeue_req(struct request *req)
{
204
	blk_mq_requeue_request(req, !blk_mq_queue_stopped(req->q));
205
}
206
EXPORT_SYMBOL_GPL(nvme_requeue_req);
207

208
struct request *nvme_alloc_request(struct request_queue *q,
209
		struct nvme_command *cmd, unsigned int flags, int qid)
210 211 212
{
	struct request *req;

213 214 215 216 217 218
	if (qid == NVME_QID_ANY) {
		req = blk_mq_alloc_request(q, nvme_is_write(cmd), flags);
	} else {
		req = blk_mq_alloc_request_hctx(q, nvme_is_write(cmd), flags,
				qid ? qid - 1 : 0);
	}
219
	if (IS_ERR(req))
220
		return req;
221 222 223

	req->cmd_type = REQ_TYPE_DRV_PRIV;
	req->cmd_flags |= REQ_FAILFAST_DRIVER;
224
	nvme_req(req)->cmd = cmd;
225

226 227
	return req;
}
228
EXPORT_SYMBOL_GPL(nvme_alloc_request);
229

M
Ming Lin 已提交
230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257
static inline void nvme_setup_flush(struct nvme_ns *ns,
		struct nvme_command *cmnd)
{
	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->common.opcode = nvme_cmd_flush;
	cmnd->common.nsid = cpu_to_le32(ns->ns_id);
}

static inline int nvme_setup_discard(struct nvme_ns *ns, struct request *req,
		struct nvme_command *cmnd)
{
	struct nvme_dsm_range *range;
	unsigned int nr_bytes = blk_rq_bytes(req);

	range = kmalloc(sizeof(*range), GFP_ATOMIC);
	if (!range)
		return BLK_MQ_RQ_QUEUE_BUSY;

	range->cattr = cpu_to_le32(0);
	range->nlb = cpu_to_le32(nr_bytes >> ns->lba_shift);
	range->slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));

	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->dsm.opcode = nvme_cmd_dsm;
	cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
	cmnd->dsm.nr = 0;
	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);

258 259 260 261
	req->special_vec.bv_page = virt_to_page(range);
	req->special_vec.bv_offset = offset_in_page(range);
	req->special_vec.bv_len = sizeof(*range);
	req->rq_flags |= RQF_SPECIAL_PAYLOAD;
M
Ming Lin 已提交
262

263
	return BLK_MQ_RQ_QUEUE_OK;
M
Ming Lin 已提交
264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309
}

static inline void nvme_setup_rw(struct nvme_ns *ns, struct request *req,
		struct nvme_command *cmnd)
{
	u16 control = 0;
	u32 dsmgmt = 0;

	if (req->cmd_flags & REQ_FUA)
		control |= NVME_RW_FUA;
	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
		control |= NVME_RW_LR;

	if (req->cmd_flags & REQ_RAHEAD)
		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;

	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
	cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
	cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);

	if (ns->ms) {
		switch (ns->pi_type) {
		case NVME_NS_DPS_PI_TYPE3:
			control |= NVME_RW_PRINFO_PRCHK_GUARD;
			break;
		case NVME_NS_DPS_PI_TYPE1:
		case NVME_NS_DPS_PI_TYPE2:
			control |= NVME_RW_PRINFO_PRCHK_GUARD |
					NVME_RW_PRINFO_PRCHK_REF;
			cmnd->rw.reftag = cpu_to_le32(
					nvme_block_nr(ns, blk_rq_pos(req)));
			break;
		}
		if (!blk_integrity_rq(req))
			control |= NVME_RW_PRINFO_PRACT;
	}

	cmnd->rw.control = cpu_to_le16(control);
	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
}

int nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
		struct nvme_command *cmd)
{
310
	int ret = BLK_MQ_RQ_QUEUE_OK;
M
Ming Lin 已提交
311 312

	if (req->cmd_type == REQ_TYPE_DRV_PRIV)
313
		memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
314
	else if (req_op(req) == REQ_OP_FLUSH)
M
Ming Lin 已提交
315
		nvme_setup_flush(ns, cmd);
M
Mike Christie 已提交
316
	else if (req_op(req) == REQ_OP_DISCARD)
M
Ming Lin 已提交
317 318 319 320
		ret = nvme_setup_discard(ns, req, cmd);
	else
		nvme_setup_rw(ns, req, cmd);

321 322
	cmd->common.command_id = req->tag;

M
Ming Lin 已提交
323 324 325 326
	return ret;
}
EXPORT_SYMBOL_GPL(nvme_setup_cmd);

327 328 329 330 331
/*
 * Returns 0 on success.  If the result is negative, it's a Linux error code;
 * if the result is positive, it's an NVM Express status code
 */
int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
332
		union nvme_result *result, void *buffer, unsigned bufflen,
333
		unsigned timeout, int qid, int at_head, int flags)
334 335 336 337
{
	struct request *req;
	int ret;

338
	req = nvme_alloc_request(q, cmd, flags, qid);
339 340 341 342 343
	if (IS_ERR(req))
		return PTR_ERR(req);

	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;

344 345 346 347
	if (buffer && bufflen) {
		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
		if (ret)
			goto out;
348 349
	}

350
	blk_execute_rq(req->q, NULL, req, at_head);
351 352
	if (result)
		*result = nvme_req(req)->result;
353 354 355 356 357
	ret = req->errors;
 out:
	blk_mq_free_request(req);
	return ret;
}
358
EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
359 360 361 362

int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
		void *buffer, unsigned bufflen)
{
363 364
	return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
			NVME_QID_ANY, 0, 0);
365
}
366
EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
367

368 369 370 371
int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
		void __user *ubuffer, unsigned bufflen,
		void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
		u32 *result, unsigned timeout)
372
{
373
	bool write = nvme_is_write(cmd);
374 375
	struct nvme_ns *ns = q->queuedata;
	struct gendisk *disk = ns ? ns->disk : NULL;
376
	struct request *req;
377 378
	struct bio *bio = NULL;
	void *meta = NULL;
379 380
	int ret;

381
	req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
382 383 384 385 386 387
	if (IS_ERR(req))
		return PTR_ERR(req);

	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;

	if (ubuffer && bufflen) {
388 389 390 391 392 393
		ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
				GFP_KERNEL);
		if (ret)
			goto out;
		bio = req->bio;

394 395 396 397 398 399 400 401
		if (!disk)
			goto submit;
		bio->bi_bdev = bdget_disk(disk, 0);
		if (!bio->bi_bdev) {
			ret = -ENODEV;
			goto out_unmap;
		}

402
		if (meta_buffer && meta_len) {
403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419
			struct bio_integrity_payload *bip;

			meta = kmalloc(meta_len, GFP_KERNEL);
			if (!meta) {
				ret = -ENOMEM;
				goto out_unmap;
			}

			if (write) {
				if (copy_from_user(meta, meta_buffer,
						meta_len)) {
					ret = -EFAULT;
					goto out_free_meta;
				}
			}

			bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
420 421
			if (IS_ERR(bip)) {
				ret = PTR_ERR(bip);
422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438
				goto out_free_meta;
			}

			bip->bip_iter.bi_size = meta_len;
			bip->bip_iter.bi_sector = meta_seed;

			ret = bio_integrity_add_page(bio, virt_to_page(meta),
					meta_len, offset_in_page(meta));
			if (ret != meta_len) {
				ret = -ENOMEM;
				goto out_free_meta;
			}
		}
	}
 submit:
	blk_execute_rq(req->q, disk, req, 0);
	ret = req->errors;
439
	if (result)
440
		*result = le32_to_cpu(nvme_req(req)->result.u32);
441 442 443 444 445 446 447 448 449 450 451 452
	if (meta && !ret && !write) {
		if (copy_to_user(meta_buffer, meta, meta_len))
			ret = -EFAULT;
	}
 out_free_meta:
	kfree(meta);
 out_unmap:
	if (bio) {
		if (disk && bio->bi_bdev)
			bdput(bio->bi_bdev);
		blk_rq_unmap_user(bio);
	}
453 454 455 456 457
 out:
	blk_mq_free_request(req);
	return ret;
}

458 459 460 461 462 463 464 465
int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
		void __user *ubuffer, unsigned bufflen, u32 *result,
		unsigned timeout)
{
	return __nvme_submit_user_cmd(q, cmd, ubuffer, bufflen, NULL, 0, 0,
			result, timeout);
}

S
Sagi Grimberg 已提交
466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533
static void nvme_keep_alive_end_io(struct request *rq, int error)
{
	struct nvme_ctrl *ctrl = rq->end_io_data;

	blk_mq_free_request(rq);

	if (error) {
		dev_err(ctrl->device,
			"failed nvme_keep_alive_end_io error=%d\n", error);
		return;
	}

	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
}

static int nvme_keep_alive(struct nvme_ctrl *ctrl)
{
	struct nvme_command c;
	struct request *rq;

	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_keep_alive;

	rq = nvme_alloc_request(ctrl->admin_q, &c, BLK_MQ_REQ_RESERVED,
			NVME_QID_ANY);
	if (IS_ERR(rq))
		return PTR_ERR(rq);

	rq->timeout = ctrl->kato * HZ;
	rq->end_io_data = ctrl;

	blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);

	return 0;
}

static void nvme_keep_alive_work(struct work_struct *work)
{
	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
			struct nvme_ctrl, ka_work);

	if (nvme_keep_alive(ctrl)) {
		/* allocation failure, reset the controller */
		dev_err(ctrl->device, "keep-alive failed\n");
		ctrl->ops->reset_ctrl(ctrl);
		return;
	}
}

void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
{
	if (unlikely(ctrl->kato == 0))
		return;

	INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
}
EXPORT_SYMBOL_GPL(nvme_start_keep_alive);

void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
{
	if (unlikely(ctrl->kato == 0))
		return;

	cancel_delayed_work_sync(&ctrl->ka_work);
}
EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);

534
int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
535 536 537 538 539 540
{
	struct nvme_command c = { };
	int error;

	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
	c.identify.opcode = nvme_admin_identify;
541
	c.identify.cns = cpu_to_le32(NVME_ID_CNS_CTRL);
542 543 544 545 546 547 548 549 550 551 552 553

	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
	if (!*id)
		return -ENOMEM;

	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
			sizeof(struct nvme_id_ctrl));
	if (error)
		kfree(*id);
	return error;
}

554 555 556 557 558
static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
{
	struct nvme_command c = { };

	c.identify.opcode = nvme_admin_identify;
559
	c.identify.cns = cpu_to_le32(NVME_ID_CNS_NS_ACTIVE_LIST);
560 561 562 563
	c.identify.nsid = cpu_to_le32(nsid);
	return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list, 0x1000);
}

564
int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584
		struct nvme_id_ns **id)
{
	struct nvme_command c = { };
	int error;

	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
	c.identify.opcode = nvme_admin_identify,
	c.identify.nsid = cpu_to_le32(nsid),

	*id = kmalloc(sizeof(struct nvme_id_ns), GFP_KERNEL);
	if (!*id)
		return -ENOMEM;

	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
			sizeof(struct nvme_id_ns));
	if (error)
		kfree(*id);
	return error;
}

585
int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
586
		      void *buffer, size_t buflen, u32 *result)
587 588
{
	struct nvme_command c;
589
	union nvme_result res;
590
	int ret;
591 592 593 594 595 596

	memset(&c, 0, sizeof(c));
	c.features.opcode = nvme_admin_get_features;
	c.features.nsid = cpu_to_le32(nsid);
	c.features.fid = cpu_to_le32(fid);

597
	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res, buffer, buflen, 0,
598
			NVME_QID_ANY, 0, 0);
599
	if (ret >= 0 && result)
600
		*result = le32_to_cpu(res.u32);
601
	return ret;
602 603
}

604
int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
605
		      void *buffer, size_t buflen, u32 *result)
606 607
{
	struct nvme_command c;
608
	union nvme_result res;
609
	int ret;
610 611 612 613 614 615

	memset(&c, 0, sizeof(c));
	c.features.opcode = nvme_admin_set_features;
	c.features.fid = cpu_to_le32(fid);
	c.features.dword11 = cpu_to_le32(dword11);

616
	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
617
			buffer, buflen, 0, NVME_QID_ANY, 0, 0);
618
	if (ret >= 0 && result)
619
		*result = le32_to_cpu(res.u32);
620
	return ret;
621 622
}

623
int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log)
624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643
{
	struct nvme_command c = { };
	int error;

	c.common.opcode = nvme_admin_get_log_page,
	c.common.nsid = cpu_to_le32(0xFFFFFFFF),
	c.common.cdw10[0] = cpu_to_le32(
			(((sizeof(struct nvme_smart_log) / 4) - 1) << 16) |
			 NVME_LOG_SMART),

	*log = kmalloc(sizeof(struct nvme_smart_log), GFP_KERNEL);
	if (!*log)
		return -ENOMEM;

	error = nvme_submit_sync_cmd(dev->admin_q, &c, *log,
			sizeof(struct nvme_smart_log));
	if (error)
		kfree(*log);
	return error;
}
644

C
Christoph Hellwig 已提交
645 646 647 648 649 650
int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
{
	u32 q_count = (*count - 1) | ((*count - 1) << 16);
	u32 result;
	int status, nr_io_queues;

651
	status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
C
Christoph Hellwig 已提交
652
			&result);
653
	if (status < 0)
C
Christoph Hellwig 已提交
654 655
		return status;

656 657 658 659 660 661 662 663 664 665 666 667 668
	/*
	 * Degraded controllers might return an error when setting the queue
	 * count.  We still want to be able to bring them online and offer
	 * access to the admin queue, as that might be only way to fix them up.
	 */
	if (status > 0) {
		dev_err(ctrl->dev, "Could not set queue count (%d)\n", status);
		*count = 0;
	} else {
		nr_io_queues = min(result & 0xffff, result >> 16) + 1;
		*count = min(*count, nr_io_queues);
	}

C
Christoph Hellwig 已提交
669 670
	return 0;
}
671
EXPORT_SYMBOL_GPL(nvme_set_queue_count);
C
Christoph Hellwig 已提交
672

673 674 675 676 677 678 679 680 681
static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
{
	struct nvme_user_io io;
	struct nvme_command c;
	unsigned length, meta_len;
	void __user *metadata;

	if (copy_from_user(&io, uio, sizeof(io)))
		return -EFAULT;
682 683
	if (io.flags)
		return -EINVAL;
684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722

	switch (io.opcode) {
	case nvme_cmd_write:
	case nvme_cmd_read:
	case nvme_cmd_compare:
		break;
	default:
		return -EINVAL;
	}

	length = (io.nblocks + 1) << ns->lba_shift;
	meta_len = (io.nblocks + 1) * ns->ms;
	metadata = (void __user *)(uintptr_t)io.metadata;

	if (ns->ext) {
		length += meta_len;
		meta_len = 0;
	} else if (meta_len) {
		if ((io.metadata & 3) || !io.metadata)
			return -EINVAL;
	}

	memset(&c, 0, sizeof(c));
	c.rw.opcode = io.opcode;
	c.rw.flags = io.flags;
	c.rw.nsid = cpu_to_le32(ns->ns_id);
	c.rw.slba = cpu_to_le64(io.slba);
	c.rw.length = cpu_to_le16(io.nblocks);
	c.rw.control = cpu_to_le16(io.control);
	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
	c.rw.reftag = cpu_to_le32(io.reftag);
	c.rw.apptag = cpu_to_le16(io.apptag);
	c.rw.appmask = cpu_to_le16(io.appmask);

	return __nvme_submit_user_cmd(ns->queue, &c,
			(void __user *)(uintptr_t)io.addr, length,
			metadata, meta_len, io.slba, NULL, 0);
}

723
static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
724 725 726 727 728 729 730 731 732 733 734
			struct nvme_passthru_cmd __user *ucmd)
{
	struct nvme_passthru_cmd cmd;
	struct nvme_command c;
	unsigned timeout = 0;
	int status;

	if (!capable(CAP_SYS_ADMIN))
		return -EACCES;
	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
		return -EFAULT;
735 736
	if (cmd.flags)
		return -EINVAL;
737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754

	memset(&c, 0, sizeof(c));
	c.common.opcode = cmd.opcode;
	c.common.flags = cmd.flags;
	c.common.nsid = cpu_to_le32(cmd.nsid);
	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
	c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
	c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
	c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
	c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
	c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
	c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);

	if (cmd.timeout_ms)
		timeout = msecs_to_jiffies(cmd.timeout_ms);

	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
755
			(void __user *)(uintptr_t)cmd.addr, cmd.data_len,
756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779
			&cmd.result, timeout);
	if (status >= 0) {
		if (put_user(cmd.result, &ucmd->result))
			return -EFAULT;
	}

	return status;
}

static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
		unsigned int cmd, unsigned long arg)
{
	struct nvme_ns *ns = bdev->bd_disk->private_data;

	switch (cmd) {
	case NVME_IOCTL_ID:
		force_successful_syscall_return();
		return ns->ns_id;
	case NVME_IOCTL_ADMIN_CMD:
		return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
	case NVME_IOCTL_IO_CMD:
		return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
	case NVME_IOCTL_SUBMIT_IO:
		return nvme_submit_io(ns, (void __user *)arg);
780
#ifdef CONFIG_BLK_DEV_NVME_SCSI
781 782 783 784
	case SG_GET_VERSION_NUM:
		return nvme_sg_get_version_num((void __user *)arg);
	case SG_IO:
		return nvme_sg_io(ns, (void __user *)arg);
785
#endif
786
	default:
787 788 789 790
#ifdef CONFIG_NVM
		if (ns->ndev)
			return nvme_nvm_ioctl(ns, cmd, arg);
#endif
791
		if (is_sed_ioctl(cmd))
792
			return sed_ioctl(ns->ctrl->opal_dev, cmd,
793
					 (void __user *) arg);
794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818
		return -ENOTTY;
	}
}

#ifdef CONFIG_COMPAT
static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
			unsigned int cmd, unsigned long arg)
{
	switch (cmd) {
	case SG_IO:
		return -ENOIOCTLCMD;
	}
	return nvme_ioctl(bdev, mode, cmd, arg);
}
#else
#define nvme_compat_ioctl	NULL
#endif

static int nvme_open(struct block_device *bdev, fmode_t mode)
{
	return nvme_get_ns_from_disk(bdev->bd_disk) ? 0 : -ENXIO;
}

static void nvme_release(struct gendisk *disk, fmode_t mode)
{
819 820 821 822
	struct nvme_ns *ns = disk->private_data;

	module_put(ns->ctrl->ops->module);
	nvme_put_ns(ns);
823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838
}

static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	/* some standard values */
	geo->heads = 1 << 6;
	geo->sectors = 1 << 5;
	geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
	return 0;
}

#ifdef CONFIG_BLK_DEV_INTEGRITY
static void nvme_init_integrity(struct nvme_ns *ns)
{
	struct blk_integrity integrity;

839
	memset(&integrity, 0, sizeof(integrity));
840 841 842
	switch (ns->pi_type) {
	case NVME_NS_DPS_PI_TYPE3:
		integrity.profile = &t10_pi_type3_crc;
843 844
		integrity.tag_size = sizeof(u16) + sizeof(u32);
		integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
845 846 847 848
		break;
	case NVME_NS_DPS_PI_TYPE1:
	case NVME_NS_DPS_PI_TYPE2:
		integrity.profile = &t10_pi_type1_crc;
849 850
		integrity.tag_size = sizeof(u16);
		integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867
		break;
	default:
		integrity.profile = NULL;
		break;
	}
	integrity.tuple_size = ns->ms;
	blk_integrity_register(ns->disk, &integrity);
	blk_queue_max_integrity_segments(ns->queue, 1);
}
#else
static void nvme_init_integrity(struct nvme_ns *ns)
{
}
#endif /* CONFIG_BLK_DEV_INTEGRITY */

static void nvme_config_discard(struct nvme_ns *ns)
{
868
	struct nvme_ctrl *ctrl = ns->ctrl;
869
	u32 logical_block_size = queue_logical_block_size(ns->queue);
870 871 872 873 874 875

	if (ctrl->quirks & NVME_QUIRK_DISCARD_ZEROES)
		ns->queue->limits.discard_zeroes_data = 1;
	else
		ns->queue->limits.discard_zeroes_data = 0;

876 877
	ns->queue->limits.discard_alignment = logical_block_size;
	ns->queue->limits.discard_granularity = logical_block_size;
878
	blk_queue_max_discard_sectors(ns->queue, UINT_MAX);
879 880 881
	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
}

882
static int nvme_revalidate_ns(struct nvme_ns *ns, struct nvme_id_ns **id)
883
{
884
	if (nvme_identify_ns(ns->ctrl, ns->ns_id, id)) {
885
		dev_warn(ns->ctrl->dev, "%s: Identify failure\n", __func__);
886 887 888
		return -ENODEV;
	}

889 890 891
	if ((*id)->ncap == 0) {
		kfree(*id);
		return -ENODEV;
892 893
	}

894
	if (ns->ctrl->vs >= NVME_VS(1, 1, 0))
895
		memcpy(ns->eui, (*id)->eui64, sizeof(ns->eui));
896
	if (ns->ctrl->vs >= NVME_VS(1, 2, 0))
897 898 899 900 901 902 903 904 905 906 907
		memcpy(ns->uuid, (*id)->nguid, sizeof(ns->uuid));

	return 0;
}

static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
{
	struct nvme_ns *ns = disk->private_data;
	u8 lbaf, pi_type;
	u16 old_ms;
	unsigned short bs;
908

909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935
	old_ms = ns->ms;
	lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
	ns->lba_shift = id->lbaf[lbaf].ds;
	ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);

	/*
	 * If identify namespace failed, use default 512 byte block size so
	 * block layer can use before failing read/write for 0 capacity.
	 */
	if (ns->lba_shift == 0)
		ns->lba_shift = 9;
	bs = 1 << ns->lba_shift;
	/* XXX: PI implementation requires metadata equal t10 pi tuple size */
	pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
					id->dps & NVME_NS_DPS_PI_MASK : 0;

	blk_mq_freeze_queue(disk->queue);
	if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
				ns->ms != old_ms ||
				bs != queue_logical_block_size(disk->queue) ||
				(ns->ms && ns->ext)))
		blk_integrity_unregister(disk);

	ns->pi_type = pi_type;
	blk_queue_logical_block_size(ns->queue, bs);

K
Keith Busch 已提交
936
	if (ns->ms && !blk_get_integrity(disk) && !ns->ext)
937 938 939 940 941 942 943 944 945
		nvme_init_integrity(ns);
	if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk))
		set_capacity(disk, 0);
	else
		set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));

	if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
		nvme_config_discard(ns);
	blk_mq_unfreeze_queue(disk->queue);
946
}
947

948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963
static int nvme_revalidate_disk(struct gendisk *disk)
{
	struct nvme_ns *ns = disk->private_data;
	struct nvme_id_ns *id = NULL;
	int ret;

	if (test_bit(NVME_NS_DEAD, &ns->flags)) {
		set_capacity(disk, 0);
		return -ENODEV;
	}

	ret = nvme_revalidate_ns(ns, &id);
	if (ret)
		return ret;

	__nvme_revalidate_disk(disk, id);
964
	kfree(id);
965

966 967 968 969 970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996 997 998 999 1000 1001 1002 1003 1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042
	return 0;
}

static char nvme_pr_type(enum pr_type type)
{
	switch (type) {
	case PR_WRITE_EXCLUSIVE:
		return 1;
	case PR_EXCLUSIVE_ACCESS:
		return 2;
	case PR_WRITE_EXCLUSIVE_REG_ONLY:
		return 3;
	case PR_EXCLUSIVE_ACCESS_REG_ONLY:
		return 4;
	case PR_WRITE_EXCLUSIVE_ALL_REGS:
		return 5;
	case PR_EXCLUSIVE_ACCESS_ALL_REGS:
		return 6;
	default:
		return 0;
	}
};

static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
				u64 key, u64 sa_key, u8 op)
{
	struct nvme_ns *ns = bdev->bd_disk->private_data;
	struct nvme_command c;
	u8 data[16] = { 0, };

	put_unaligned_le64(key, &data[0]);
	put_unaligned_le64(sa_key, &data[8]);

	memset(&c, 0, sizeof(c));
	c.common.opcode = op;
	c.common.nsid = cpu_to_le32(ns->ns_id);
	c.common.cdw10[0] = cpu_to_le32(cdw10);

	return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
}

static int nvme_pr_register(struct block_device *bdev, u64 old,
		u64 new, unsigned flags)
{
	u32 cdw10;

	if (flags & ~PR_FL_IGNORE_KEY)
		return -EOPNOTSUPP;

	cdw10 = old ? 2 : 0;
	cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
	cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
}

static int nvme_pr_reserve(struct block_device *bdev, u64 key,
		enum pr_type type, unsigned flags)
{
	u32 cdw10;

	if (flags & ~PR_FL_IGNORE_KEY)
		return -EOPNOTSUPP;

	cdw10 = nvme_pr_type(type) << 8;
	cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
}

static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
		enum pr_type type, bool abort)
{
	u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
}

static int nvme_pr_clear(struct block_device *bdev, u64 key)
{
1043
	u32 cdw10 = 1 | (key ? 1 << 3 : 0);
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
}

static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
{
	u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
}

static const struct pr_ops nvme_pr_ops = {
	.pr_register	= nvme_pr_register,
	.pr_reserve	= nvme_pr_reserve,
	.pr_release	= nvme_pr_release,
	.pr_preempt	= nvme_pr_preempt,
	.pr_clear	= nvme_pr_clear,
};

1061
#ifdef CONFIG_BLK_SED_OPAL
1062 1063
int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
		bool send)
1064
{
1065
	struct nvme_ctrl *ctrl = data;
1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082
	struct nvme_command cmd;

	memset(&cmd, 0, sizeof(cmd));
	if (send)
		cmd.common.opcode = nvme_admin_security_send;
	else
		cmd.common.opcode = nvme_admin_security_recv;
	cmd.common.nsid = 0;
	cmd.common.cdw10[0] = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
	cmd.common.cdw10[1] = cpu_to_le32(len);

	return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
				      ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0);
}
EXPORT_SYMBOL_GPL(nvme_sec_submit);
#endif /* CONFIG_BLK_SED_OPAL */

1083
static const struct block_device_operations nvme_fops = {
1084 1085 1086 1087 1088 1089 1090 1091 1092 1093
	.owner		= THIS_MODULE,
	.ioctl		= nvme_ioctl,
	.compat_ioctl	= nvme_compat_ioctl,
	.open		= nvme_open,
	.release	= nvme_release,
	.getgeo		= nvme_getgeo,
	.revalidate_disk= nvme_revalidate_disk,
	.pr_ops		= &nvme_pr_ops,
};

1094 1095 1096 1097 1098 1099 1100 1101
static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
{
	unsigned long timeout =
		((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
	u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
	int ret;

	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
K
Keith Busch 已提交
1102 1103
		if (csts == ~0)
			return -ENODEV;
1104 1105 1106 1107 1108 1109 1110
		if ((csts & NVME_CSTS_RDY) == bit)
			break;

		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
1111
			dev_err(ctrl->device,
1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129 1130 1131 1132 1133 1134 1135 1136
				"Device not ready; aborting %s\n", enabled ?
						"initialisation" : "reset");
			return -ENODEV;
		}
	}

	return ret;
}

/*
 * If the device has been passed off to us in an enabled state, just clear
 * the enabled bit.  The spec says we should set the 'shutdown notification
 * bits', but doing so may cause the device to complete commands to the
 * admin queue ... and we don't know what memory that might be pointing at!
 */
int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
{
	int ret;

	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
	ctrl->ctrl_config &= ~NVME_CC_ENABLE;

	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
	if (ret)
		return ret;
1137 1138 1139 1140 1141 1142 1143 1144 1145

	/* Checking for ctrl->tagset is a trick to avoid sleeping on module
	 * load, since we only need the quirk on reset_controller. Notice
	 * that the HGST device needs this delay only in firmware activation
	 * procedure; unfortunately we have no (easy) way to verify this.
	 */
	if ((ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY) && ctrl->tagset)
		msleep(NVME_QUIRK_DELAY_AMOUNT);

1146 1147
	return nvme_wait_ready(ctrl, cap, false);
}
1148
EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160

int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
{
	/*
	 * Default to a 4K page size, with the intention to update this
	 * path in the future to accomodate architectures with differing
	 * kernel and IO page sizes.
	 */
	unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
	int ret;

	if (page_shift < dev_page_min) {
1161
		dev_err(ctrl->device,
1162 1163 1164 1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179
			"Minimum device page size %u too large for host (%u)\n",
			1 << dev_page_min, 1 << page_shift);
		return -ENODEV;
	}

	ctrl->page_size = 1 << page_shift;

	ctrl->ctrl_config = NVME_CC_CSS_NVM;
	ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
	ctrl->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
	ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
	ctrl->ctrl_config |= NVME_CC_ENABLE;

	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
	if (ret)
		return ret;
	return nvme_wait_ready(ctrl, cap, true);
}
1180
EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202

int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
{
	unsigned long timeout = SHUTDOWN_TIMEOUT + jiffies;
	u32 csts;
	int ret;

	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
	ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;

	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
	if (ret)
		return ret;

	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
		if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
			break;

		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
1203
			dev_err(ctrl->device,
1204 1205 1206 1207 1208 1209 1210
				"Device shutdown incomplete; abort shutdown\n");
			return -ENODEV;
		}
	}

	return ret;
}
1211
EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
1212

1213 1214 1215
static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
		struct request_queue *q)
{
1216 1217
	bool vwc = false;

1218
	if (ctrl->max_hw_sectors) {
1219 1220 1221
		u32 max_segments =
			(ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;

1222
		blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1223
		blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1224
	}
K
Keith Busch 已提交
1225 1226
	if (ctrl->quirks & NVME_QUIRK_STRIPE_SIZE)
		blk_queue_chunk_sectors(q, ctrl->max_hw_sectors);
1227
	blk_queue_virt_boundary(q, ctrl->page_size - 1);
1228 1229 1230
	if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
		vwc = true;
	blk_queue_write_cache(q, vwc, vwc);
1231 1232
}

1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
/*
 * Initialize the cached copies of the Identify data and various controller
 * register in our nvme_ctrl structure.  This should be called as soon as
 * the admin queue is fully up and running.
 */
int nvme_init_identify(struct nvme_ctrl *ctrl)
{
	struct nvme_id_ctrl *id;
	u64 cap;
	int ret, page_shift;
1243
	u32 max_hw_sectors;
1244

1245 1246
	ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
	if (ret) {
1247
		dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
1248 1249 1250
		return ret;
	}

1251 1252
	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
	if (ret) {
1253
		dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
1254 1255 1256 1257
		return ret;
	}
	page_shift = NVME_CAP_MPSMIN(cap) + 12;

1258
	if (ctrl->vs >= NVME_VS(1, 1, 0))
1259 1260
		ctrl->subsystem = NVME_CAP_NSSRC(cap);

1261 1262
	ret = nvme_identify_ctrl(ctrl, &id);
	if (ret) {
1263
		dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
1264 1265 1266
		return -EIO;
	}

1267
	ctrl->oacs = le16_to_cpu(id->oacs);
1268
	ctrl->vid = le16_to_cpu(id->vid);
1269
	ctrl->oncs = le16_to_cpup(&id->oncs);
1270
	atomic_set(&ctrl->abort_limit, id->acl + 1);
1271
	ctrl->vwc = id->vwc;
M
Ming Lin 已提交
1272
	ctrl->cntlid = le16_to_cpup(&id->cntlid);
1273 1274 1275 1276
	memcpy(ctrl->serial, id->sn, sizeof(id->sn));
	memcpy(ctrl->model, id->mn, sizeof(id->mn));
	memcpy(ctrl->firmware_rev, id->fr, sizeof(id->fr));
	if (id->mdts)
1277
		max_hw_sectors = 1 << (id->mdts + page_shift - 9);
1278
	else
1279 1280 1281
		max_hw_sectors = UINT_MAX;
	ctrl->max_hw_sectors =
		min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
1282

1283
	nvme_set_queue_limits(ctrl, ctrl->admin_q);
1284
	ctrl->sgls = le32_to_cpu(id->sgls);
S
Sagi Grimberg 已提交
1285
	ctrl->kas = le16_to_cpu(id->kas);
1286 1287 1288 1289 1290 1291 1292 1293 1294 1295 1296 1297 1298

	if (ctrl->ops->is_fabrics) {
		ctrl->icdoff = le16_to_cpu(id->icdoff);
		ctrl->ioccsz = le32_to_cpu(id->ioccsz);
		ctrl->iorcsz = le32_to_cpu(id->iorcsz);
		ctrl->maxcmd = le16_to_cpu(id->maxcmd);

		/*
		 * In fabrics we need to verify the cntlid matches the
		 * admin connect
		 */
		if (ctrl->cntlid != le16_to_cpu(id->cntlid))
			ret = -EINVAL;
S
Sagi Grimberg 已提交
1299 1300 1301 1302 1303 1304

		if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
			dev_err(ctrl->dev,
				"keep-alive support is mandatory for fabrics\n");
			ret = -EINVAL;
		}
1305 1306 1307
	} else {
		ctrl->cntlid = le16_to_cpu(id->cntlid);
	}
1308

1309
	kfree(id);
1310
	return ret;
1311
}
1312
EXPORT_SYMBOL_GPL(nvme_init_identify);
1313

1314
static int nvme_dev_open(struct inode *inode, struct file *file)
1315
{
1316 1317 1318
	struct nvme_ctrl *ctrl;
	int instance = iminor(inode);
	int ret = -ENODEV;
1319

1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337
	spin_lock(&dev_list_lock);
	list_for_each_entry(ctrl, &nvme_ctrl_list, node) {
		if (ctrl->instance != instance)
			continue;

		if (!ctrl->admin_q) {
			ret = -EWOULDBLOCK;
			break;
		}
		if (!kref_get_unless_zero(&ctrl->kref))
			break;
		file->private_data = ctrl;
		ret = 0;
		break;
	}
	spin_unlock(&dev_list_lock);

	return ret;
1338 1339
}

1340
static int nvme_dev_release(struct inode *inode, struct file *file)
1341
{
1342 1343 1344 1345
	nvme_put_ctrl(file->private_data);
	return 0;
}

1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358
static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
{
	struct nvme_ns *ns;
	int ret;

	mutex_lock(&ctrl->namespaces_mutex);
	if (list_empty(&ctrl->namespaces)) {
		ret = -ENOTTY;
		goto out_unlock;
	}

	ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
	if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
1359
		dev_warn(ctrl->device,
1360 1361 1362 1363 1364
			"NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
		ret = -EINVAL;
		goto out_unlock;
	}

1365
	dev_warn(ctrl->device,
1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378
		"using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
	kref_get(&ns->kref);
	mutex_unlock(&ctrl->namespaces_mutex);

	ret = nvme_user_cmd(ctrl, ns, argp);
	nvme_put_ns(ns);
	return ret;

out_unlock:
	mutex_unlock(&ctrl->namespaces_mutex);
	return ret;
}

1379 1380 1381 1382 1383 1384 1385 1386 1387 1388
static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
		unsigned long arg)
{
	struct nvme_ctrl *ctrl = file->private_data;
	void __user *argp = (void __user *)arg;

	switch (cmd) {
	case NVME_IOCTL_ADMIN_CMD:
		return nvme_user_cmd(ctrl, NULL, argp);
	case NVME_IOCTL_IO_CMD:
1389
		return nvme_dev_user_cmd(ctrl, argp);
1390
	case NVME_IOCTL_RESET:
1391
		dev_warn(ctrl->device, "resetting controller\n");
1392 1393 1394
		return ctrl->ops->reset_ctrl(ctrl);
	case NVME_IOCTL_SUBSYS_RESET:
		return nvme_reset_subsystem(ctrl);
K
Keith Busch 已提交
1395 1396 1397
	case NVME_IOCTL_RESCAN:
		nvme_queue_scan(ctrl);
		return 0;
1398 1399 1400 1401 1402 1403 1404 1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417 1418 1419 1420 1421
	default:
		return -ENOTTY;
	}
}

static const struct file_operations nvme_dev_fops = {
	.owner		= THIS_MODULE,
	.open		= nvme_dev_open,
	.release	= nvme_dev_release,
	.unlocked_ioctl	= nvme_dev_ioctl,
	.compat_ioctl	= nvme_dev_ioctl,
};

static ssize_t nvme_sysfs_reset(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	int ret;

	ret = ctrl->ops->reset_ctrl(ctrl);
	if (ret < 0)
		return ret;
	return count;
1422
}
1423
static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
1424

K
Keith Busch 已提交
1425 1426 1427 1428 1429 1430 1431 1432 1433 1434 1435
static ssize_t nvme_sysfs_rescan(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	nvme_queue_scan(ctrl);
	return count;
}
static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);

1436 1437 1438
static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
								char *buf)
{
1439
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458 1459
	struct nvme_ctrl *ctrl = ns->ctrl;
	int serial_len = sizeof(ctrl->serial);
	int model_len = sizeof(ctrl->model);

	if (memchr_inv(ns->uuid, 0, sizeof(ns->uuid)))
		return sprintf(buf, "eui.%16phN\n", ns->uuid);

	if (memchr_inv(ns->eui, 0, sizeof(ns->eui)))
		return sprintf(buf, "eui.%8phN\n", ns->eui);

	while (ctrl->serial[serial_len - 1] == ' ')
		serial_len--;
	while (ctrl->model[model_len - 1] == ' ')
		model_len--;

	return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", ctrl->vid,
		serial_len, ctrl->serial, model_len, ctrl->model, ns->ns_id);
}
static DEVICE_ATTR(wwid, S_IRUGO, wwid_show, NULL);

1460 1461 1462
static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
								char *buf)
{
1463
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1464 1465 1466 1467 1468 1469 1470
	return sprintf(buf, "%pU\n", ns->uuid);
}
static DEVICE_ATTR(uuid, S_IRUGO, uuid_show, NULL);

static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
								char *buf)
{
1471
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1472 1473 1474 1475 1476 1477 1478
	return sprintf(buf, "%8phd\n", ns->eui);
}
static DEVICE_ATTR(eui, S_IRUGO, eui_show, NULL);

static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
								char *buf)
{
1479
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1480 1481 1482 1483 1484
	return sprintf(buf, "%d\n", ns->ns_id);
}
static DEVICE_ATTR(nsid, S_IRUGO, nsid_show, NULL);

static struct attribute *nvme_ns_attrs[] = {
1485
	&dev_attr_wwid.attr,
1486 1487 1488 1489 1490 1491
	&dev_attr_uuid.attr,
	&dev_attr_eui.attr,
	&dev_attr_nsid.attr,
	NULL,
};

M
Ming Lin 已提交
1492
static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj,
1493 1494 1495
		struct attribute *a, int n)
{
	struct device *dev = container_of(kobj, struct device, kobj);
1496
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510

	if (a == &dev_attr_uuid.attr) {
		if (!memchr_inv(ns->uuid, 0, sizeof(ns->uuid)))
			return 0;
	}
	if (a == &dev_attr_eui.attr) {
		if (!memchr_inv(ns->eui, 0, sizeof(ns->eui)))
			return 0;
	}
	return a->mode;
}

static const struct attribute_group nvme_ns_attr_group = {
	.attrs		= nvme_ns_attrs,
M
Ming Lin 已提交
1511
	.is_visible	= nvme_ns_attrs_are_visible,
1512 1513
};

M
Ming Lin 已提交
1514
#define nvme_show_str_function(field)						\
1515 1516 1517 1518 1519 1520 1521 1522
static ssize_t  field##_show(struct device *dev,				\
			    struct device_attribute *attr, char *buf)		\
{										\
        struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
        return sprintf(buf, "%.*s\n", (int)sizeof(ctrl->field), ctrl->field);	\
}										\
static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);

M
Ming Lin 已提交
1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535
#define nvme_show_int_function(field)						\
static ssize_t  field##_show(struct device *dev,				\
			    struct device_attribute *attr, char *buf)		\
{										\
        struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
        return sprintf(buf, "%d\n", ctrl->field);	\
}										\
static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);

nvme_show_str_function(model);
nvme_show_str_function(serial);
nvme_show_str_function(firmware_rev);
nvme_show_int_function(cntlid);
1536

M
Ming Lin 已提交
1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579
static ssize_t nvme_sysfs_delete(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	if (device_remove_file_self(dev, attr))
		ctrl->ops->delete_ctrl(ctrl);
	return count;
}
static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);

static ssize_t nvme_sysfs_show_transport(struct device *dev,
					 struct device_attribute *attr,
					 char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
}
static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);

static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
					 struct device_attribute *attr,
					 char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	return snprintf(buf, PAGE_SIZE, "%s\n",
			ctrl->ops->get_subsysnqn(ctrl));
}
static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);

static ssize_t nvme_sysfs_show_address(struct device *dev,
					 struct device_attribute *attr,
					 char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
}
static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);

1580 1581
static struct attribute *nvme_dev_attrs[] = {
	&dev_attr_reset_controller.attr,
K
Keith Busch 已提交
1582
	&dev_attr_rescan_controller.attr,
1583 1584 1585
	&dev_attr_model.attr,
	&dev_attr_serial.attr,
	&dev_attr_firmware_rev.attr,
M
Ming Lin 已提交
1586
	&dev_attr_cntlid.attr,
M
Ming Lin 已提交
1587 1588 1589 1590
	&dev_attr_delete_controller.attr,
	&dev_attr_transport.attr,
	&dev_attr_subsysnqn.attr,
	&dev_attr_address.attr,
1591 1592 1593
	NULL
};

M
Ming Lin 已提交
1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604 1605 1606 1607 1608 1609 1610 1611 1612 1613 1614 1615
#define CHECK_ATTR(ctrl, a, name)		\
	if ((a) == &dev_attr_##name.attr &&	\
	    !(ctrl)->ops->get_##name)		\
		return 0

static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
		struct attribute *a, int n)
{
	struct device *dev = container_of(kobj, struct device, kobj);
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	if (a == &dev_attr_delete_controller.attr) {
		if (!ctrl->ops->delete_ctrl)
			return 0;
	}

	CHECK_ATTR(ctrl, a, subsysnqn);
	CHECK_ATTR(ctrl, a, address);

	return a->mode;
}

1616
static struct attribute_group nvme_dev_attrs_group = {
M
Ming Lin 已提交
1617 1618
	.attrs		= nvme_dev_attrs,
	.is_visible	= nvme_dev_attrs_are_visible,
1619 1620 1621 1622 1623 1624 1625
};

static const struct attribute_group *nvme_dev_attr_groups[] = {
	&nvme_dev_attrs_group,
	NULL,
};

1626 1627 1628 1629 1630 1631 1632 1633
static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
	struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);

	return nsa->ns_id - nsb->ns_id;
}

1634
static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
1635
{
1636
	struct nvme_ns *ns, *ret = NULL;
1637

1638
	mutex_lock(&ctrl->namespaces_mutex);
1639
	list_for_each_entry(ns, &ctrl->namespaces, list) {
1640 1641 1642 1643 1644
		if (ns->ns_id == nsid) {
			kref_get(&ns->kref);
			ret = ns;
			break;
		}
1645 1646 1647
		if (ns->ns_id > nsid)
			break;
	}
1648 1649
	mutex_unlock(&ctrl->namespaces_mutex);
	return ret;
1650 1651 1652 1653 1654 1655
}

static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
{
	struct nvme_ns *ns;
	struct gendisk *disk;
1656 1657
	struct nvme_id_ns *id;
	char disk_name[DISK_NAME_LEN];
1658 1659 1660 1661 1662 1663
	int node = dev_to_node(ctrl->dev);

	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
	if (!ns)
		return;

1664 1665 1666 1667
	ns->instance = ida_simple_get(&ctrl->ns_ida, 1, 0, GFP_KERNEL);
	if (ns->instance < 0)
		goto out_free_ns;

1668 1669
	ns->queue = blk_mq_init_queue(ctrl->tagset);
	if (IS_ERR(ns->queue))
1670
		goto out_release_instance;
1671 1672 1673 1674 1675 1676 1677 1678 1679
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
	ns->queue->queuedata = ns;
	ns->ctrl = ctrl;

	kref_init(&ns->kref);
	ns->ns_id = nsid;
	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */

	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
1680
	nvme_set_queue_limits(ctrl, ns->queue);
1681

1682
	sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->instance);
1683

1684 1685 1686
	if (nvme_revalidate_ns(ns, &id))
		goto out_free_queue;

1687 1688 1689 1690 1691
	if (nvme_nvm_ns_supported(ns, id) &&
				nvme_nvm_register(ns, disk_name, node)) {
		dev_warn(ctrl->dev, "%s: LightNVM init failure\n", __func__);
		goto out_free_id;
	}
1692

1693 1694 1695
	disk = alloc_disk_node(0, node);
	if (!disk)
		goto out_free_id;
1696

1697 1698 1699 1700 1701 1702 1703 1704
	disk->fops = &nvme_fops;
	disk->private_data = ns;
	disk->queue = ns->queue;
	disk->flags = GENHD_FL_EXT_DEVT;
	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
	ns->disk = disk;

	__nvme_revalidate_disk(disk, id);
1705

1706 1707 1708 1709
	mutex_lock(&ctrl->namespaces_mutex);
	list_add_tail(&ns->list, &ctrl->namespaces);
	mutex_unlock(&ctrl->namespaces_mutex);

1710
	kref_get(&ctrl->kref);
1711 1712 1713

	kfree(id);

1714
	device_add_disk(ctrl->device, ns->disk);
1715 1716 1717 1718
	if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
					&nvme_ns_attr_group))
		pr_warn("%s: failed to create sysfs group for identification\n",
			ns->disk->disk_name);
1719 1720 1721
	if (ns->ndev && nvme_nvm_register_sysfs(ns))
		pr_warn("%s: failed to register lightnvm sysfs group for identification\n",
			ns->disk->disk_name);
1722
	return;
1723 1724
 out_free_id:
	kfree(id);
1725 1726
 out_free_queue:
	blk_cleanup_queue(ns->queue);
1727 1728
 out_release_instance:
	ida_simple_remove(&ctrl->ns_ida, ns->instance);
1729 1730 1731 1732 1733 1734
 out_free_ns:
	kfree(ns);
}

static void nvme_ns_remove(struct nvme_ns *ns)
{
1735 1736
	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
		return;
1737

1738
	if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
1739 1740
		if (blk_get_integrity(ns->disk))
			blk_integrity_unregister(ns->disk);
1741 1742
		sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
					&nvme_ns_attr_group);
1743 1744
		if (ns->ndev)
			nvme_nvm_unregister_sysfs(ns);
1745 1746 1747 1748
		del_gendisk(ns->disk);
		blk_mq_abort_requeue_list(ns->queue);
		blk_cleanup_queue(ns->queue);
	}
1749 1750

	mutex_lock(&ns->ctrl->namespaces_mutex);
1751
	list_del_init(&ns->list);
1752 1753
	mutex_unlock(&ns->ctrl->namespaces_mutex);

1754 1755 1756
	nvme_put_ns(ns);
}

1757 1758 1759 1760
static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
{
	struct nvme_ns *ns;

1761
	ns = nvme_find_get_ns(ctrl, nsid);
1762
	if (ns) {
1763
		if (ns->disk && revalidate_disk(ns->disk))
1764
			nvme_ns_remove(ns);
1765
		nvme_put_ns(ns);
1766 1767 1768 1769
	} else
		nvme_alloc_ns(ctrl, nsid);
}

1770 1771 1772 1773 1774 1775 1776 1777 1778 1779 1780
static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
					unsigned nsid)
{
	struct nvme_ns *ns, *next;

	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
		if (ns->ns_id > nsid)
			nvme_ns_remove(ns);
	}
}

1781 1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794
static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
{
	struct nvme_ns *ns;
	__le32 *ns_list;
	unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024);
	int ret = 0;

	ns_list = kzalloc(0x1000, GFP_KERNEL);
	if (!ns_list)
		return -ENOMEM;

	for (i = 0; i < num_lists; i++) {
		ret = nvme_identify_ns_list(ctrl, prev, ns_list);
		if (ret)
1795
			goto free;
1796 1797 1798 1799 1800 1801 1802 1803 1804

		for (j = 0; j < min(nn, 1024U); j++) {
			nsid = le32_to_cpu(ns_list[j]);
			if (!nsid)
				goto out;

			nvme_validate_ns(ctrl, nsid);

			while (++prev < nsid) {
1805 1806
				ns = nvme_find_get_ns(ctrl, prev);
				if (ns) {
1807
					nvme_ns_remove(ns);
1808 1809
					nvme_put_ns(ns);
				}
1810 1811 1812 1813 1814
			}
		}
		nn -= j;
	}
 out:
1815 1816
	nvme_remove_invalid_namespaces(ctrl, prev);
 free:
1817 1818 1819 1820
	kfree(ns_list);
	return ret;
}

1821
static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn)
1822 1823 1824
{
	unsigned i;

1825 1826 1827
	for (i = 1; i <= nn; i++)
		nvme_validate_ns(ctrl, i);

1828
	nvme_remove_invalid_namespaces(ctrl, nn);
1829 1830
}

1831
static void nvme_scan_work(struct work_struct *work)
1832
{
1833 1834
	struct nvme_ctrl *ctrl =
		container_of(work, struct nvme_ctrl, scan_work);
1835
	struct nvme_id_ctrl *id;
1836
	unsigned nn;
1837

1838 1839 1840
	if (ctrl->state != NVME_CTRL_LIVE)
		return;

1841 1842
	if (nvme_identify_ctrl(ctrl, &id))
		return;
1843 1844

	nn = le32_to_cpu(id->nn);
1845
	if (ctrl->vs >= NVME_VS(1, 1, 0) &&
1846 1847 1848 1849
	    !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
		if (!nvme_scan_ns_list(ctrl, nn))
			goto done;
	}
1850
	nvme_scan_ns_sequential(ctrl, nn);
1851
 done:
1852
	mutex_lock(&ctrl->namespaces_mutex);
1853
	list_sort(NULL, &ctrl->namespaces, ns_cmp);
1854
	mutex_unlock(&ctrl->namespaces_mutex);
1855 1856
	kfree(id);
}
1857 1858 1859 1860 1861 1862 1863 1864 1865 1866 1867

void nvme_queue_scan(struct nvme_ctrl *ctrl)
{
	/*
	 * Do not queue new scan work when a controller is reset during
	 * removal.
	 */
	if (ctrl->state == NVME_CTRL_LIVE)
		schedule_work(&ctrl->scan_work);
}
EXPORT_SYMBOL_GPL(nvme_queue_scan);
1868

1869 1870 1871 1872 1873
/*
 * This function iterates the namespace list unlocked to allow recovery from
 * controller failure. It is up to the caller to ensure the namespace list is
 * not modified by scan work while this function is executing.
 */
1874 1875 1876 1877
void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns, *next;

1878 1879 1880 1881 1882 1883 1884 1885 1886
	/*
	 * The dead states indicates the controller was not gracefully
	 * disconnected. In that case, we won't be able to flush any data while
	 * removing the namespaces' disks; fail all the queues now to avoid
	 * potentially having to clean up the failed sync later.
	 */
	if (ctrl->state == NVME_CTRL_DEAD)
		nvme_kill_queues(ctrl);

1887 1888 1889
	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
		nvme_ns_remove(ns);
}
1890
EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
1891

1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902 1903 1904 1905 1906 1907
static void nvme_async_event_work(struct work_struct *work)
{
	struct nvme_ctrl *ctrl =
		container_of(work, struct nvme_ctrl, async_event_work);

	spin_lock_irq(&ctrl->lock);
	while (ctrl->event_limit > 0) {
		int aer_idx = --ctrl->event_limit;

		spin_unlock_irq(&ctrl->lock);
		ctrl->ops->submit_async_event(ctrl, aer_idx);
		spin_lock_irq(&ctrl->lock);
	}
	spin_unlock_irq(&ctrl->lock);
}

1908 1909
void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
		union nvme_result *res)
1910
{
1911 1912
	u32 result = le32_to_cpu(res->u32);
	bool done = true;
1913

1914 1915 1916 1917 1918
	switch (le16_to_cpu(status) >> 1) {
	case NVME_SC_SUCCESS:
		done = false;
		/*FALLTHRU*/
	case NVME_SC_ABORT_REQ:
1919 1920
		++ctrl->event_limit;
		schedule_work(&ctrl->async_event_work);
1921 1922 1923
		break;
	default:
		break;
1924 1925
	}

1926
	if (done)
1927 1928 1929 1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940 1941 1942 1943 1944 1945 1946
		return;

	switch (result & 0xff07) {
	case NVME_AER_NOTICE_NS_CHANGED:
		dev_info(ctrl->device, "rescanning\n");
		nvme_queue_scan(ctrl);
		break;
	default:
		dev_warn(ctrl->device, "async event result %08x\n", result);
	}
}
EXPORT_SYMBOL_GPL(nvme_complete_async_event);

void nvme_queue_async_events(struct nvme_ctrl *ctrl)
{
	ctrl->event_limit = NVME_NR_AERS;
	schedule_work(&ctrl->async_event_work);
}
EXPORT_SYMBOL_GPL(nvme_queue_async_events);

1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972 1973 1974 1975
static DEFINE_IDA(nvme_instance_ida);

static int nvme_set_instance(struct nvme_ctrl *ctrl)
{
	int instance, error;

	do {
		if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
			return -ENODEV;

		spin_lock(&dev_list_lock);
		error = ida_get_new(&nvme_instance_ida, &instance);
		spin_unlock(&dev_list_lock);
	} while (error == -EAGAIN);

	if (error)
		return -ENODEV;

	ctrl->instance = instance;
	return 0;
}

static void nvme_release_instance(struct nvme_ctrl *ctrl)
{
	spin_lock(&dev_list_lock);
	ida_remove(&nvme_instance_ida, ctrl->instance);
	spin_unlock(&dev_list_lock);
}

1976
void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
1977
{
1978
	flush_work(&ctrl->async_event_work);
1979 1980 1981
	flush_work(&ctrl->scan_work);
	nvme_remove_namespaces(ctrl);

1982
	device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
1983 1984 1985 1986

	spin_lock(&dev_list_lock);
	list_del(&ctrl->node);
	spin_unlock(&dev_list_lock);
1987
}
1988
EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
1989 1990 1991 1992

static void nvme_free_ctrl(struct kref *kref)
{
	struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
1993 1994 1995

	put_device(ctrl->device);
	nvme_release_instance(ctrl);
1996
	ida_destroy(&ctrl->ns_ida);
1997 1998 1999 2000 2001 2002 2003 2004

	ctrl->ops->free_ctrl(ctrl);
}

void nvme_put_ctrl(struct nvme_ctrl *ctrl)
{
	kref_put(&ctrl->kref, nvme_free_ctrl);
}
2005
EXPORT_SYMBOL_GPL(nvme_put_ctrl);
2006 2007 2008 2009 2010 2011 2012 2013 2014 2015 2016

/*
 * Initialize a NVMe controller structures.  This needs to be called during
 * earliest initialization so that we have the initialized structured around
 * during probing.
 */
int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
		const struct nvme_ctrl_ops *ops, unsigned long quirks)
{
	int ret;

2017 2018
	ctrl->state = NVME_CTRL_NEW;
	spin_lock_init(&ctrl->lock);
2019
	INIT_LIST_HEAD(&ctrl->namespaces);
2020
	mutex_init(&ctrl->namespaces_mutex);
2021 2022 2023 2024
	kref_init(&ctrl->kref);
	ctrl->dev = dev;
	ctrl->ops = ops;
	ctrl->quirks = quirks;
2025
	INIT_WORK(&ctrl->scan_work, nvme_scan_work);
2026
	INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
2027 2028 2029 2030 2031

	ret = nvme_set_instance(ctrl);
	if (ret)
		goto out;

2032
	ctrl->device = device_create_with_groups(nvme_class, ctrl->dev,
2033
				MKDEV(nvme_char_major, ctrl->instance),
2034
				ctrl, nvme_dev_attr_groups,
2035
				"nvme%d", ctrl->instance);
2036 2037 2038 2039 2040
	if (IS_ERR(ctrl->device)) {
		ret = PTR_ERR(ctrl->device);
		goto out_release_instance;
	}
	get_device(ctrl->device);
2041
	ida_init(&ctrl->ns_ida);
2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052

	spin_lock(&dev_list_lock);
	list_add_tail(&ctrl->node, &nvme_ctrl_list);
	spin_unlock(&dev_list_lock);

	return 0;
out_release_instance:
	nvme_release_instance(ctrl);
out:
	return ret;
}
2053
EXPORT_SYMBOL_GPL(nvme_init_ctrl);
2054

2055 2056 2057 2058 2059 2060 2061 2062 2063 2064 2065
/**
 * nvme_kill_queues(): Ends all namespace queues
 * @ctrl: the dead controller that needs to end
 *
 * Call this function when the driver determines it is unable to get the
 * controller in a state capable of servicing IO.
 */
void nvme_kill_queues(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns;

2066 2067
	mutex_lock(&ctrl->namespaces_mutex);
	list_for_each_entry(ns, &ctrl->namespaces, list) {
2068 2069 2070 2071
		/*
		 * Revalidating a dead namespace sets capacity to 0. This will
		 * end buffered writers dirtying pages that can't be synced.
		 */
2072
		if (ns->disk && !test_and_set_bit(NVME_NS_DEAD, &ns->flags))
2073 2074 2075 2076 2077 2078
			revalidate_disk(ns->disk);

		blk_set_queue_dying(ns->queue);
		blk_mq_abort_requeue_list(ns->queue);
		blk_mq_start_stopped_hw_queues(ns->queue, true);
	}
2079
	mutex_unlock(&ctrl->namespaces_mutex);
2080
}
2081
EXPORT_SYMBOL_GPL(nvme_kill_queues);
2082

2083
void nvme_stop_queues(struct nvme_ctrl *ctrl)
2084 2085 2086
{
	struct nvme_ns *ns;

2087
	mutex_lock(&ctrl->namespaces_mutex);
2088
	list_for_each_entry(ns, &ctrl->namespaces, list)
2089
		blk_mq_quiesce_queue(ns->queue);
2090
	mutex_unlock(&ctrl->namespaces_mutex);
2091
}
2092
EXPORT_SYMBOL_GPL(nvme_stop_queues);
2093

2094
void nvme_start_queues(struct nvme_ctrl *ctrl)
2095 2096 2097
{
	struct nvme_ns *ns;

2098 2099
	mutex_lock(&ctrl->namespaces_mutex);
	list_for_each_entry(ns, &ctrl->namespaces, list) {
2100 2101 2102
		blk_mq_start_stopped_hw_queues(ns->queue, true);
		blk_mq_kick_requeue_list(ns->queue);
	}
2103
	mutex_unlock(&ctrl->namespaces_mutex);
2104
}
2105
EXPORT_SYMBOL_GPL(nvme_start_queues);
2106

2107 2108 2109 2110
int __init nvme_core_init(void)
{
	int result;

2111 2112 2113
	result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
							&nvme_dev_fops);
	if (result < 0)
2114
		return result;
2115 2116 2117 2118 2119 2120 2121 2122 2123
	else if (result > 0)
		nvme_char_major = result;

	nvme_class = class_create(THIS_MODULE, "nvme");
	if (IS_ERR(nvme_class)) {
		result = PTR_ERR(nvme_class);
		goto unregister_chrdev;
	}

2124
	return 0;
2125 2126 2127 2128

 unregister_chrdev:
	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
	return result;
2129 2130 2131 2132
}

void nvme_core_exit(void)
{
2133 2134
	class_destroy(nvme_class);
	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
2135
}
2136 2137 2138 2139 2140

MODULE_LICENSE("GPL");
MODULE_VERSION("1.0");
module_init(nvme_core_init);
module_exit(nvme_core_exit);