core.c 60.8 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * NVM Express device driver
 * Copyright (c) 2011-2014, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/blkdev.h>
#include <linux/blk-mq.h>
17
#include <linux/delay.h>
18
#include <linux/errno.h>
19
#include <linux/hdreg.h>
20
#include <linux/kernel.h>
21 22
#include <linux/module.h>
#include <linux/list_sort.h>
23 24
#include <linux/slab.h>
#include <linux/types.h>
25 26 27 28
#include <linux/pr.h>
#include <linux/ptrace.h>
#include <linux/nvme_ioctl.h>
#include <linux/t10-pi.h>
29
#include <linux/pm_qos.h>
30 31
#include <scsi/sg.h>
#include <asm/unaligned.h>
32 33

#include "nvme.h"
S
Sagi Grimberg 已提交
34
#include "fabrics.h"
35

36 37
#define NVME_MINORS		(1U << MINORBITS)

38 39 40
unsigned char admin_timeout = 60;
module_param(admin_timeout, byte, 0644);
MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
41
EXPORT_SYMBOL_GPL(admin_timeout);
42 43 44 45

unsigned char nvme_io_timeout = 30;
module_param_named(io_timeout, nvme_io_timeout, byte, 0644);
MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
46
EXPORT_SYMBOL_GPL(nvme_io_timeout);
47 48 49 50 51

unsigned char shutdown_timeout = 5;
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");

K
Keith Busch 已提交
52 53 54 55
unsigned int nvme_max_retries = 5;
module_param_named(max_retries, nvme_max_retries, uint, 0644);
MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
EXPORT_SYMBOL_GPL(nvme_max_retries);
56

57 58 59
static int nvme_char_major;
module_param(nvme_char_major, int, 0);

60 61 62 63 64
static unsigned long default_ps_max_latency_us = 25000;
module_param(default_ps_max_latency_us, ulong, 0644);
MODULE_PARM_DESC(default_ps_max_latency_us,
		 "max power saving latency for new devices; use PM QOS to change per device");

65
static LIST_HEAD(nvme_ctrl_list);
M
Ming Lin 已提交
66
static DEFINE_SPINLOCK(dev_list_lock);
67

68 69
static struct class *nvme_class;

70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
static inline bool nvme_req_needs_retry(struct request *req, u16 status)
{
	return !(status & NVME_SC_DNR || blk_noretry_request(req)) &&
		(jiffies - req->start_time) < req->timeout &&
		req->retries < nvme_max_retries;
}

void nvme_complete_rq(struct request *req)
{
	int error = 0;

	if (unlikely(req->errors)) {
		if (nvme_req_needs_retry(req, req->errors)) {
			req->retries++;
			blk_mq_requeue_request(req,
					!blk_mq_queue_stopped(req->q));
			return;
		}

		if (blk_rq_is_passthrough(req))
			error = req->errors;
		else
			error = nvme_error_status(req->errors);
	}

	blk_mq_end_request(req, error);
}
EXPORT_SYMBOL_GPL(nvme_complete_rq);

99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115
void nvme_cancel_request(struct request *req, void *data, bool reserved)
{
	int status;

	if (!blk_mq_request_started(req))
		return;

	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
				"Cancelling I/O %d", req->tag);

	status = NVME_SC_ABORT_REQ;
	if (blk_queue_dying(req->q))
		status |= NVME_SC_DNR;
	blk_mq_complete_request(req, status);
}
EXPORT_SYMBOL_GPL(nvme_cancel_request);

116 117 118
bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
		enum nvme_ctrl_state new_state)
{
119
	enum nvme_ctrl_state old_state;
120 121 122
	bool changed = false;

	spin_lock_irq(&ctrl->lock);
123 124

	old_state = ctrl->state;
125 126 127
	switch (new_state) {
	case NVME_CTRL_LIVE:
		switch (old_state) {
128
		case NVME_CTRL_NEW:
129
		case NVME_CTRL_RESETTING:
130
		case NVME_CTRL_RECONNECTING:
131 132 133 134 135 136 137 138 139
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
	case NVME_CTRL_RESETTING:
		switch (old_state) {
		case NVME_CTRL_NEW:
140 141 142 143 144 145 146 147 148 149
		case NVME_CTRL_LIVE:
		case NVME_CTRL_RECONNECTING:
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
	case NVME_CTRL_RECONNECTING:
		switch (old_state) {
150 151 152 153 154 155 156 157 158 159 160
		case NVME_CTRL_LIVE:
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
	case NVME_CTRL_DELETING:
		switch (old_state) {
		case NVME_CTRL_LIVE:
		case NVME_CTRL_RESETTING:
161
		case NVME_CTRL_RECONNECTING:
162 163 164 165 166 167
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
168 169 170 171 172 173 174 175 176
	case NVME_CTRL_DEAD:
		switch (old_state) {
		case NVME_CTRL_DELETING:
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
177 178 179 180 181 182 183
	default:
		break;
	}

	if (changed)
		ctrl->state = new_state;

184 185
	spin_unlock_irq(&ctrl->lock);

186 187 188 189
	return changed;
}
EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);

190 191 192 193
static void nvme_free_ns(struct kref *kref)
{
	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);

194 195
	if (ns->ndev)
		nvme_nvm_unregister(ns);
196

197 198 199 200 201
	if (ns->disk) {
		spin_lock(&dev_list_lock);
		ns->disk->private_data = NULL;
		spin_unlock(&dev_list_lock);
	}
202 203

	put_disk(ns->disk);
204 205
	ida_simple_remove(&ns->ctrl->ns_ida, ns->instance);
	nvme_put_ctrl(ns->ctrl);
206 207 208
	kfree(ns);
}

209
static void nvme_put_ns(struct nvme_ns *ns)
210 211 212 213 214 215 216 217 218 219
{
	kref_put(&ns->kref, nvme_free_ns);
}

static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk)
{
	struct nvme_ns *ns;

	spin_lock(&dev_list_lock);
	ns = disk->private_data;
220 221 222 223 224 225
	if (ns) {
		if (!kref_get_unless_zero(&ns->kref))
			goto fail;
		if (!try_module_get(ns->ctrl->ops->module))
			goto fail_put_ns;
	}
226 227 228
	spin_unlock(&dev_list_lock);

	return ns;
229 230 231 232 233 234

fail_put_ns:
	kref_put(&ns->kref, nvme_free_ns);
fail:
	spin_unlock(&dev_list_lock);
	return NULL;
235 236
}

237
struct request *nvme_alloc_request(struct request_queue *q,
238
		struct nvme_command *cmd, unsigned int flags, int qid)
239
{
240
	unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
241 242
	struct request *req;

243
	if (qid == NVME_QID_ANY) {
244
		req = blk_mq_alloc_request(q, op, flags);
245
	} else {
246
		req = blk_mq_alloc_request_hctx(q, op, flags,
247 248
				qid ? qid - 1 : 0);
	}
249
	if (IS_ERR(req))
250
		return req;
251 252

	req->cmd_flags |= REQ_FAILFAST_DRIVER;
253
	nvme_req(req)->cmd = cmd;
254

255 256
	return req;
}
257
EXPORT_SYMBOL_GPL(nvme_alloc_request);
258

M
Ming Lin 已提交
259 260 261 262 263 264 265 266 267 268 269
static inline void nvme_setup_flush(struct nvme_ns *ns,
		struct nvme_command *cmnd)
{
	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->common.opcode = nvme_cmd_flush;
	cmnd->common.nsid = cpu_to_le32(ns->ns_id);
}

static inline int nvme_setup_discard(struct nvme_ns *ns, struct request *req,
		struct nvme_command *cmnd)
{
270
	unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
M
Ming Lin 已提交
271
	struct nvme_dsm_range *range;
272
	struct bio *bio;
M
Ming Lin 已提交
273

274
	range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC);
M
Ming Lin 已提交
275 276 277
	if (!range)
		return BLK_MQ_RQ_QUEUE_BUSY;

278 279 280 281 282 283 284 285 286 287 288 289 290 291
	__rq_for_each_bio(bio, req) {
		u64 slba = nvme_block_nr(ns, bio->bi_iter.bi_sector);
		u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;

		range[n].cattr = cpu_to_le32(0);
		range[n].nlb = cpu_to_le32(nlb);
		range[n].slba = cpu_to_le64(slba);
		n++;
	}

	if (WARN_ON_ONCE(n != segments)) {
		kfree(range);
		return BLK_MQ_RQ_QUEUE_ERROR;
	}
M
Ming Lin 已提交
292 293 294 295

	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->dsm.opcode = nvme_cmd_dsm;
	cmnd->dsm.nsid = cpu_to_le32(ns->ns_id);
296
	cmnd->dsm.nr = segments - 1;
M
Ming Lin 已提交
297 298
	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);

299 300
	req->special_vec.bv_page = virt_to_page(range);
	req->special_vec.bv_offset = offset_in_page(range);
301
	req->special_vec.bv_len = sizeof(*range) * segments;
302
	req->rq_flags |= RQF_SPECIAL_PAYLOAD;
M
Ming Lin 已提交
303

304
	return BLK_MQ_RQ_QUEUE_OK;
M
Ming Lin 已提交
305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350
}

static inline void nvme_setup_rw(struct nvme_ns *ns, struct request *req,
		struct nvme_command *cmnd)
{
	u16 control = 0;
	u32 dsmgmt = 0;

	if (req->cmd_flags & REQ_FUA)
		control |= NVME_RW_FUA;
	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
		control |= NVME_RW_LR;

	if (req->cmd_flags & REQ_RAHEAD)
		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;

	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
	cmnd->rw.nsid = cpu_to_le32(ns->ns_id);
	cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);

	if (ns->ms) {
		switch (ns->pi_type) {
		case NVME_NS_DPS_PI_TYPE3:
			control |= NVME_RW_PRINFO_PRCHK_GUARD;
			break;
		case NVME_NS_DPS_PI_TYPE1:
		case NVME_NS_DPS_PI_TYPE2:
			control |= NVME_RW_PRINFO_PRCHK_GUARD |
					NVME_RW_PRINFO_PRCHK_REF;
			cmnd->rw.reftag = cpu_to_le32(
					nvme_block_nr(ns, blk_rq_pos(req)));
			break;
		}
		if (!blk_integrity_rq(req))
			control |= NVME_RW_PRINFO_PRACT;
	}

	cmnd->rw.control = cpu_to_le16(control);
	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
}

int nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
		struct nvme_command *cmd)
{
351
	int ret = BLK_MQ_RQ_QUEUE_OK;
M
Ming Lin 已提交
352

353 354 355 356 357
	if (!(req->rq_flags & RQF_DONTPREP)) {
		req->retries = 0;
		req->rq_flags |= RQF_DONTPREP;
	}

358 359 360
	switch (req_op(req)) {
	case REQ_OP_DRV_IN:
	case REQ_OP_DRV_OUT:
361
		memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
362 363
		break;
	case REQ_OP_FLUSH:
M
Ming Lin 已提交
364
		nvme_setup_flush(ns, cmd);
365 366
		break;
	case REQ_OP_DISCARD:
M
Ming Lin 已提交
367
		ret = nvme_setup_discard(ns, req, cmd);
368 369 370
		break;
	case REQ_OP_READ:
	case REQ_OP_WRITE:
M
Ming Lin 已提交
371
		nvme_setup_rw(ns, req, cmd);
372 373 374 375 376
		break;
	default:
		WARN_ON_ONCE(1);
		return BLK_MQ_RQ_QUEUE_ERROR;
	}
M
Ming Lin 已提交
377

378
	cmd->common.command_id = req->tag;
M
Ming Lin 已提交
379 380 381 382
	return ret;
}
EXPORT_SYMBOL_GPL(nvme_setup_cmd);

383 384 385 386 387
/*
 * Returns 0 on success.  If the result is negative, it's a Linux error code;
 * if the result is positive, it's an NVM Express status code
 */
int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
388
		union nvme_result *result, void *buffer, unsigned bufflen,
389
		unsigned timeout, int qid, int at_head, int flags)
390 391 392 393
{
	struct request *req;
	int ret;

394
	req = nvme_alloc_request(q, cmd, flags, qid);
395 396 397 398 399
	if (IS_ERR(req))
		return PTR_ERR(req);

	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;

400 401 402 403
	if (buffer && bufflen) {
		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
		if (ret)
			goto out;
404 405
	}

406
	blk_execute_rq(req->q, NULL, req, at_head);
407 408
	if (result)
		*result = nvme_req(req)->result;
409 410 411 412 413
	ret = req->errors;
 out:
	blk_mq_free_request(req);
	return ret;
}
414
EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
415 416 417 418

int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
		void *buffer, unsigned bufflen)
{
419 420
	return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
			NVME_QID_ANY, 0, 0);
421
}
422
EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
423

424 425 426 427
int __nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
		void __user *ubuffer, unsigned bufflen,
		void __user *meta_buffer, unsigned meta_len, u32 meta_seed,
		u32 *result, unsigned timeout)
428
{
429
	bool write = nvme_is_write(cmd);
430 431
	struct nvme_ns *ns = q->queuedata;
	struct gendisk *disk = ns ? ns->disk : NULL;
432
	struct request *req;
433 434
	struct bio *bio = NULL;
	void *meta = NULL;
435 436
	int ret;

437
	req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
438 439 440 441 442 443
	if (IS_ERR(req))
		return PTR_ERR(req);

	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;

	if (ubuffer && bufflen) {
444 445 446 447 448 449
		ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
				GFP_KERNEL);
		if (ret)
			goto out;
		bio = req->bio;

450 451 452 453 454 455 456 457
		if (!disk)
			goto submit;
		bio->bi_bdev = bdget_disk(disk, 0);
		if (!bio->bi_bdev) {
			ret = -ENODEV;
			goto out_unmap;
		}

458
		if (meta_buffer && meta_len) {
459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475
			struct bio_integrity_payload *bip;

			meta = kmalloc(meta_len, GFP_KERNEL);
			if (!meta) {
				ret = -ENOMEM;
				goto out_unmap;
			}

			if (write) {
				if (copy_from_user(meta, meta_buffer,
						meta_len)) {
					ret = -EFAULT;
					goto out_free_meta;
				}
			}

			bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
476 477
			if (IS_ERR(bip)) {
				ret = PTR_ERR(bip);
478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494
				goto out_free_meta;
			}

			bip->bip_iter.bi_size = meta_len;
			bip->bip_iter.bi_sector = meta_seed;

			ret = bio_integrity_add_page(bio, virt_to_page(meta),
					meta_len, offset_in_page(meta));
			if (ret != meta_len) {
				ret = -ENOMEM;
				goto out_free_meta;
			}
		}
	}
 submit:
	blk_execute_rq(req->q, disk, req, 0);
	ret = req->errors;
495
	if (result)
496
		*result = le32_to_cpu(nvme_req(req)->result.u32);
497 498 499 500 501 502 503 504 505 506 507 508
	if (meta && !ret && !write) {
		if (copy_to_user(meta_buffer, meta, meta_len))
			ret = -EFAULT;
	}
 out_free_meta:
	kfree(meta);
 out_unmap:
	if (bio) {
		if (disk && bio->bi_bdev)
			bdput(bio->bi_bdev);
		blk_rq_unmap_user(bio);
	}
509 510 511 512 513
 out:
	blk_mq_free_request(req);
	return ret;
}

514 515 516 517 518 519 520 521
int nvme_submit_user_cmd(struct request_queue *q, struct nvme_command *cmd,
		void __user *ubuffer, unsigned bufflen, u32 *result,
		unsigned timeout)
{
	return __nvme_submit_user_cmd(q, cmd, ubuffer, bufflen, NULL, 0, 0,
			result, timeout);
}

S
Sagi Grimberg 已提交
522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589
static void nvme_keep_alive_end_io(struct request *rq, int error)
{
	struct nvme_ctrl *ctrl = rq->end_io_data;

	blk_mq_free_request(rq);

	if (error) {
		dev_err(ctrl->device,
			"failed nvme_keep_alive_end_io error=%d\n", error);
		return;
	}

	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
}

static int nvme_keep_alive(struct nvme_ctrl *ctrl)
{
	struct nvme_command c;
	struct request *rq;

	memset(&c, 0, sizeof(c));
	c.common.opcode = nvme_admin_keep_alive;

	rq = nvme_alloc_request(ctrl->admin_q, &c, BLK_MQ_REQ_RESERVED,
			NVME_QID_ANY);
	if (IS_ERR(rq))
		return PTR_ERR(rq);

	rq->timeout = ctrl->kato * HZ;
	rq->end_io_data = ctrl;

	blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);

	return 0;
}

static void nvme_keep_alive_work(struct work_struct *work)
{
	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
			struct nvme_ctrl, ka_work);

	if (nvme_keep_alive(ctrl)) {
		/* allocation failure, reset the controller */
		dev_err(ctrl->device, "keep-alive failed\n");
		ctrl->ops->reset_ctrl(ctrl);
		return;
	}
}

void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
{
	if (unlikely(ctrl->kato == 0))
		return;

	INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
}
EXPORT_SYMBOL_GPL(nvme_start_keep_alive);

void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
{
	if (unlikely(ctrl->kato == 0))
		return;

	cancel_delayed_work_sync(&ctrl->ka_work);
}
EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);

590
int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
591 592 593 594 595 596
{
	struct nvme_command c = { };
	int error;

	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
	c.identify.opcode = nvme_admin_identify;
597
	c.identify.cns = NVME_ID_CNS_CTRL;
598 599 600 601 602 603 604 605 606 607 608 609

	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
	if (!*id)
		return -ENOMEM;

	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
			sizeof(struct nvme_id_ctrl));
	if (error)
		kfree(*id);
	return error;
}

610 611 612 613 614
static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
{
	struct nvme_command c = { };

	c.identify.opcode = nvme_admin_identify;
615
	c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
616 617 618 619
	c.identify.nsid = cpu_to_le32(nsid);
	return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list, 0x1000);
}

620
int nvme_identify_ns(struct nvme_ctrl *dev, unsigned nsid,
621 622 623 624 625 626
		struct nvme_id_ns **id)
{
	struct nvme_command c = { };
	int error;

	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
627 628
	c.identify.opcode = nvme_admin_identify;
	c.identify.nsid = cpu_to_le32(nsid);
629
	c.identify.cns = NVME_ID_CNS_NS;
630 631 632 633 634 635 636 637 638 639 640 641

	*id = kmalloc(sizeof(struct nvme_id_ns), GFP_KERNEL);
	if (!*id)
		return -ENOMEM;

	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
			sizeof(struct nvme_id_ns));
	if (error)
		kfree(*id);
	return error;
}

642
int nvme_get_features(struct nvme_ctrl *dev, unsigned fid, unsigned nsid,
643
		      void *buffer, size_t buflen, u32 *result)
644 645
{
	struct nvme_command c;
646
	union nvme_result res;
647
	int ret;
648 649 650 651 652 653

	memset(&c, 0, sizeof(c));
	c.features.opcode = nvme_admin_get_features;
	c.features.nsid = cpu_to_le32(nsid);
	c.features.fid = cpu_to_le32(fid);

654
	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res, buffer, buflen, 0,
655
			NVME_QID_ANY, 0, 0);
656
	if (ret >= 0 && result)
657
		*result = le32_to_cpu(res.u32);
658
	return ret;
659 660
}

661
int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
662
		      void *buffer, size_t buflen, u32 *result)
663 664
{
	struct nvme_command c;
665
	union nvme_result res;
666
	int ret;
667 668 669 670 671 672

	memset(&c, 0, sizeof(c));
	c.features.opcode = nvme_admin_set_features;
	c.features.fid = cpu_to_le32(fid);
	c.features.dword11 = cpu_to_le32(dword11);

673
	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
674
			buffer, buflen, 0, NVME_QID_ANY, 0, 0);
675
	if (ret >= 0 && result)
676
		*result = le32_to_cpu(res.u32);
677
	return ret;
678 679
}

680
int nvme_get_log_page(struct nvme_ctrl *dev, struct nvme_smart_log **log)
681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700
{
	struct nvme_command c = { };
	int error;

	c.common.opcode = nvme_admin_get_log_page,
	c.common.nsid = cpu_to_le32(0xFFFFFFFF),
	c.common.cdw10[0] = cpu_to_le32(
			(((sizeof(struct nvme_smart_log) / 4) - 1) << 16) |
			 NVME_LOG_SMART),

	*log = kmalloc(sizeof(struct nvme_smart_log), GFP_KERNEL);
	if (!*log)
		return -ENOMEM;

	error = nvme_submit_sync_cmd(dev->admin_q, &c, *log,
			sizeof(struct nvme_smart_log));
	if (error)
		kfree(*log);
	return error;
}
701

C
Christoph Hellwig 已提交
702 703 704 705 706 707
int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
{
	u32 q_count = (*count - 1) | ((*count - 1) << 16);
	u32 result;
	int status, nr_io_queues;

708
	status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
C
Christoph Hellwig 已提交
709
			&result);
710
	if (status < 0)
C
Christoph Hellwig 已提交
711 712
		return status;

713 714 715 716 717 718 719 720 721 722 723 724 725
	/*
	 * Degraded controllers might return an error when setting the queue
	 * count.  We still want to be able to bring them online and offer
	 * access to the admin queue, as that might be only way to fix them up.
	 */
	if (status > 0) {
		dev_err(ctrl->dev, "Could not set queue count (%d)\n", status);
		*count = 0;
	} else {
		nr_io_queues = min(result & 0xffff, result >> 16) + 1;
		*count = min(*count, nr_io_queues);
	}

C
Christoph Hellwig 已提交
726 727
	return 0;
}
728
EXPORT_SYMBOL_GPL(nvme_set_queue_count);
C
Christoph Hellwig 已提交
729

730 731 732 733 734 735 736 737 738
static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
{
	struct nvme_user_io io;
	struct nvme_command c;
	unsigned length, meta_len;
	void __user *metadata;

	if (copy_from_user(&io, uio, sizeof(io)))
		return -EFAULT;
739 740
	if (io.flags)
		return -EINVAL;
741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779

	switch (io.opcode) {
	case nvme_cmd_write:
	case nvme_cmd_read:
	case nvme_cmd_compare:
		break;
	default:
		return -EINVAL;
	}

	length = (io.nblocks + 1) << ns->lba_shift;
	meta_len = (io.nblocks + 1) * ns->ms;
	metadata = (void __user *)(uintptr_t)io.metadata;

	if (ns->ext) {
		length += meta_len;
		meta_len = 0;
	} else if (meta_len) {
		if ((io.metadata & 3) || !io.metadata)
			return -EINVAL;
	}

	memset(&c, 0, sizeof(c));
	c.rw.opcode = io.opcode;
	c.rw.flags = io.flags;
	c.rw.nsid = cpu_to_le32(ns->ns_id);
	c.rw.slba = cpu_to_le64(io.slba);
	c.rw.length = cpu_to_le16(io.nblocks);
	c.rw.control = cpu_to_le16(io.control);
	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
	c.rw.reftag = cpu_to_le32(io.reftag);
	c.rw.apptag = cpu_to_le16(io.apptag);
	c.rw.appmask = cpu_to_le16(io.appmask);

	return __nvme_submit_user_cmd(ns->queue, &c,
			(void __user *)(uintptr_t)io.addr, length,
			metadata, meta_len, io.slba, NULL, 0);
}

780
static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
781 782 783 784 785 786 787 788 789 790 791
			struct nvme_passthru_cmd __user *ucmd)
{
	struct nvme_passthru_cmd cmd;
	struct nvme_command c;
	unsigned timeout = 0;
	int status;

	if (!capable(CAP_SYS_ADMIN))
		return -EACCES;
	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
		return -EFAULT;
792 793
	if (cmd.flags)
		return -EINVAL;
794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811

	memset(&c, 0, sizeof(c));
	c.common.opcode = cmd.opcode;
	c.common.flags = cmd.flags;
	c.common.nsid = cpu_to_le32(cmd.nsid);
	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
	c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
	c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
	c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
	c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
	c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
	c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);

	if (cmd.timeout_ms)
		timeout = msecs_to_jiffies(cmd.timeout_ms);

	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
812
			(void __user *)(uintptr_t)cmd.addr, cmd.data_len,
813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
			&cmd.result, timeout);
	if (status >= 0) {
		if (put_user(cmd.result, &ucmd->result))
			return -EFAULT;
	}

	return status;
}

static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
		unsigned int cmd, unsigned long arg)
{
	struct nvme_ns *ns = bdev->bd_disk->private_data;

	switch (cmd) {
	case NVME_IOCTL_ID:
		force_successful_syscall_return();
		return ns->ns_id;
	case NVME_IOCTL_ADMIN_CMD:
		return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
	case NVME_IOCTL_IO_CMD:
		return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
	case NVME_IOCTL_SUBMIT_IO:
		return nvme_submit_io(ns, (void __user *)arg);
837
#ifdef CONFIG_BLK_DEV_NVME_SCSI
838 839 840 841
	case SG_GET_VERSION_NUM:
		return nvme_sg_get_version_num((void __user *)arg);
	case SG_IO:
		return nvme_sg_io(ns, (void __user *)arg);
842
#endif
843
	default:
844 845 846 847
#ifdef CONFIG_NVM
		if (ns->ndev)
			return nvme_nvm_ioctl(ns, cmd, arg);
#endif
848
		if (is_sed_ioctl(cmd))
849
			return sed_ioctl(ns->ctrl->opal_dev, cmd,
850
					 (void __user *) arg);
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875
		return -ENOTTY;
	}
}

#ifdef CONFIG_COMPAT
static int nvme_compat_ioctl(struct block_device *bdev, fmode_t mode,
			unsigned int cmd, unsigned long arg)
{
	switch (cmd) {
	case SG_IO:
		return -ENOIOCTLCMD;
	}
	return nvme_ioctl(bdev, mode, cmd, arg);
}
#else
#define nvme_compat_ioctl	NULL
#endif

static int nvme_open(struct block_device *bdev, fmode_t mode)
{
	return nvme_get_ns_from_disk(bdev->bd_disk) ? 0 : -ENXIO;
}

static void nvme_release(struct gendisk *disk, fmode_t mode)
{
876 877 878 879
	struct nvme_ns *ns = disk->private_data;

	module_put(ns->ctrl->ops->module);
	nvme_put_ns(ns);
880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895
}

static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	/* some standard values */
	geo->heads = 1 << 6;
	geo->sectors = 1 << 5;
	geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
	return 0;
}

#ifdef CONFIG_BLK_DEV_INTEGRITY
static void nvme_init_integrity(struct nvme_ns *ns)
{
	struct blk_integrity integrity;

896
	memset(&integrity, 0, sizeof(integrity));
897 898 899
	switch (ns->pi_type) {
	case NVME_NS_DPS_PI_TYPE3:
		integrity.profile = &t10_pi_type3_crc;
900 901
		integrity.tag_size = sizeof(u16) + sizeof(u32);
		integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
902 903 904 905
		break;
	case NVME_NS_DPS_PI_TYPE1:
	case NVME_NS_DPS_PI_TYPE2:
		integrity.profile = &t10_pi_type1_crc;
906 907
		integrity.tag_size = sizeof(u16);
		integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924
		break;
	default:
		integrity.profile = NULL;
		break;
	}
	integrity.tuple_size = ns->ms;
	blk_integrity_register(ns->disk, &integrity);
	blk_queue_max_integrity_segments(ns->queue, 1);
}
#else
static void nvme_init_integrity(struct nvme_ns *ns)
{
}
#endif /* CONFIG_BLK_DEV_INTEGRITY */

static void nvme_config_discard(struct nvme_ns *ns)
{
925
	struct nvme_ctrl *ctrl = ns->ctrl;
926
	u32 logical_block_size = queue_logical_block_size(ns->queue);
927

928 929 930
	BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
			NVME_DSM_MAX_RANGES);

931 932 933 934 935
	if (ctrl->quirks & NVME_QUIRK_DISCARD_ZEROES)
		ns->queue->limits.discard_zeroes_data = 1;
	else
		ns->queue->limits.discard_zeroes_data = 0;

936 937
	ns->queue->limits.discard_alignment = logical_block_size;
	ns->queue->limits.discard_granularity = logical_block_size;
938
	blk_queue_max_discard_sectors(ns->queue, UINT_MAX);
939
	blk_queue_max_discard_segments(ns->queue, NVME_DSM_MAX_RANGES);
940 941 942
	queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, ns->queue);
}

943
static int nvme_revalidate_ns(struct nvme_ns *ns, struct nvme_id_ns **id)
944
{
945
	if (nvme_identify_ns(ns->ctrl, ns->ns_id, id)) {
946
		dev_warn(ns->ctrl->dev, "%s: Identify failure\n", __func__);
947 948 949
		return -ENODEV;
	}

950 951 952
	if ((*id)->ncap == 0) {
		kfree(*id);
		return -ENODEV;
953 954
	}

955
	if (ns->ctrl->vs >= NVME_VS(1, 1, 0))
956
		memcpy(ns->eui, (*id)->eui64, sizeof(ns->eui));
957
	if (ns->ctrl->vs >= NVME_VS(1, 2, 0))
958 959 960 961 962 963 964 965 966 967 968
		memcpy(ns->uuid, (*id)->nguid, sizeof(ns->uuid));

	return 0;
}

static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
{
	struct nvme_ns *ns = disk->private_data;
	u8 lbaf, pi_type;
	u16 old_ms;
	unsigned short bs;
969

970 971 972 973 974 975 976 977 978 979 980 981 982 983 984 985 986 987 988 989 990 991 992 993 994 995 996
	old_ms = ns->ms;
	lbaf = id->flbas & NVME_NS_FLBAS_LBA_MASK;
	ns->lba_shift = id->lbaf[lbaf].ds;
	ns->ms = le16_to_cpu(id->lbaf[lbaf].ms);
	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);

	/*
	 * If identify namespace failed, use default 512 byte block size so
	 * block layer can use before failing read/write for 0 capacity.
	 */
	if (ns->lba_shift == 0)
		ns->lba_shift = 9;
	bs = 1 << ns->lba_shift;
	/* XXX: PI implementation requires metadata equal t10 pi tuple size */
	pi_type = ns->ms == sizeof(struct t10_pi_tuple) ?
					id->dps & NVME_NS_DPS_PI_MASK : 0;

	blk_mq_freeze_queue(disk->queue);
	if (blk_get_integrity(disk) && (ns->pi_type != pi_type ||
				ns->ms != old_ms ||
				bs != queue_logical_block_size(disk->queue) ||
				(ns->ms && ns->ext)))
		blk_integrity_unregister(disk);

	ns->pi_type = pi_type;
	blk_queue_logical_block_size(ns->queue, bs);

K
Keith Busch 已提交
997
	if (ns->ms && !blk_get_integrity(disk) && !ns->ext)
998 999 1000 1001 1002 1003 1004 1005 1006
		nvme_init_integrity(ns);
	if (ns->ms && !(ns->ms == 8 && ns->pi_type) && !blk_get_integrity(disk))
		set_capacity(disk, 0);
	else
		set_capacity(disk, le64_to_cpup(&id->nsze) << (ns->lba_shift - 9));

	if (ns->ctrl->oncs & NVME_CTRL_ONCS_DSM)
		nvme_config_discard(ns);
	blk_mq_unfreeze_queue(disk->queue);
1007
}
1008

1009 1010 1011 1012 1013 1014 1015 1016 1017 1018 1019 1020 1021 1022 1023 1024
static int nvme_revalidate_disk(struct gendisk *disk)
{
	struct nvme_ns *ns = disk->private_data;
	struct nvme_id_ns *id = NULL;
	int ret;

	if (test_bit(NVME_NS_DEAD, &ns->flags)) {
		set_capacity(disk, 0);
		return -ENODEV;
	}

	ret = nvme_revalidate_ns(ns, &id);
	if (ret)
		return ret;

	__nvme_revalidate_disk(disk, id);
1025
	kfree(id);
1026

1027 1028 1029 1030 1031 1032 1033 1034 1035 1036 1037 1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103
	return 0;
}

static char nvme_pr_type(enum pr_type type)
{
	switch (type) {
	case PR_WRITE_EXCLUSIVE:
		return 1;
	case PR_EXCLUSIVE_ACCESS:
		return 2;
	case PR_WRITE_EXCLUSIVE_REG_ONLY:
		return 3;
	case PR_EXCLUSIVE_ACCESS_REG_ONLY:
		return 4;
	case PR_WRITE_EXCLUSIVE_ALL_REGS:
		return 5;
	case PR_EXCLUSIVE_ACCESS_ALL_REGS:
		return 6;
	default:
		return 0;
	}
};

static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
				u64 key, u64 sa_key, u8 op)
{
	struct nvme_ns *ns = bdev->bd_disk->private_data;
	struct nvme_command c;
	u8 data[16] = { 0, };

	put_unaligned_le64(key, &data[0]);
	put_unaligned_le64(sa_key, &data[8]);

	memset(&c, 0, sizeof(c));
	c.common.opcode = op;
	c.common.nsid = cpu_to_le32(ns->ns_id);
	c.common.cdw10[0] = cpu_to_le32(cdw10);

	return nvme_submit_sync_cmd(ns->queue, &c, data, 16);
}

static int nvme_pr_register(struct block_device *bdev, u64 old,
		u64 new, unsigned flags)
{
	u32 cdw10;

	if (flags & ~PR_FL_IGNORE_KEY)
		return -EOPNOTSUPP;

	cdw10 = old ? 2 : 0;
	cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
	cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
}

static int nvme_pr_reserve(struct block_device *bdev, u64 key,
		enum pr_type type, unsigned flags)
{
	u32 cdw10;

	if (flags & ~PR_FL_IGNORE_KEY)
		return -EOPNOTSUPP;

	cdw10 = nvme_pr_type(type) << 8;
	cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
}

static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
		enum pr_type type, bool abort)
{
	u32 cdw10 = nvme_pr_type(type) << 8 | abort ? 2 : 1;
	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
}

static int nvme_pr_clear(struct block_device *bdev, u64 key)
{
1104
	u32 cdw10 = 1 | (key ? 1 << 3 : 0);
1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
}

static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
{
	u32 cdw10 = nvme_pr_type(type) << 8 | key ? 1 << 3 : 0;
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
}

static const struct pr_ops nvme_pr_ops = {
	.pr_register	= nvme_pr_register,
	.pr_reserve	= nvme_pr_reserve,
	.pr_release	= nvme_pr_release,
	.pr_preempt	= nvme_pr_preempt,
	.pr_clear	= nvme_pr_clear,
};

1122
#ifdef CONFIG_BLK_SED_OPAL
1123 1124
int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
		bool send)
1125
{
1126
	struct nvme_ctrl *ctrl = data;
1127 1128 1129 1130 1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143
	struct nvme_command cmd;

	memset(&cmd, 0, sizeof(cmd));
	if (send)
		cmd.common.opcode = nvme_admin_security_send;
	else
		cmd.common.opcode = nvme_admin_security_recv;
	cmd.common.nsid = 0;
	cmd.common.cdw10[0] = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
	cmd.common.cdw10[1] = cpu_to_le32(len);

	return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
				      ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0);
}
EXPORT_SYMBOL_GPL(nvme_sec_submit);
#endif /* CONFIG_BLK_SED_OPAL */

1144
static const struct block_device_operations nvme_fops = {
1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
	.owner		= THIS_MODULE,
	.ioctl		= nvme_ioctl,
	.compat_ioctl	= nvme_compat_ioctl,
	.open		= nvme_open,
	.release	= nvme_release,
	.getgeo		= nvme_getgeo,
	.revalidate_disk= nvme_revalidate_disk,
	.pr_ops		= &nvme_pr_ops,
};

1155 1156 1157 1158 1159 1160 1161 1162
static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
{
	unsigned long timeout =
		((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
	u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
	int ret;

	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
K
Keith Busch 已提交
1163 1164
		if (csts == ~0)
			return -ENODEV;
1165 1166 1167 1168 1169 1170 1171
		if ((csts & NVME_CSTS_RDY) == bit)
			break;

		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
1172
			dev_err(ctrl->device,
1173 1174 1175 1176 1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193 1194 1195 1196 1197
				"Device not ready; aborting %s\n", enabled ?
						"initialisation" : "reset");
			return -ENODEV;
		}
	}

	return ret;
}

/*
 * If the device has been passed off to us in an enabled state, just clear
 * the enabled bit.  The spec says we should set the 'shutdown notification
 * bits', but doing so may cause the device to complete commands to the
 * admin queue ... and we don't know what memory that might be pointing at!
 */
int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
{
	int ret;

	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
	ctrl->ctrl_config &= ~NVME_CC_ENABLE;

	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
	if (ret)
		return ret;
1198

1199
	if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
1200 1201
		msleep(NVME_QUIRK_DELAY_AMOUNT);

1202 1203
	return nvme_wait_ready(ctrl, cap, false);
}
1204
EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
1205 1206 1207 1208 1209 1210 1211 1212 1213 1214 1215 1216

int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
{
	/*
	 * Default to a 4K page size, with the intention to update this
	 * path in the future to accomodate architectures with differing
	 * kernel and IO page sizes.
	 */
	unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
	int ret;

	if (page_shift < dev_page_min) {
1217
		dev_err(ctrl->device,
1218 1219 1220 1221 1222 1223 1224 1225 1226 1227 1228 1229 1230 1231 1232 1233 1234 1235
			"Minimum device page size %u too large for host (%u)\n",
			1 << dev_page_min, 1 << page_shift);
		return -ENODEV;
	}

	ctrl->page_size = 1 << page_shift;

	ctrl->ctrl_config = NVME_CC_CSS_NVM;
	ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
	ctrl->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE;
	ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
	ctrl->ctrl_config |= NVME_CC_ENABLE;

	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
	if (ret)
		return ret;
	return nvme_wait_ready(ctrl, cap, true);
}
1236
EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
1237 1238 1239 1240 1241 1242 1243 1244 1245 1246 1247 1248 1249 1250 1251 1252 1253 1254 1255 1256 1257 1258

int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
{
	unsigned long timeout = SHUTDOWN_TIMEOUT + jiffies;
	u32 csts;
	int ret;

	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
	ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;

	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
	if (ret)
		return ret;

	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
		if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
			break;

		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
1259
			dev_err(ctrl->device,
1260 1261 1262 1263 1264 1265 1266
				"Device shutdown incomplete; abort shutdown\n");
			return -ENODEV;
		}
	}

	return ret;
}
1267
EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
1268

1269 1270 1271
static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
		struct request_queue *q)
{
1272 1273
	bool vwc = false;

1274
	if (ctrl->max_hw_sectors) {
1275 1276 1277
		u32 max_segments =
			(ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;

1278
		blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1279
		blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1280
	}
K
Keith Busch 已提交
1281 1282
	if (ctrl->quirks & NVME_QUIRK_STRIPE_SIZE)
		blk_queue_chunk_sectors(q, ctrl->max_hw_sectors);
1283
	blk_queue_virt_boundary(q, ctrl->page_size - 1);
1284 1285 1286
	if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
		vwc = true;
	blk_queue_write_cache(q, vwc, vwc);
1287 1288
}

1289 1290 1291 1292 1293 1294 1295 1296 1297 1298 1299 1300 1301 1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313 1314 1315 1316 1317 1318 1319 1320 1321 1322 1323 1324 1325 1326 1327 1328 1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349 1350 1351 1352 1353 1354 1355 1356 1357 1358 1359 1360 1361 1362 1363 1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400 1401 1402 1403 1404
static void nvme_configure_apst(struct nvme_ctrl *ctrl)
{
	/*
	 * APST (Autonomous Power State Transition) lets us program a
	 * table of power state transitions that the controller will
	 * perform automatically.  We configure it with a simple
	 * heuristic: we are willing to spend at most 2% of the time
	 * transitioning between power states.  Therefore, when running
	 * in any given state, we will enter the next lower-power
	 * non-operational state after waiting 100 * (enlat + exlat)
	 * microseconds, as long as that state's total latency is under
	 * the requested maximum latency.
	 *
	 * We will not autonomously enter any non-operational state for
	 * which the total latency exceeds ps_max_latency_us.  Users
	 * can set ps_max_latency_us to zero to turn off APST.
	 */

	unsigned apste;
	struct nvme_feat_auto_pst *table;
	int ret;

	/*
	 * If APST isn't supported or if we haven't been initialized yet,
	 * then don't do anything.
	 */
	if (!ctrl->apsta)
		return;

	if (ctrl->npss > 31) {
		dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
		return;
	}

	table = kzalloc(sizeof(*table), GFP_KERNEL);
	if (!table)
		return;

	if (ctrl->ps_max_latency_us == 0) {
		/* Turn off APST. */
		apste = 0;
	} else {
		__le64 target = cpu_to_le64(0);
		int state;

		/*
		 * Walk through all states from lowest- to highest-power.
		 * According to the spec, lower-numbered states use more
		 * power.  NPSS, despite the name, is the index of the
		 * lowest-power state, not the number of states.
		 */
		for (state = (int)ctrl->npss; state >= 0; state--) {
			u64 total_latency_us, transition_ms;

			if (target)
				table->entries[state] = target;

			/*
			 * Is this state a useful non-operational state for
			 * higher-power states to autonomously transition to?
			 */
			if (!(ctrl->psd[state].flags &
			      NVME_PS_FLAGS_NON_OP_STATE))
				continue;

			total_latency_us =
				(u64)le32_to_cpu(ctrl->psd[state].entry_lat) +
				+ le32_to_cpu(ctrl->psd[state].exit_lat);
			if (total_latency_us > ctrl->ps_max_latency_us)
				continue;

			/*
			 * This state is good.  Use it as the APST idle
			 * target for higher power states.
			 */
			transition_ms = total_latency_us + 19;
			do_div(transition_ms, 20);
			if (transition_ms > (1 << 24) - 1)
				transition_ms = (1 << 24) - 1;

			target = cpu_to_le64((state << 3) |
					     (transition_ms << 8));
		}

		apste = 1;
	}

	ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
				table, sizeof(*table), NULL);
	if (ret)
		dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);

	kfree(table);
}

static void nvme_set_latency_tolerance(struct device *dev, s32 val)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	u64 latency;

	switch (val) {
	case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
	case PM_QOS_LATENCY_ANY:
		latency = U64_MAX;
		break;

	default:
		latency = val;
	}

	if (ctrl->ps_max_latency_us != latency) {
		ctrl->ps_max_latency_us = latency;
		nvme_configure_apst(ctrl);
	}
}

1405 1406 1407 1408 1409 1410 1411 1412 1413 1414 1415 1416 1417
struct nvme_core_quirk_entry {
	/*
	 * NVMe model and firmware strings are padded with spaces.  For
	 * simplicity, strings in the quirk table are padded with NULLs
	 * instead.
	 */
	u16 vid;
	const char *mn;
	const char *fr;
	unsigned long quirks;
};

static const struct nvme_core_quirk_entry core_quirks[] = {
1418 1419 1420 1421 1422 1423 1424 1425 1426 1427
	/*
	 * Seen on a Samsung "SM951 NVMe SAMSUNG 256GB": using APST causes
	 * the controller to go out to lunch.  It dies when the watchdog
	 * timer reads CSTS and gets 0xffffffff.
	 */
	{
		.vid = 0x144d,
		.fr = "BXW75D0Q",
		.quirks = NVME_QUIRK_NO_APST,
	},
1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438 1439 1440 1441 1442 1443 1444 1445 1446 1447 1448 1449 1450 1451 1452 1453 1454 1455 1456 1457 1458
};

/* match is null-terminated but idstr is space-padded. */
static bool string_matches(const char *idstr, const char *match, size_t len)
{
	size_t matchlen;

	if (!match)
		return true;

	matchlen = strlen(match);
	WARN_ON_ONCE(matchlen > len);

	if (memcmp(idstr, match, matchlen))
		return false;

	for (; matchlen < len; matchlen++)
		if (idstr[matchlen] != ' ')
			return false;

	return true;
}

static bool quirk_matches(const struct nvme_id_ctrl *id,
			  const struct nvme_core_quirk_entry *q)
{
	return q->vid == le16_to_cpu(id->vid) &&
		string_matches(id->mn, q->mn, sizeof(id->mn)) &&
		string_matches(id->fr, q->fr, sizeof(id->fr));
}

1459 1460 1461 1462 1463 1464 1465 1466 1467 1468
/*
 * Initialize the cached copies of the Identify data and various controller
 * register in our nvme_ctrl structure.  This should be called as soon as
 * the admin queue is fully up and running.
 */
int nvme_init_identify(struct nvme_ctrl *ctrl)
{
	struct nvme_id_ctrl *id;
	u64 cap;
	int ret, page_shift;
1469
	u32 max_hw_sectors;
1470
	u8 prev_apsta;
1471

1472 1473
	ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
	if (ret) {
1474
		dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
1475 1476 1477
		return ret;
	}

1478 1479
	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
	if (ret) {
1480
		dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
1481 1482 1483 1484
		return ret;
	}
	page_shift = NVME_CAP_MPSMIN(cap) + 12;

1485
	if (ctrl->vs >= NVME_VS(1, 1, 0))
1486 1487
		ctrl->subsystem = NVME_CAP_NSSRC(cap);

1488 1489
	ret = nvme_identify_ctrl(ctrl, &id);
	if (ret) {
1490
		dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
1491 1492 1493
		return -EIO;
	}

1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511
	if (!ctrl->identified) {
		/*
		 * Check for quirks.  Quirk can depend on firmware version,
		 * so, in principle, the set of quirks present can change
		 * across a reset.  As a possible future enhancement, we
		 * could re-scan for quirks every time we reinitialize
		 * the device, but we'd have to make sure that the driver
		 * behaves intelligently if the quirks change.
		 */

		int i;

		for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
			if (quirk_matches(id, &core_quirks[i]))
				ctrl->quirks |= core_quirks[i].quirks;
		}
	}

1512
	ctrl->oacs = le16_to_cpu(id->oacs);
1513
	ctrl->vid = le16_to_cpu(id->vid);
1514
	ctrl->oncs = le16_to_cpup(&id->oncs);
1515
	atomic_set(&ctrl->abort_limit, id->acl + 1);
1516
	ctrl->vwc = id->vwc;
M
Ming Lin 已提交
1517
	ctrl->cntlid = le16_to_cpup(&id->cntlid);
1518 1519 1520 1521
	memcpy(ctrl->serial, id->sn, sizeof(id->sn));
	memcpy(ctrl->model, id->mn, sizeof(id->mn));
	memcpy(ctrl->firmware_rev, id->fr, sizeof(id->fr));
	if (id->mdts)
1522
		max_hw_sectors = 1 << (id->mdts + page_shift - 9);
1523
	else
1524 1525 1526
		max_hw_sectors = UINT_MAX;
	ctrl->max_hw_sectors =
		min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
1527

1528
	nvme_set_queue_limits(ctrl, ctrl->admin_q);
1529
	ctrl->sgls = le32_to_cpu(id->sgls);
S
Sagi Grimberg 已提交
1530
	ctrl->kas = le16_to_cpu(id->kas);
1531

1532 1533 1534 1535 1536
	ctrl->npss = id->npss;
	prev_apsta = ctrl->apsta;
	ctrl->apsta = (ctrl->quirks & NVME_QUIRK_NO_APST) ? 0 : id->apsta;
	memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));

1537 1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548
	if (ctrl->ops->is_fabrics) {
		ctrl->icdoff = le16_to_cpu(id->icdoff);
		ctrl->ioccsz = le32_to_cpu(id->ioccsz);
		ctrl->iorcsz = le32_to_cpu(id->iorcsz);
		ctrl->maxcmd = le16_to_cpu(id->maxcmd);

		/*
		 * In fabrics we need to verify the cntlid matches the
		 * admin connect
		 */
		if (ctrl->cntlid != le16_to_cpu(id->cntlid))
			ret = -EINVAL;
S
Sagi Grimberg 已提交
1549 1550 1551 1552 1553 1554

		if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
			dev_err(ctrl->dev,
				"keep-alive support is mandatory for fabrics\n");
			ret = -EINVAL;
		}
1555 1556 1557
	} else {
		ctrl->cntlid = le16_to_cpu(id->cntlid);
	}
1558

1559
	kfree(id);
1560

1561 1562 1563 1564 1565 1566 1567
	if (ctrl->apsta && !prev_apsta)
		dev_pm_qos_expose_latency_tolerance(ctrl->device);
	else if (!ctrl->apsta && prev_apsta)
		dev_pm_qos_hide_latency_tolerance(ctrl->device);

	nvme_configure_apst(ctrl);

1568
	ctrl->identified = true;
1569

1570
	return ret;
1571
}
1572
EXPORT_SYMBOL_GPL(nvme_init_identify);
1573

1574
static int nvme_dev_open(struct inode *inode, struct file *file)
1575
{
1576 1577 1578
	struct nvme_ctrl *ctrl;
	int instance = iminor(inode);
	int ret = -ENODEV;
1579

1580 1581 1582 1583 1584 1585 1586 1587 1588 1589 1590 1591 1592 1593 1594 1595 1596 1597
	spin_lock(&dev_list_lock);
	list_for_each_entry(ctrl, &nvme_ctrl_list, node) {
		if (ctrl->instance != instance)
			continue;

		if (!ctrl->admin_q) {
			ret = -EWOULDBLOCK;
			break;
		}
		if (!kref_get_unless_zero(&ctrl->kref))
			break;
		file->private_data = ctrl;
		ret = 0;
		break;
	}
	spin_unlock(&dev_list_lock);

	return ret;
1598 1599
}

1600
static int nvme_dev_release(struct inode *inode, struct file *file)
1601
{
1602 1603 1604 1605
	nvme_put_ctrl(file->private_data);
	return 0;
}

1606 1607 1608 1609 1610 1611 1612 1613 1614 1615 1616 1617 1618
static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
{
	struct nvme_ns *ns;
	int ret;

	mutex_lock(&ctrl->namespaces_mutex);
	if (list_empty(&ctrl->namespaces)) {
		ret = -ENOTTY;
		goto out_unlock;
	}

	ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
	if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
1619
		dev_warn(ctrl->device,
1620 1621 1622 1623 1624
			"NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
		ret = -EINVAL;
		goto out_unlock;
	}

1625
	dev_warn(ctrl->device,
1626 1627 1628 1629 1630 1631 1632 1633 1634 1635 1636 1637 1638
		"using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
	kref_get(&ns->kref);
	mutex_unlock(&ctrl->namespaces_mutex);

	ret = nvme_user_cmd(ctrl, ns, argp);
	nvme_put_ns(ns);
	return ret;

out_unlock:
	mutex_unlock(&ctrl->namespaces_mutex);
	return ret;
}

1639 1640 1641 1642 1643 1644 1645 1646 1647 1648
static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
		unsigned long arg)
{
	struct nvme_ctrl *ctrl = file->private_data;
	void __user *argp = (void __user *)arg;

	switch (cmd) {
	case NVME_IOCTL_ADMIN_CMD:
		return nvme_user_cmd(ctrl, NULL, argp);
	case NVME_IOCTL_IO_CMD:
1649
		return nvme_dev_user_cmd(ctrl, argp);
1650
	case NVME_IOCTL_RESET:
1651
		dev_warn(ctrl->device, "resetting controller\n");
1652 1653 1654
		return ctrl->ops->reset_ctrl(ctrl);
	case NVME_IOCTL_SUBSYS_RESET:
		return nvme_reset_subsystem(ctrl);
K
Keith Busch 已提交
1655 1656 1657
	case NVME_IOCTL_RESCAN:
		nvme_queue_scan(ctrl);
		return 0;
1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678 1679 1680 1681
	default:
		return -ENOTTY;
	}
}

static const struct file_operations nvme_dev_fops = {
	.owner		= THIS_MODULE,
	.open		= nvme_dev_open,
	.release	= nvme_dev_release,
	.unlocked_ioctl	= nvme_dev_ioctl,
	.compat_ioctl	= nvme_dev_ioctl,
};

static ssize_t nvme_sysfs_reset(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	int ret;

	ret = ctrl->ops->reset_ctrl(ctrl);
	if (ret < 0)
		return ret;
	return count;
1682
}
1683
static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
1684

K
Keith Busch 已提交
1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695
static ssize_t nvme_sysfs_rescan(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	nvme_queue_scan(ctrl);
	return count;
}
static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);

1696 1697 1698
static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
								char *buf)
{
1699
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1700 1701 1702 1703 1704 1705 1706 1707 1708 1709 1710 1711 1712 1713 1714 1715 1716 1717 1718 1719
	struct nvme_ctrl *ctrl = ns->ctrl;
	int serial_len = sizeof(ctrl->serial);
	int model_len = sizeof(ctrl->model);

	if (memchr_inv(ns->uuid, 0, sizeof(ns->uuid)))
		return sprintf(buf, "eui.%16phN\n", ns->uuid);

	if (memchr_inv(ns->eui, 0, sizeof(ns->eui)))
		return sprintf(buf, "eui.%8phN\n", ns->eui);

	while (ctrl->serial[serial_len - 1] == ' ')
		serial_len--;
	while (ctrl->model[model_len - 1] == ' ')
		model_len--;

	return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", ctrl->vid,
		serial_len, ctrl->serial, model_len, ctrl->model, ns->ns_id);
}
static DEVICE_ATTR(wwid, S_IRUGO, wwid_show, NULL);

1720 1721 1722
static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
								char *buf)
{
1723
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1724 1725 1726 1727 1728 1729 1730
	return sprintf(buf, "%pU\n", ns->uuid);
}
static DEVICE_ATTR(uuid, S_IRUGO, uuid_show, NULL);

static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
								char *buf)
{
1731
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1732 1733 1734 1735 1736 1737 1738
	return sprintf(buf, "%8phd\n", ns->eui);
}
static DEVICE_ATTR(eui, S_IRUGO, eui_show, NULL);

static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
								char *buf)
{
1739
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1740 1741 1742 1743 1744
	return sprintf(buf, "%d\n", ns->ns_id);
}
static DEVICE_ATTR(nsid, S_IRUGO, nsid_show, NULL);

static struct attribute *nvme_ns_attrs[] = {
1745
	&dev_attr_wwid.attr,
1746 1747 1748 1749 1750 1751
	&dev_attr_uuid.attr,
	&dev_attr_eui.attr,
	&dev_attr_nsid.attr,
	NULL,
};

M
Ming Lin 已提交
1752
static umode_t nvme_ns_attrs_are_visible(struct kobject *kobj,
1753 1754 1755
		struct attribute *a, int n)
{
	struct device *dev = container_of(kobj, struct device, kobj);
1756
	struct nvme_ns *ns = nvme_get_ns_from_dev(dev);
1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769 1770

	if (a == &dev_attr_uuid.attr) {
		if (!memchr_inv(ns->uuid, 0, sizeof(ns->uuid)))
			return 0;
	}
	if (a == &dev_attr_eui.attr) {
		if (!memchr_inv(ns->eui, 0, sizeof(ns->eui)))
			return 0;
	}
	return a->mode;
}

static const struct attribute_group nvme_ns_attr_group = {
	.attrs		= nvme_ns_attrs,
M
Ming Lin 已提交
1771
	.is_visible	= nvme_ns_attrs_are_visible,
1772 1773
};

M
Ming Lin 已提交
1774
#define nvme_show_str_function(field)						\
1775 1776 1777 1778 1779 1780 1781 1782
static ssize_t  field##_show(struct device *dev,				\
			    struct device_attribute *attr, char *buf)		\
{										\
        struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
        return sprintf(buf, "%.*s\n", (int)sizeof(ctrl->field), ctrl->field);	\
}										\
static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);

M
Ming Lin 已提交
1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795
#define nvme_show_int_function(field)						\
static ssize_t  field##_show(struct device *dev,				\
			    struct device_attribute *attr, char *buf)		\
{										\
        struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
        return sprintf(buf, "%d\n", ctrl->field);	\
}										\
static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);

nvme_show_str_function(model);
nvme_show_str_function(serial);
nvme_show_str_function(firmware_rev);
nvme_show_int_function(cntlid);
1796

M
Ming Lin 已提交
1797 1798 1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815 1816 1817 1818
static ssize_t nvme_sysfs_delete(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	if (device_remove_file_self(dev, attr))
		ctrl->ops->delete_ctrl(ctrl);
	return count;
}
static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);

static ssize_t nvme_sysfs_show_transport(struct device *dev,
					 struct device_attribute *attr,
					 char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
}
static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);

1819 1820 1821 1822 1823 1824 1825 1826 1827 1828 1829 1830 1831 1832 1833 1834 1835 1836 1837 1838 1839 1840 1841
static ssize_t nvme_sysfs_show_state(struct device *dev,
				     struct device_attribute *attr,
				     char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	static const char *const state_name[] = {
		[NVME_CTRL_NEW]		= "new",
		[NVME_CTRL_LIVE]	= "live",
		[NVME_CTRL_RESETTING]	= "resetting",
		[NVME_CTRL_RECONNECTING]= "reconnecting",
		[NVME_CTRL_DELETING]	= "deleting",
		[NVME_CTRL_DEAD]	= "dead",
	};

	if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
	    state_name[ctrl->state])
		return sprintf(buf, "%s\n", state_name[ctrl->state]);

	return sprintf(buf, "unknown state\n");
}

static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);

M
Ming Lin 已提交
1842 1843 1844 1845 1846 1847 1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859 1860 1861 1862
static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
					 struct device_attribute *attr,
					 char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	return snprintf(buf, PAGE_SIZE, "%s\n",
			ctrl->ops->get_subsysnqn(ctrl));
}
static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);

static ssize_t nvme_sysfs_show_address(struct device *dev,
					 struct device_attribute *attr,
					 char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
}
static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);

1863 1864
static struct attribute *nvme_dev_attrs[] = {
	&dev_attr_reset_controller.attr,
K
Keith Busch 已提交
1865
	&dev_attr_rescan_controller.attr,
1866 1867 1868
	&dev_attr_model.attr,
	&dev_attr_serial.attr,
	&dev_attr_firmware_rev.attr,
M
Ming Lin 已提交
1869
	&dev_attr_cntlid.attr,
M
Ming Lin 已提交
1870 1871 1872 1873
	&dev_attr_delete_controller.attr,
	&dev_attr_transport.attr,
	&dev_attr_subsysnqn.attr,
	&dev_attr_address.attr,
1874
	&dev_attr_state.attr,
1875 1876 1877
	NULL
};

M
Ming Lin 已提交
1878 1879 1880 1881 1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893 1894 1895 1896 1897 1898 1899
#define CHECK_ATTR(ctrl, a, name)		\
	if ((a) == &dev_attr_##name.attr &&	\
	    !(ctrl)->ops->get_##name)		\
		return 0

static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
		struct attribute *a, int n)
{
	struct device *dev = container_of(kobj, struct device, kobj);
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	if (a == &dev_attr_delete_controller.attr) {
		if (!ctrl->ops->delete_ctrl)
			return 0;
	}

	CHECK_ATTR(ctrl, a, subsysnqn);
	CHECK_ATTR(ctrl, a, address);

	return a->mode;
}

1900
static struct attribute_group nvme_dev_attrs_group = {
M
Ming Lin 已提交
1901 1902
	.attrs		= nvme_dev_attrs,
	.is_visible	= nvme_dev_attrs_are_visible,
1903 1904 1905 1906 1907 1908 1909
};

static const struct attribute_group *nvme_dev_attr_groups[] = {
	&nvme_dev_attrs_group,
	NULL,
};

1910 1911 1912 1913 1914 1915 1916 1917
static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
	struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);

	return nsa->ns_id - nsb->ns_id;
}

1918
static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
1919
{
1920
	struct nvme_ns *ns, *ret = NULL;
1921

1922
	mutex_lock(&ctrl->namespaces_mutex);
1923
	list_for_each_entry(ns, &ctrl->namespaces, list) {
1924 1925 1926 1927 1928
		if (ns->ns_id == nsid) {
			kref_get(&ns->kref);
			ret = ns;
			break;
		}
1929 1930 1931
		if (ns->ns_id > nsid)
			break;
	}
1932 1933
	mutex_unlock(&ctrl->namespaces_mutex);
	return ret;
1934 1935 1936 1937 1938 1939
}

static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
{
	struct nvme_ns *ns;
	struct gendisk *disk;
1940 1941
	struct nvme_id_ns *id;
	char disk_name[DISK_NAME_LEN];
1942 1943 1944 1945 1946 1947
	int node = dev_to_node(ctrl->dev);

	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
	if (!ns)
		return;

1948 1949 1950 1951
	ns->instance = ida_simple_get(&ctrl->ns_ida, 1, 0, GFP_KERNEL);
	if (ns->instance < 0)
		goto out_free_ns;

1952 1953
	ns->queue = blk_mq_init_queue(ctrl->tagset);
	if (IS_ERR(ns->queue))
1954
		goto out_release_instance;
1955 1956 1957 1958 1959 1960 1961 1962 1963
	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, ns->queue);
	ns->queue->queuedata = ns;
	ns->ctrl = ctrl;

	kref_init(&ns->kref);
	ns->ns_id = nsid;
	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */

	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
1964
	nvme_set_queue_limits(ctrl, ns->queue);
1965

1966
	sprintf(disk_name, "nvme%dn%d", ctrl->instance, ns->instance);
1967

1968 1969 1970
	if (nvme_revalidate_ns(ns, &id))
		goto out_free_queue;

1971 1972 1973 1974 1975
	if (nvme_nvm_ns_supported(ns, id) &&
				nvme_nvm_register(ns, disk_name, node)) {
		dev_warn(ctrl->dev, "%s: LightNVM init failure\n", __func__);
		goto out_free_id;
	}
1976

1977 1978 1979
	disk = alloc_disk_node(0, node);
	if (!disk)
		goto out_free_id;
1980

1981 1982 1983 1984 1985 1986 1987 1988
	disk->fops = &nvme_fops;
	disk->private_data = ns;
	disk->queue = ns->queue;
	disk->flags = GENHD_FL_EXT_DEVT;
	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
	ns->disk = disk;

	__nvme_revalidate_disk(disk, id);
1989

1990 1991 1992 1993
	mutex_lock(&ctrl->namespaces_mutex);
	list_add_tail(&ns->list, &ctrl->namespaces);
	mutex_unlock(&ctrl->namespaces_mutex);

1994
	kref_get(&ctrl->kref);
1995 1996 1997

	kfree(id);

1998
	device_add_disk(ctrl->device, ns->disk);
1999 2000 2001 2002
	if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
					&nvme_ns_attr_group))
		pr_warn("%s: failed to create sysfs group for identification\n",
			ns->disk->disk_name);
2003 2004 2005
	if (ns->ndev && nvme_nvm_register_sysfs(ns))
		pr_warn("%s: failed to register lightnvm sysfs group for identification\n",
			ns->disk->disk_name);
2006
	return;
2007 2008
 out_free_id:
	kfree(id);
2009 2010
 out_free_queue:
	blk_cleanup_queue(ns->queue);
2011 2012
 out_release_instance:
	ida_simple_remove(&ctrl->ns_ida, ns->instance);
2013 2014 2015 2016 2017 2018
 out_free_ns:
	kfree(ns);
}

static void nvme_ns_remove(struct nvme_ns *ns)
{
2019 2020
	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
		return;
2021

2022
	if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
2023 2024
		if (blk_get_integrity(ns->disk))
			blk_integrity_unregister(ns->disk);
2025 2026
		sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
					&nvme_ns_attr_group);
2027 2028
		if (ns->ndev)
			nvme_nvm_unregister_sysfs(ns);
2029 2030 2031 2032
		del_gendisk(ns->disk);
		blk_mq_abort_requeue_list(ns->queue);
		blk_cleanup_queue(ns->queue);
	}
2033 2034

	mutex_lock(&ns->ctrl->namespaces_mutex);
2035
	list_del_init(&ns->list);
2036 2037
	mutex_unlock(&ns->ctrl->namespaces_mutex);

2038 2039 2040
	nvme_put_ns(ns);
}

2041 2042 2043 2044
static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
{
	struct nvme_ns *ns;

2045
	ns = nvme_find_get_ns(ctrl, nsid);
2046
	if (ns) {
2047
		if (ns->disk && revalidate_disk(ns->disk))
2048
			nvme_ns_remove(ns);
2049
		nvme_put_ns(ns);
2050 2051 2052 2053
	} else
		nvme_alloc_ns(ctrl, nsid);
}

2054 2055 2056 2057 2058 2059 2060 2061 2062 2063 2064
static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
					unsigned nsid)
{
	struct nvme_ns *ns, *next;

	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
		if (ns->ns_id > nsid)
			nvme_ns_remove(ns);
	}
}

2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078
static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
{
	struct nvme_ns *ns;
	__le32 *ns_list;
	unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024);
	int ret = 0;

	ns_list = kzalloc(0x1000, GFP_KERNEL);
	if (!ns_list)
		return -ENOMEM;

	for (i = 0; i < num_lists; i++) {
		ret = nvme_identify_ns_list(ctrl, prev, ns_list);
		if (ret)
2079
			goto free;
2080 2081 2082 2083 2084 2085 2086 2087 2088

		for (j = 0; j < min(nn, 1024U); j++) {
			nsid = le32_to_cpu(ns_list[j]);
			if (!nsid)
				goto out;

			nvme_validate_ns(ctrl, nsid);

			while (++prev < nsid) {
2089 2090
				ns = nvme_find_get_ns(ctrl, prev);
				if (ns) {
2091
					nvme_ns_remove(ns);
2092 2093
					nvme_put_ns(ns);
				}
2094 2095 2096 2097 2098
			}
		}
		nn -= j;
	}
 out:
2099 2100
	nvme_remove_invalid_namespaces(ctrl, prev);
 free:
2101 2102 2103 2104
	kfree(ns_list);
	return ret;
}

2105
static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn)
2106 2107 2108
{
	unsigned i;

2109 2110 2111
	for (i = 1; i <= nn; i++)
		nvme_validate_ns(ctrl, i);

2112
	nvme_remove_invalid_namespaces(ctrl, nn);
2113 2114
}

2115
static void nvme_scan_work(struct work_struct *work)
2116
{
2117 2118
	struct nvme_ctrl *ctrl =
		container_of(work, struct nvme_ctrl, scan_work);
2119
	struct nvme_id_ctrl *id;
2120
	unsigned nn;
2121

2122 2123 2124
	if (ctrl->state != NVME_CTRL_LIVE)
		return;

2125 2126
	if (nvme_identify_ctrl(ctrl, &id))
		return;
2127 2128

	nn = le32_to_cpu(id->nn);
2129
	if (ctrl->vs >= NVME_VS(1, 1, 0) &&
2130 2131 2132 2133
	    !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
		if (!nvme_scan_ns_list(ctrl, nn))
			goto done;
	}
2134
	nvme_scan_ns_sequential(ctrl, nn);
2135
 done:
2136
	mutex_lock(&ctrl->namespaces_mutex);
2137
	list_sort(NULL, &ctrl->namespaces, ns_cmp);
2138
	mutex_unlock(&ctrl->namespaces_mutex);
2139 2140
	kfree(id);
}
2141 2142 2143 2144 2145 2146 2147 2148 2149 2150 2151

void nvme_queue_scan(struct nvme_ctrl *ctrl)
{
	/*
	 * Do not queue new scan work when a controller is reset during
	 * removal.
	 */
	if (ctrl->state == NVME_CTRL_LIVE)
		schedule_work(&ctrl->scan_work);
}
EXPORT_SYMBOL_GPL(nvme_queue_scan);
2152

2153 2154 2155 2156 2157
/*
 * This function iterates the namespace list unlocked to allow recovery from
 * controller failure. It is up to the caller to ensure the namespace list is
 * not modified by scan work while this function is executing.
 */
2158 2159 2160 2161
void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns, *next;

2162 2163 2164 2165 2166 2167 2168 2169 2170
	/*
	 * The dead states indicates the controller was not gracefully
	 * disconnected. In that case, we won't be able to flush any data while
	 * removing the namespaces' disks; fail all the queues now to avoid
	 * potentially having to clean up the failed sync later.
	 */
	if (ctrl->state == NVME_CTRL_DEAD)
		nvme_kill_queues(ctrl);

2171 2172 2173
	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list)
		nvme_ns_remove(ns);
}
2174
EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
2175

2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191
static void nvme_async_event_work(struct work_struct *work)
{
	struct nvme_ctrl *ctrl =
		container_of(work, struct nvme_ctrl, async_event_work);

	spin_lock_irq(&ctrl->lock);
	while (ctrl->event_limit > 0) {
		int aer_idx = --ctrl->event_limit;

		spin_unlock_irq(&ctrl->lock);
		ctrl->ops->submit_async_event(ctrl, aer_idx);
		spin_lock_irq(&ctrl->lock);
	}
	spin_unlock_irq(&ctrl->lock);
}

2192 2193
void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
		union nvme_result *res)
2194
{
2195 2196
	u32 result = le32_to_cpu(res->u32);
	bool done = true;
2197

2198 2199 2200 2201 2202
	switch (le16_to_cpu(status) >> 1) {
	case NVME_SC_SUCCESS:
		done = false;
		/*FALLTHRU*/
	case NVME_SC_ABORT_REQ:
2203 2204
		++ctrl->event_limit;
		schedule_work(&ctrl->async_event_work);
2205 2206 2207
		break;
	default:
		break;
2208 2209
	}

2210
	if (done)
2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230
		return;

	switch (result & 0xff07) {
	case NVME_AER_NOTICE_NS_CHANGED:
		dev_info(ctrl->device, "rescanning\n");
		nvme_queue_scan(ctrl);
		break;
	default:
		dev_warn(ctrl->device, "async event result %08x\n", result);
	}
}
EXPORT_SYMBOL_GPL(nvme_complete_async_event);

void nvme_queue_async_events(struct nvme_ctrl *ctrl)
{
	ctrl->event_limit = NVME_NR_AERS;
	schedule_work(&ctrl->async_event_work);
}
EXPORT_SYMBOL_GPL(nvme_queue_async_events);

2231 2232 2233 2234 2235 2236 2237 2238 2239 2240 2241 2242 2243 2244 2245 2246 2247 2248 2249 2250 2251 2252 2253 2254 2255 2256 2257 2258 2259
static DEFINE_IDA(nvme_instance_ida);

static int nvme_set_instance(struct nvme_ctrl *ctrl)
{
	int instance, error;

	do {
		if (!ida_pre_get(&nvme_instance_ida, GFP_KERNEL))
			return -ENODEV;

		spin_lock(&dev_list_lock);
		error = ida_get_new(&nvme_instance_ida, &instance);
		spin_unlock(&dev_list_lock);
	} while (error == -EAGAIN);

	if (error)
		return -ENODEV;

	ctrl->instance = instance;
	return 0;
}

static void nvme_release_instance(struct nvme_ctrl *ctrl)
{
	spin_lock(&dev_list_lock);
	ida_remove(&nvme_instance_ida, ctrl->instance);
	spin_unlock(&dev_list_lock);
}

2260
void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
2261
{
2262
	flush_work(&ctrl->async_event_work);
2263 2264 2265
	flush_work(&ctrl->scan_work);
	nvme_remove_namespaces(ctrl);

2266
	device_destroy(nvme_class, MKDEV(nvme_char_major, ctrl->instance));
2267 2268 2269 2270

	spin_lock(&dev_list_lock);
	list_del(&ctrl->node);
	spin_unlock(&dev_list_lock);
2271
}
2272
EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
2273 2274 2275 2276

static void nvme_free_ctrl(struct kref *kref)
{
	struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref);
2277 2278 2279

	put_device(ctrl->device);
	nvme_release_instance(ctrl);
2280
	ida_destroy(&ctrl->ns_ida);
2281 2282 2283 2284 2285 2286 2287 2288

	ctrl->ops->free_ctrl(ctrl);
}

void nvme_put_ctrl(struct nvme_ctrl *ctrl)
{
	kref_put(&ctrl->kref, nvme_free_ctrl);
}
2289
EXPORT_SYMBOL_GPL(nvme_put_ctrl);
2290 2291 2292 2293 2294 2295 2296 2297 2298 2299 2300

/*
 * Initialize a NVMe controller structures.  This needs to be called during
 * earliest initialization so that we have the initialized structured around
 * during probing.
 */
int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
		const struct nvme_ctrl_ops *ops, unsigned long quirks)
{
	int ret;

2301 2302
	ctrl->state = NVME_CTRL_NEW;
	spin_lock_init(&ctrl->lock);
2303
	INIT_LIST_HEAD(&ctrl->namespaces);
2304
	mutex_init(&ctrl->namespaces_mutex);
2305 2306 2307 2308
	kref_init(&ctrl->kref);
	ctrl->dev = dev;
	ctrl->ops = ops;
	ctrl->quirks = quirks;
2309
	INIT_WORK(&ctrl->scan_work, nvme_scan_work);
2310
	INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
2311 2312 2313 2314 2315

	ret = nvme_set_instance(ctrl);
	if (ret)
		goto out;

2316
	ctrl->device = device_create_with_groups(nvme_class, ctrl->dev,
2317
				MKDEV(nvme_char_major, ctrl->instance),
2318
				ctrl, nvme_dev_attr_groups,
2319
				"nvme%d", ctrl->instance);
2320 2321 2322 2323 2324
	if (IS_ERR(ctrl->device)) {
		ret = PTR_ERR(ctrl->device);
		goto out_release_instance;
	}
	get_device(ctrl->device);
2325
	ida_init(&ctrl->ns_ida);
2326 2327 2328 2329 2330

	spin_lock(&dev_list_lock);
	list_add_tail(&ctrl->node, &nvme_ctrl_list);
	spin_unlock(&dev_list_lock);

2331 2332 2333 2334 2335 2336 2337 2338
	/*
	 * Initialize latency tolerance controls.  The sysfs files won't
	 * be visible to userspace unless the device actually supports APST.
	 */
	ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
		min(default_ps_max_latency_us, (unsigned long)S32_MAX));

2339 2340 2341 2342 2343 2344
	return 0;
out_release_instance:
	nvme_release_instance(ctrl);
out:
	return ret;
}
2345
EXPORT_SYMBOL_GPL(nvme_init_ctrl);
2346

2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357
/**
 * nvme_kill_queues(): Ends all namespace queues
 * @ctrl: the dead controller that needs to end
 *
 * Call this function when the driver determines it is unable to get the
 * controller in a state capable of servicing IO.
 */
void nvme_kill_queues(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns;

2358 2359
	mutex_lock(&ctrl->namespaces_mutex);
	list_for_each_entry(ns, &ctrl->namespaces, list) {
2360 2361 2362 2363
		/*
		 * Revalidating a dead namespace sets capacity to 0. This will
		 * end buffered writers dirtying pages that can't be synced.
		 */
2364 2365 2366
		if (!ns->disk || test_and_set_bit(NVME_NS_DEAD, &ns->flags))
			continue;
		revalidate_disk(ns->disk);
2367 2368 2369 2370
		blk_set_queue_dying(ns->queue);
		blk_mq_abort_requeue_list(ns->queue);
		blk_mq_start_stopped_hw_queues(ns->queue, true);
	}
2371
	mutex_unlock(&ctrl->namespaces_mutex);
2372
}
2373
EXPORT_SYMBOL_GPL(nvme_kill_queues);
2374

K
Keith Busch 已提交
2375 2376 2377 2378 2379 2380 2381 2382 2383 2384 2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397 2398 2399 2400 2401 2402 2403 2404 2405 2406 2407 2408 2409 2410 2411 2412 2413 2414 2415 2416
void nvme_unfreeze(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns;

	mutex_lock(&ctrl->namespaces_mutex);
	list_for_each_entry(ns, &ctrl->namespaces, list)
		blk_mq_unfreeze_queue(ns->queue);
	mutex_unlock(&ctrl->namespaces_mutex);
}
EXPORT_SYMBOL_GPL(nvme_unfreeze);

void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
{
	struct nvme_ns *ns;

	mutex_lock(&ctrl->namespaces_mutex);
	list_for_each_entry(ns, &ctrl->namespaces, list) {
		timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
		if (timeout <= 0)
			break;
	}
	mutex_unlock(&ctrl->namespaces_mutex);
}
EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);

void nvme_wait_freeze(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns;

	mutex_lock(&ctrl->namespaces_mutex);
	list_for_each_entry(ns, &ctrl->namespaces, list)
		blk_mq_freeze_queue_wait(ns->queue);
	mutex_unlock(&ctrl->namespaces_mutex);
}
EXPORT_SYMBOL_GPL(nvme_wait_freeze);

void nvme_start_freeze(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns;

	mutex_lock(&ctrl->namespaces_mutex);
	list_for_each_entry(ns, &ctrl->namespaces, list)
2417
		blk_freeze_queue_start(ns->queue);
K
Keith Busch 已提交
2418 2419 2420 2421
	mutex_unlock(&ctrl->namespaces_mutex);
}
EXPORT_SYMBOL_GPL(nvme_start_freeze);

2422
void nvme_stop_queues(struct nvme_ctrl *ctrl)
2423 2424 2425
{
	struct nvme_ns *ns;

2426
	mutex_lock(&ctrl->namespaces_mutex);
2427
	list_for_each_entry(ns, &ctrl->namespaces, list)
2428
		blk_mq_quiesce_queue(ns->queue);
2429
	mutex_unlock(&ctrl->namespaces_mutex);
2430
}
2431
EXPORT_SYMBOL_GPL(nvme_stop_queues);
2432

2433
void nvme_start_queues(struct nvme_ctrl *ctrl)
2434 2435 2436
{
	struct nvme_ns *ns;

2437 2438
	mutex_lock(&ctrl->namespaces_mutex);
	list_for_each_entry(ns, &ctrl->namespaces, list) {
2439 2440 2441
		blk_mq_start_stopped_hw_queues(ns->queue, true);
		blk_mq_kick_requeue_list(ns->queue);
	}
2442
	mutex_unlock(&ctrl->namespaces_mutex);
2443
}
2444
EXPORT_SYMBOL_GPL(nvme_start_queues);
2445

2446 2447 2448 2449
int __init nvme_core_init(void)
{
	int result;

2450 2451 2452
	result = __register_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme",
							&nvme_dev_fops);
	if (result < 0)
2453
		return result;
2454 2455 2456 2457 2458 2459 2460 2461 2462
	else if (result > 0)
		nvme_char_major = result;

	nvme_class = class_create(THIS_MODULE, "nvme");
	if (IS_ERR(nvme_class)) {
		result = PTR_ERR(nvme_class);
		goto unregister_chrdev;
	}

2463
	return 0;
2464 2465 2466 2467

 unregister_chrdev:
	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
	return result;
2468 2469 2470 2471
}

void nvme_core_exit(void)
{
2472 2473
	class_destroy(nvme_class);
	__unregister_chrdev(nvme_char_major, 0, NVME_MINORS, "nvme");
2474
}
2475 2476 2477 2478 2479

MODULE_LICENSE("GPL");
MODULE_VERSION("1.0");
module_init(nvme_core_init);
module_exit(nvme_core_exit);