core.c 90.6 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16
/*
 * NVM Express device driver
 * Copyright (c) 2011-2014, Intel Corporation.
 *
 * This program is free software; you can redistribute it and/or modify it
 * under the terms and conditions of the GNU General Public License,
 * version 2, as published by the Free Software Foundation.
 *
 * This program is distributed in the hope it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
 * more details.
 */

#include <linux/blkdev.h>
#include <linux/blk-mq.h>
17
#include <linux/delay.h>
18
#include <linux/errno.h>
19
#include <linux/hdreg.h>
20
#include <linux/kernel.h>
21 22
#include <linux/module.h>
#include <linux/list_sort.h>
23 24
#include <linux/slab.h>
#include <linux/types.h>
25 26 27 28
#include <linux/pr.h>
#include <linux/ptrace.h>
#include <linux/nvme_ioctl.h>
#include <linux/t10-pi.h>
29
#include <linux/pm_qos.h>
30
#include <asm/unaligned.h>
31

32 33 34
#define CREATE_TRACE_POINTS
#include "trace.h"

35
#include "nvme.h"
S
Sagi Grimberg 已提交
36
#include "fabrics.h"
37

38 39
#define NVME_MINORS		(1U << MINORBITS)

40 41
unsigned int admin_timeout = 60;
module_param(admin_timeout, uint, 0644);
42
MODULE_PARM_DESC(admin_timeout, "timeout in seconds for admin commands");
43
EXPORT_SYMBOL_GPL(admin_timeout);
44

45 46
unsigned int nvme_io_timeout = 30;
module_param_named(io_timeout, nvme_io_timeout, uint, 0644);
47
MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O");
48
EXPORT_SYMBOL_GPL(nvme_io_timeout);
49

50
static unsigned char shutdown_timeout = 5;
51 52 53
module_param(shutdown_timeout, byte, 0644);
MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown");

54 55
static u8 nvme_max_retries = 5;
module_param_named(max_retries, nvme_max_retries, byte, 0644);
K
Keith Busch 已提交
56
MODULE_PARM_DESC(max_retries, "max number of retries a command may have");
57

58
static unsigned long default_ps_max_latency_us = 100000;
59 60 61 62
module_param(default_ps_max_latency_us, ulong, 0644);
MODULE_PARM_DESC(default_ps_max_latency_us,
		 "max power saving latency for new devices; use PM QOS to change per device");

63 64 65 66
static bool force_apst;
module_param(force_apst, bool, 0644);
MODULE_PARM_DESC(force_apst, "allow APST for newly enumerated devices even if quirked off");

67 68 69 70
static bool streams;
module_param(streams, bool, 0644);
MODULE_PARM_DESC(streams, "turn on support for Streams write directives");

71 72 73 74 75 76 77 78 79 80 81
/*
 * nvme_wq - hosts nvme related works that are not reset or delete
 * nvme_reset_wq - hosts nvme reset works
 * nvme_delete_wq - hosts nvme delete works
 *
 * nvme_wq will host works such are scan, aen handling, fw activation,
 * keep-alive error recovery, periodic reconnects etc. nvme_reset_wq
 * runs reset works which also flush works hosted on nvme_wq for
 * serialization purposes. nvme_delete_wq host controller deletion
 * works which flush reset works for serialization.
 */
82 83 84
struct workqueue_struct *nvme_wq;
EXPORT_SYMBOL_GPL(nvme_wq);

85 86 87 88 89 90
struct workqueue_struct *nvme_reset_wq;
EXPORT_SYMBOL_GPL(nvme_reset_wq);

struct workqueue_struct *nvme_delete_wq;
EXPORT_SYMBOL_GPL(nvme_delete_wq);

C
Christoph Hellwig 已提交
91 92 93
static DEFINE_IDA(nvme_subsystems_ida);
static LIST_HEAD(nvme_subsystems);
static DEFINE_MUTEX(nvme_subsystems_lock);
94

95
static DEFINE_IDA(nvme_instance_ida);
96
static dev_t nvme_chr_devt;
97
static struct class *nvme_class;
C
Christoph Hellwig 已提交
98
static struct class *nvme_subsys_class;
99

100 101
static void nvme_ns_remove(struct nvme_ns *ns);
static int nvme_revalidate_disk(struct gendisk *disk);
102

103 104 105 106
int nvme_reset_ctrl(struct nvme_ctrl *ctrl)
{
	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
		return -EBUSY;
107
	if (!queue_work(nvme_reset_wq, &ctrl->reset_work))
108 109 110 111 112
		return -EBUSY;
	return 0;
}
EXPORT_SYMBOL_GPL(nvme_reset_ctrl);

S
Sagi Grimberg 已提交
113
int nvme_reset_ctrl_sync(struct nvme_ctrl *ctrl)
114 115 116 117
{
	int ret;

	ret = nvme_reset_ctrl(ctrl);
118
	if (!ret) {
119
		flush_work(&ctrl->reset_work);
120 121 122 123
		if (ctrl->state != NVME_CTRL_LIVE)
			ret = -ENETRESET;
	}

124 125
	return ret;
}
S
Sagi Grimberg 已提交
126
EXPORT_SYMBOL_GPL(nvme_reset_ctrl_sync);
127

128 129 130 131 132
static void nvme_delete_ctrl_work(struct work_struct *work)
{
	struct nvme_ctrl *ctrl =
		container_of(work, struct nvme_ctrl, delete_work);

133 134 135
	dev_info(ctrl->device,
		 "Removing ctrl: NQN \"%s\"\n", ctrl->opts->subsysnqn);

136
	flush_work(&ctrl->reset_work);
137 138
	nvme_stop_ctrl(ctrl);
	nvme_remove_namespaces(ctrl);
139
	ctrl->ops->delete_ctrl(ctrl);
140 141
	nvme_uninit_ctrl(ctrl);
	nvme_put_ctrl(ctrl);
142 143 144 145 146 147
}

int nvme_delete_ctrl(struct nvme_ctrl *ctrl)
{
	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_DELETING))
		return -EBUSY;
148
	if (!queue_work(nvme_delete_wq, &ctrl->delete_work))
149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170
		return -EBUSY;
	return 0;
}
EXPORT_SYMBOL_GPL(nvme_delete_ctrl);

int nvme_delete_ctrl_sync(struct nvme_ctrl *ctrl)
{
	int ret = 0;

	/*
	 * Keep a reference until the work is flushed since ->delete_ctrl
	 * can free the controller.
	 */
	nvme_get_ctrl(ctrl);
	ret = nvme_delete_ctrl(ctrl);
	if (!ret)
		flush_work(&ctrl->delete_work);
	nvme_put_ctrl(ctrl);
	return ret;
}
EXPORT_SYMBOL_GPL(nvme_delete_ctrl_sync);

171 172 173 174 175
static inline bool nvme_ns_has_pi(struct nvme_ns *ns)
{
	return ns->pi_type && ns->ms == sizeof(struct t10_pi_tuple);
}

176
static blk_status_t nvme_error_status(struct request *req)
177 178 179
{
	switch (nvme_req(req)->status & 0x7ff) {
	case NVME_SC_SUCCESS:
180
		return BLK_STS_OK;
181
	case NVME_SC_CAP_EXCEEDED:
182
		return BLK_STS_NOSPC;
183 184 185
	case NVME_SC_LBA_RANGE:
		return BLK_STS_TARGET;
	case NVME_SC_BAD_ATTRIBUTES:
186
	case NVME_SC_ONCS_NOT_SUPPORTED:
187 188 189
	case NVME_SC_INVALID_OPCODE:
	case NVME_SC_INVALID_FIELD:
	case NVME_SC_INVALID_NS:
190
		return BLK_STS_NOTSUPP;
191 192 193
	case NVME_SC_WRITE_FAULT:
	case NVME_SC_READ_ERROR:
	case NVME_SC_UNWRITTEN_BLOCK:
194 195
	case NVME_SC_ACCESS_DENIED:
	case NVME_SC_READ_ONLY:
196
	case NVME_SC_COMPARE_FAILED:
197
		return BLK_STS_MEDIUM;
198 199 200 201 202 203 204
	case NVME_SC_GUARD_CHECK:
	case NVME_SC_APPTAG_CHECK:
	case NVME_SC_REFTAG_CHECK:
	case NVME_SC_INVALID_PI:
		return BLK_STS_PROTECTION;
	case NVME_SC_RESERVATION_CONFLICT:
		return BLK_STS_NEXUS;
205 206
	default:
		return BLK_STS_IOERR;
207 208 209
	}
}

210
static inline bool nvme_req_needs_retry(struct request *req)
211
{
212 213
	if (blk_noretry_request(req))
		return false;
214
	if (nvme_req(req)->status & NVME_SC_DNR)
215
		return false;
216
	if (nvme_req(req)->retries >= nvme_max_retries)
217 218
		return false;
	return true;
219 220 221 222
}

void nvme_complete_rq(struct request *req)
{
223 224
	blk_status_t status = nvme_error_status(req);

225 226
	trace_nvme_complete_rq(req);

227 228
	if (unlikely(status != BLK_STS_OK && nvme_req_needs_retry(req))) {
		if (nvme_req_needs_failover(req, status)) {
229 230 231 232 233 234 235 236 237
			nvme_failover_req(req);
			return;
		}

		if (!blk_queue_dying(req->q)) {
			nvme_req(req)->retries++;
			blk_mq_requeue_request(req, true);
			return;
		}
238
	}
239
	blk_mq_end_request(req, status);
240 241 242
}
EXPORT_SYMBOL_GPL(nvme_complete_rq);

243 244 245 246 247 248 249 250
void nvme_cancel_request(struct request *req, void *data, bool reserved)
{
	if (!blk_mq_request_started(req))
		return;

	dev_dbg_ratelimited(((struct nvme_ctrl *) data)->device,
				"Cancelling I/O %d", req->tag);

251
	nvme_req(req)->status = NVME_SC_ABORT_REQ;
252
	blk_mq_complete_request(req);
253

254 255 256
}
EXPORT_SYMBOL_GPL(nvme_cancel_request);

257 258 259
bool nvme_change_ctrl_state(struct nvme_ctrl *ctrl,
		enum nvme_ctrl_state new_state)
{
260
	enum nvme_ctrl_state old_state;
261
	unsigned long flags;
262 263
	bool changed = false;

264
	spin_lock_irqsave(&ctrl->lock, flags);
265 266

	old_state = ctrl->state;
267
	switch (new_state) {
268 269
	case NVME_CTRL_ADMIN_ONLY:
		switch (old_state) {
270
		case NVME_CTRL_CONNECTING:
271 272 273 274 275 276
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
277 278
	case NVME_CTRL_LIVE:
		switch (old_state) {
279
		case NVME_CTRL_NEW:
280
		case NVME_CTRL_RESETTING:
281
		case NVME_CTRL_CONNECTING:
282 283 284 285 286 287 288 289 290
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
	case NVME_CTRL_RESETTING:
		switch (old_state) {
		case NVME_CTRL_NEW:
291
		case NVME_CTRL_LIVE:
292
		case NVME_CTRL_ADMIN_ONLY:
293 294 295 296 297 298
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
299
	case NVME_CTRL_CONNECTING:
300
		switch (old_state) {
301
		case NVME_CTRL_NEW:
302
		case NVME_CTRL_RESETTING:
303 304 305 306 307 308 309 310 311
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
	case NVME_CTRL_DELETING:
		switch (old_state) {
		case NVME_CTRL_LIVE:
312
		case NVME_CTRL_ADMIN_ONLY:
313
		case NVME_CTRL_RESETTING:
314
		case NVME_CTRL_CONNECTING:
315 316 317 318 319 320
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
321 322 323 324 325 326 327 328 329
	case NVME_CTRL_DEAD:
		switch (old_state) {
		case NVME_CTRL_DELETING:
			changed = true;
			/* FALLTHRU */
		default:
			break;
		}
		break;
330 331 332 333 334 335 336
	default:
		break;
	}

	if (changed)
		ctrl->state = new_state;

337
	spin_unlock_irqrestore(&ctrl->lock, flags);
338 339
	if (changed && ctrl->state == NVME_CTRL_LIVE)
		nvme_kick_requeue_lists(ctrl);
340 341 342 343
	return changed;
}
EXPORT_SYMBOL_GPL(nvme_change_ctrl_state);

C
Christoph Hellwig 已提交
344 345 346 347 348
static void nvme_free_ns_head(struct kref *ref)
{
	struct nvme_ns_head *head =
		container_of(ref, struct nvme_ns_head, ref);

349
	nvme_mpath_remove_disk(head);
C
Christoph Hellwig 已提交
350 351 352 353 354 355 356 357 358 359 360
	ida_simple_remove(&head->subsys->ns_ida, head->instance);
	list_del_init(&head->entry);
	cleanup_srcu_struct(&head->srcu);
	kfree(head);
}

static void nvme_put_ns_head(struct nvme_ns_head *head)
{
	kref_put(&head->ref, nvme_free_ns_head);
}

361 362 363 364
static void nvme_free_ns(struct kref *kref)
{
	struct nvme_ns *ns = container_of(kref, struct nvme_ns, kref);

365 366
	if (ns->ndev)
		nvme_nvm_unregister(ns);
367 368

	put_disk(ns->disk);
C
Christoph Hellwig 已提交
369
	nvme_put_ns_head(ns->head);
370
	nvme_put_ctrl(ns->ctrl);
371 372 373
	kfree(ns);
}

374
static void nvme_put_ns(struct nvme_ns *ns)
375 376 377 378
{
	kref_put(&ns->kref, nvme_free_ns);
}

379 380 381 382 383 384 385 386 387
static inline void nvme_clear_nvme_request(struct request *req)
{
	if (!(req->rq_flags & RQF_DONTPREP)) {
		nvme_req(req)->retries = 0;
		nvme_req(req)->flags = 0;
		req->rq_flags |= RQF_DONTPREP;
	}
}

388
struct request *nvme_alloc_request(struct request_queue *q,
389
		struct nvme_command *cmd, blk_mq_req_flags_t flags, int qid)
390
{
391
	unsigned op = nvme_is_write(cmd) ? REQ_OP_DRV_OUT : REQ_OP_DRV_IN;
392 393
	struct request *req;

394
	if (qid == NVME_QID_ANY) {
395
		req = blk_mq_alloc_request(q, op, flags);
396
	} else {
397
		req = blk_mq_alloc_request_hctx(q, op, flags,
398 399
				qid ? qid - 1 : 0);
	}
400
	if (IS_ERR(req))
401
		return req;
402 403

	req->cmd_flags |= REQ_FAILFAST_DRIVER;
404
	nvme_clear_nvme_request(req);
405
	nvme_req(req)->cmd = cmd;
406

407 408
	return req;
}
409
EXPORT_SYMBOL_GPL(nvme_alloc_request);
410

411 412 413 414 415 416 417
static int nvme_toggle_streams(struct nvme_ctrl *ctrl, bool enable)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));

	c.directive.opcode = nvme_admin_directive_send;
A
Arnav Dawn 已提交
418
	c.directive.nsid = cpu_to_le32(NVME_NSID_ALL);
419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446
	c.directive.doper = NVME_DIR_SND_ID_OP_ENABLE;
	c.directive.dtype = NVME_DIR_IDENTIFY;
	c.directive.tdtype = NVME_DIR_STREAMS;
	c.directive.endir = enable ? NVME_DIR_ENDIR : 0;

	return nvme_submit_sync_cmd(ctrl->admin_q, &c, NULL, 0);
}

static int nvme_disable_streams(struct nvme_ctrl *ctrl)
{
	return nvme_toggle_streams(ctrl, false);
}

static int nvme_enable_streams(struct nvme_ctrl *ctrl)
{
	return nvme_toggle_streams(ctrl, true);
}

static int nvme_get_stream_params(struct nvme_ctrl *ctrl,
				  struct streams_directive_params *s, u32 nsid)
{
	struct nvme_command c;

	memset(&c, 0, sizeof(c));
	memset(s, 0, sizeof(*s));

	c.directive.opcode = nvme_admin_directive_recv;
	c.directive.nsid = cpu_to_le32(nsid);
447
	c.directive.numd = cpu_to_le32((sizeof(*s) >> 2) - 1);
448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467
	c.directive.doper = NVME_DIR_RCV_ST_OP_PARAM;
	c.directive.dtype = NVME_DIR_STREAMS;

	return nvme_submit_sync_cmd(ctrl->admin_q, &c, s, sizeof(*s));
}

static int nvme_configure_directives(struct nvme_ctrl *ctrl)
{
	struct streams_directive_params s;
	int ret;

	if (!(ctrl->oacs & NVME_CTRL_OACS_DIRECTIVES))
		return 0;
	if (!streams)
		return 0;

	ret = nvme_enable_streams(ctrl);
	if (ret)
		return ret;

A
Arnav Dawn 已提交
468
	ret = nvme_get_stream_params(ctrl, &s, NVME_NSID_ALL);
469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509
	if (ret)
		return ret;

	ctrl->nssa = le16_to_cpu(s.nssa);
	if (ctrl->nssa < BLK_MAX_WRITE_HINTS - 1) {
		dev_info(ctrl->device, "too few streams (%u) available\n",
					ctrl->nssa);
		nvme_disable_streams(ctrl);
		return 0;
	}

	ctrl->nr_streams = min_t(unsigned, ctrl->nssa, BLK_MAX_WRITE_HINTS - 1);
	dev_info(ctrl->device, "Using %u streams\n", ctrl->nr_streams);
	return 0;
}

/*
 * Check if 'req' has a write hint associated with it. If it does, assign
 * a valid namespace stream to the write.
 */
static void nvme_assign_write_stream(struct nvme_ctrl *ctrl,
				     struct request *req, u16 *control,
				     u32 *dsmgmt)
{
	enum rw_hint streamid = req->write_hint;

	if (streamid == WRITE_LIFE_NOT_SET || streamid == WRITE_LIFE_NONE)
		streamid = 0;
	else {
		streamid--;
		if (WARN_ON_ONCE(streamid > ctrl->nr_streams))
			return;

		*control |= NVME_RW_DTYPE_STREAMS;
		*dsmgmt |= streamid << 16;
	}

	if (streamid < ARRAY_SIZE(req->q->write_hints))
		req->q->write_hints[streamid] += blk_rq_bytes(req) >> 9;
}

M
Ming Lin 已提交
510 511 512 513 514
static inline void nvme_setup_flush(struct nvme_ns *ns,
		struct nvme_command *cmnd)
{
	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->common.opcode = nvme_cmd_flush;
C
Christoph Hellwig 已提交
515
	cmnd->common.nsid = cpu_to_le32(ns->head->ns_id);
M
Ming Lin 已提交
516 517
}

518
static blk_status_t nvme_setup_discard(struct nvme_ns *ns, struct request *req,
M
Ming Lin 已提交
519 520
		struct nvme_command *cmnd)
{
521
	unsigned short segments = blk_rq_nr_discard_segments(req), n = 0;
M
Ming Lin 已提交
522
	struct nvme_dsm_range *range;
523
	struct bio *bio;
M
Ming Lin 已提交
524

525
	range = kmalloc_array(segments, sizeof(*range), GFP_ATOMIC);
M
Ming Lin 已提交
526
	if (!range)
527
		return BLK_STS_RESOURCE;
M
Ming Lin 已提交
528

529 530 531 532
	__rq_for_each_bio(bio, req) {
		u64 slba = nvme_block_nr(ns, bio->bi_iter.bi_sector);
		u32 nlb = bio->bi_iter.bi_size >> ns->lba_shift;

K
Keith Busch 已提交
533 534 535 536 537
		if (n < segments) {
			range[n].cattr = cpu_to_le32(0);
			range[n].nlb = cpu_to_le32(nlb);
			range[n].slba = cpu_to_le64(slba);
		}
538 539 540 541 542
		n++;
	}

	if (WARN_ON_ONCE(n != segments)) {
		kfree(range);
543
		return BLK_STS_IOERR;
544
	}
M
Ming Lin 已提交
545 546 547

	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->dsm.opcode = nvme_cmd_dsm;
C
Christoph Hellwig 已提交
548
	cmnd->dsm.nsid = cpu_to_le32(ns->head->ns_id);
549
	cmnd->dsm.nr = cpu_to_le32(segments - 1);
M
Ming Lin 已提交
550 551
	cmnd->dsm.attributes = cpu_to_le32(NVME_DSMGMT_AD);

552 553
	req->special_vec.bv_page = virt_to_page(range);
	req->special_vec.bv_offset = offset_in_page(range);
554
	req->special_vec.bv_len = sizeof(*range) * segments;
555
	req->rq_flags |= RQF_SPECIAL_PAYLOAD;
M
Ming Lin 已提交
556

557
	return BLK_STS_OK;
M
Ming Lin 已提交
558 559
}

560 561
static inline blk_status_t nvme_setup_rw(struct nvme_ns *ns,
		struct request *req, struct nvme_command *cmnd)
M
Ming Lin 已提交
562
{
563
	struct nvme_ctrl *ctrl = ns->ctrl;
M
Ming Lin 已提交
564 565 566 567 568 569 570 571 572 573 574 575 576
	u16 control = 0;
	u32 dsmgmt = 0;

	if (req->cmd_flags & REQ_FUA)
		control |= NVME_RW_FUA;
	if (req->cmd_flags & (REQ_FAILFAST_DEV | REQ_RAHEAD))
		control |= NVME_RW_LR;

	if (req->cmd_flags & REQ_RAHEAD)
		dsmgmt |= NVME_RW_DSM_FREQ_PREFETCH;

	memset(cmnd, 0, sizeof(*cmnd));
	cmnd->rw.opcode = (rq_data_dir(req) ? nvme_cmd_write : nvme_cmd_read);
C
Christoph Hellwig 已提交
577
	cmnd->rw.nsid = cpu_to_le32(ns->head->ns_id);
M
Ming Lin 已提交
578 579 580
	cmnd->rw.slba = cpu_to_le64(nvme_block_nr(ns, blk_rq_pos(req)));
	cmnd->rw.length = cpu_to_le16((blk_rq_bytes(req) >> ns->lba_shift) - 1);

581 582 583
	if (req_op(req) == REQ_OP_WRITE && ctrl->nr_streams)
		nvme_assign_write_stream(ctrl, req, &control, &dsmgmt);

M
Ming Lin 已提交
584
	if (ns->ms) {
585 586 587 588 589 590 591 592 593 594 595 596
		/*
		 * If formated with metadata, the block layer always provides a
		 * metadata buffer if CONFIG_BLK_DEV_INTEGRITY is enabled.  Else
		 * we enable the PRACT bit for protection information or set the
		 * namespace capacity to zero to prevent any I/O.
		 */
		if (!blk_integrity_rq(req)) {
			if (WARN_ON_ONCE(!nvme_ns_has_pi(ns)))
				return BLK_STS_NOTSUPP;
			control |= NVME_RW_PRINFO_PRACT;
		}

M
Ming Lin 已提交
597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612
		switch (ns->pi_type) {
		case NVME_NS_DPS_PI_TYPE3:
			control |= NVME_RW_PRINFO_PRCHK_GUARD;
			break;
		case NVME_NS_DPS_PI_TYPE1:
		case NVME_NS_DPS_PI_TYPE2:
			control |= NVME_RW_PRINFO_PRCHK_GUARD |
					NVME_RW_PRINFO_PRCHK_REF;
			cmnd->rw.reftag = cpu_to_le32(
					nvme_block_nr(ns, blk_rq_pos(req)));
			break;
		}
	}

	cmnd->rw.control = cpu_to_le16(control);
	cmnd->rw.dsmgmt = cpu_to_le32(dsmgmt);
613
	return 0;
M
Ming Lin 已提交
614 615
}

616
blk_status_t nvme_setup_cmd(struct nvme_ns *ns, struct request *req,
M
Ming Lin 已提交
617 618
		struct nvme_command *cmd)
{
619
	blk_status_t ret = BLK_STS_OK;
M
Ming Lin 已提交
620

621
	nvme_clear_nvme_request(req);
622

623 624 625
	switch (req_op(req)) {
	case REQ_OP_DRV_IN:
	case REQ_OP_DRV_OUT:
626
		memcpy(cmd, nvme_req(req)->cmd, sizeof(*cmd));
627 628
		break;
	case REQ_OP_FLUSH:
M
Ming Lin 已提交
629
		nvme_setup_flush(ns, cmd);
630
		break;
631 632
	case REQ_OP_WRITE_ZEROES:
		/* currently only aliased to deallocate for a few ctrls: */
633
	case REQ_OP_DISCARD:
M
Ming Lin 已提交
634
		ret = nvme_setup_discard(ns, req, cmd);
635 636 637
		break;
	case REQ_OP_READ:
	case REQ_OP_WRITE:
638
		ret = nvme_setup_rw(ns, req, cmd);
639 640 641
		break;
	default:
		WARN_ON_ONCE(1);
642
		return BLK_STS_IOERR;
643
	}
M
Ming Lin 已提交
644

645
	cmd->common.command_id = req->tag;
646 647 648 649
	if (ns)
		trace_nvme_setup_nvm_cmd(req->q->id, cmd);
	else
		trace_nvme_setup_admin_cmd(cmd);
M
Ming Lin 已提交
650 651 652 653
	return ret;
}
EXPORT_SYMBOL_GPL(nvme_setup_cmd);

654 655 656 657 658
/*
 * Returns 0 on success.  If the result is negative, it's a Linux error code;
 * if the result is positive, it's an NVM Express status code
 */
int __nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
659
		union nvme_result *result, void *buffer, unsigned bufflen,
660 661
		unsigned timeout, int qid, int at_head,
		blk_mq_req_flags_t flags)
662 663 664 665
{
	struct request *req;
	int ret;

666
	req = nvme_alloc_request(q, cmd, flags, qid);
667 668 669 670 671
	if (IS_ERR(req))
		return PTR_ERR(req);

	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;

672 673 674 675
	if (buffer && bufflen) {
		ret = blk_rq_map_kern(q, req, buffer, bufflen, GFP_KERNEL);
		if (ret)
			goto out;
676 677
	}

678
	blk_execute_rq(req->q, NULL, req, at_head);
679 680
	if (result)
		*result = nvme_req(req)->result;
681 682 683 684
	if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
		ret = -EINTR;
	else
		ret = nvme_req(req)->status;
685 686 687 688
 out:
	blk_mq_free_request(req);
	return ret;
}
689
EXPORT_SYMBOL_GPL(__nvme_submit_sync_cmd);
690 691 692 693

int nvme_submit_sync_cmd(struct request_queue *q, struct nvme_command *cmd,
		void *buffer, unsigned bufflen)
{
694 695
	return __nvme_submit_sync_cmd(q, cmd, NULL, buffer, bufflen, 0,
			NVME_QID_ANY, 0, 0);
696
}
697
EXPORT_SYMBOL_GPL(nvme_submit_sync_cmd);
698

699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732
static void *nvme_add_user_metadata(struct bio *bio, void __user *ubuf,
		unsigned len, u32 seed, bool write)
{
	struct bio_integrity_payload *bip;
	int ret = -ENOMEM;
	void *buf;

	buf = kmalloc(len, GFP_KERNEL);
	if (!buf)
		goto out;

	ret = -EFAULT;
	if (write && copy_from_user(buf, ubuf, len))
		goto out_free_meta;

	bip = bio_integrity_alloc(bio, GFP_KERNEL, 1);
	if (IS_ERR(bip)) {
		ret = PTR_ERR(bip);
		goto out_free_meta;
	}

	bip->bip_iter.bi_size = len;
	bip->bip_iter.bi_sector = seed;
	ret = bio_integrity_add_page(bio, virt_to_page(buf), len,
			offset_in_page(buf));
	if (ret == len)
		return buf;
	ret = -ENOMEM;
out_free_meta:
	kfree(buf);
out:
	return ERR_PTR(ret);
}

733
static int nvme_submit_user_cmd(struct request_queue *q,
734 735 736
		struct nvme_command *cmd, void __user *ubuffer,
		unsigned bufflen, void __user *meta_buffer, unsigned meta_len,
		u32 meta_seed, u32 *result, unsigned timeout)
737
{
738
	bool write = nvme_is_write(cmd);
739 740
	struct nvme_ns *ns = q->queuedata;
	struct gendisk *disk = ns ? ns->disk : NULL;
741
	struct request *req;
742 743
	struct bio *bio = NULL;
	void *meta = NULL;
744 745
	int ret;

746
	req = nvme_alloc_request(q, cmd, 0, NVME_QID_ANY);
747 748 749 750
	if (IS_ERR(req))
		return PTR_ERR(req);

	req->timeout = timeout ? timeout : ADMIN_TIMEOUT;
751
	nvme_req(req)->flags |= NVME_REQ_USERCMD;
752 753

	if (ubuffer && bufflen) {
754 755 756 757 758
		ret = blk_rq_map_user(q, req, NULL, ubuffer, bufflen,
				GFP_KERNEL);
		if (ret)
			goto out;
		bio = req->bio;
759
		bio->bi_disk = disk;
760 761 762 763 764
		if (disk && meta_buffer && meta_len) {
			meta = nvme_add_user_metadata(bio, meta_buffer, meta_len,
					meta_seed, write);
			if (IS_ERR(meta)) {
				ret = PTR_ERR(meta);
765 766
				goto out_unmap;
			}
767
			req->cmd_flags |= REQ_INTEGRITY;
768 769
		}
	}
770

771
	blk_execute_rq(req->q, disk, req, 0);
772 773 774 775
	if (nvme_req(req)->flags & NVME_REQ_CANCELLED)
		ret = -EINTR;
	else
		ret = nvme_req(req)->status;
776
	if (result)
777
		*result = le32_to_cpu(nvme_req(req)->result.u32);
778 779 780 781 782 783
	if (meta && !ret && !write) {
		if (copy_to_user(meta_buffer, meta, meta_len))
			ret = -EFAULT;
	}
	kfree(meta);
 out_unmap:
784
	if (bio)
785
		blk_rq_unmap_user(bio);
786 787 788 789 790
 out:
	blk_mq_free_request(req);
	return ret;
}

791
static void nvme_keep_alive_end_io(struct request *rq, blk_status_t status)
S
Sagi Grimberg 已提交
792 793 794 795 796
{
	struct nvme_ctrl *ctrl = rq->end_io_data;

	blk_mq_free_request(rq);

797
	if (status) {
S
Sagi Grimberg 已提交
798
		dev_err(ctrl->device,
799 800
			"failed nvme_keep_alive_end_io error=%d\n",
				status);
S
Sagi Grimberg 已提交
801 802 803 804 805 806 807 808 809 810
		return;
	}

	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
}

static int nvme_keep_alive(struct nvme_ctrl *ctrl)
{
	struct request *rq;

811
	rq = nvme_alloc_request(ctrl->admin_q, &ctrl->ka_cmd, BLK_MQ_REQ_RESERVED,
S
Sagi Grimberg 已提交
812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831
			NVME_QID_ANY);
	if (IS_ERR(rq))
		return PTR_ERR(rq);

	rq->timeout = ctrl->kato * HZ;
	rq->end_io_data = ctrl;

	blk_execute_rq_nowait(rq->q, NULL, rq, 0, nvme_keep_alive_end_io);

	return 0;
}

static void nvme_keep_alive_work(struct work_struct *work)
{
	struct nvme_ctrl *ctrl = container_of(to_delayed_work(work),
			struct nvme_ctrl, ka_work);

	if (nvme_keep_alive(ctrl)) {
		/* allocation failure, reset the controller */
		dev_err(ctrl->device, "keep-alive failed\n");
832
		nvme_reset_ctrl(ctrl);
S
Sagi Grimberg 已提交
833 834 835 836
		return;
	}
}

837
static void nvme_start_keep_alive(struct nvme_ctrl *ctrl)
S
Sagi Grimberg 已提交
838 839 840 841 842
{
	if (unlikely(ctrl->kato == 0))
		return;

	INIT_DELAYED_WORK(&ctrl->ka_work, nvme_keep_alive_work);
843 844
	memset(&ctrl->ka_cmd, 0, sizeof(ctrl->ka_cmd));
	ctrl->ka_cmd.common.opcode = nvme_admin_keep_alive;
S
Sagi Grimberg 已提交
845 846 847 848 849 850 851 852 853 854 855 856
	schedule_delayed_work(&ctrl->ka_work, ctrl->kato * HZ);
}

void nvme_stop_keep_alive(struct nvme_ctrl *ctrl)
{
	if (unlikely(ctrl->kato == 0))
		return;

	cancel_delayed_work_sync(&ctrl->ka_work);
}
EXPORT_SYMBOL_GPL(nvme_stop_keep_alive);

K
Keith Busch 已提交
857
static int nvme_identify_ctrl(struct nvme_ctrl *dev, struct nvme_id_ctrl **id)
858 859 860 861 862 863
{
	struct nvme_command c = { };
	int error;

	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
	c.identify.opcode = nvme_admin_identify;
864
	c.identify.cns = NVME_ID_CNS_CTRL;
865 866 867 868 869 870 871 872 873 874 875 876

	*id = kmalloc(sizeof(struct nvme_id_ctrl), GFP_KERNEL);
	if (!*id)
		return -ENOMEM;

	error = nvme_submit_sync_cmd(dev->admin_q, &c, *id,
			sizeof(struct nvme_id_ctrl));
	if (error)
		kfree(*id);
	return error;
}

877
static int nvme_identify_ns_descs(struct nvme_ctrl *ctrl, unsigned nsid,
878
		struct nvme_ns_ids *ids)
879 880 881 882 883 884 885 886 887 888 889 890 891 892 893
{
	struct nvme_command c = { };
	int status;
	void *data;
	int pos;
	int len;

	c.identify.opcode = nvme_admin_identify;
	c.identify.nsid = cpu_to_le32(nsid);
	c.identify.cns = NVME_ID_CNS_NS_DESC_LIST;

	data = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
	if (!data)
		return -ENOMEM;

894
	status = nvme_submit_sync_cmd(ctrl->admin_q, &c, data,
895 896 897 898 899 900 901 902 903 904 905 906 907
				      NVME_IDENTIFY_DATA_SIZE);
	if (status)
		goto free_data;

	for (pos = 0; pos < NVME_IDENTIFY_DATA_SIZE; pos += len) {
		struct nvme_ns_id_desc *cur = data + pos;

		if (cur->nidl == 0)
			break;

		switch (cur->nidt) {
		case NVME_NIDT_EUI64:
			if (cur->nidl != NVME_NIDT_EUI64_LEN) {
908
				dev_warn(ctrl->device,
909 910 911 912 913
					 "ctrl returned bogus length: %d for NVME_NIDT_EUI64\n",
					 cur->nidl);
				goto free_data;
			}
			len = NVME_NIDT_EUI64_LEN;
914
			memcpy(ids->eui64, data + pos + sizeof(*cur), len);
915 916 917
			break;
		case NVME_NIDT_NGUID:
			if (cur->nidl != NVME_NIDT_NGUID_LEN) {
918
				dev_warn(ctrl->device,
919 920 921 922 923
					 "ctrl returned bogus length: %d for NVME_NIDT_NGUID\n",
					 cur->nidl);
				goto free_data;
			}
			len = NVME_NIDT_NGUID_LEN;
924
			memcpy(ids->nguid, data + pos + sizeof(*cur), len);
925 926 927
			break;
		case NVME_NIDT_UUID:
			if (cur->nidl != NVME_NIDT_UUID_LEN) {
928
				dev_warn(ctrl->device,
929 930 931 932 933
					 "ctrl returned bogus length: %d for NVME_NIDT_UUID\n",
					 cur->nidl);
				goto free_data;
			}
			len = NVME_NIDT_UUID_LEN;
934
			uuid_copy(&ids->uuid, data + pos + sizeof(*cur));
935 936 937 938 939 940 941 942 943 944 945 946 947 948
			break;
		default:
			/* Skip unnkown types */
			len = cur->nidl;
			break;
		}

		len += sizeof(*cur);
	}
free_data:
	kfree(data);
	return status;
}

949 950 951 952 953
static int nvme_identify_ns_list(struct nvme_ctrl *dev, unsigned nsid, __le32 *ns_list)
{
	struct nvme_command c = { };

	c.identify.opcode = nvme_admin_identify;
954
	c.identify.cns = NVME_ID_CNS_NS_ACTIVE_LIST;
955
	c.identify.nsid = cpu_to_le32(nsid);
956 957
	return nvme_submit_sync_cmd(dev->admin_q, &c, ns_list,
				    NVME_IDENTIFY_DATA_SIZE);
958 959
}

960 961
static struct nvme_id_ns *nvme_identify_ns(struct nvme_ctrl *ctrl,
		unsigned nsid)
962
{
963
	struct nvme_id_ns *id;
964 965 966 967
	struct nvme_command c = { };
	int error;

	/* gcc-4.4.4 (at least) has issues with initializers and anon unions */
968 969
	c.identify.opcode = nvme_admin_identify;
	c.identify.nsid = cpu_to_le32(nsid);
970
	c.identify.cns = NVME_ID_CNS_NS;
971

972 973 974
	id = kmalloc(sizeof(*id), GFP_KERNEL);
	if (!id)
		return NULL;
975

976 977 978 979 980 981 982 983
	error = nvme_submit_sync_cmd(ctrl->admin_q, &c, id, sizeof(*id));
	if (error) {
		dev_warn(ctrl->device, "Identify namespace failed\n");
		kfree(id);
		return NULL;
	}

	return id;
984 985
}

K
Keith Busch 已提交
986
static int nvme_set_features(struct nvme_ctrl *dev, unsigned fid, unsigned dword11,
987
		      void *buffer, size_t buflen, u32 *result)
988 989
{
	struct nvme_command c;
990
	union nvme_result res;
991
	int ret;
992 993 994 995 996 997

	memset(&c, 0, sizeof(c));
	c.features.opcode = nvme_admin_set_features;
	c.features.fid = cpu_to_le32(fid);
	c.features.dword11 = cpu_to_le32(dword11);

998
	ret = __nvme_submit_sync_cmd(dev->admin_q, &c, &res,
999
			buffer, buflen, 0, NVME_QID_ANY, 0, 0);
1000
	if (ret >= 0 && result)
1001
		*result = le32_to_cpu(res.u32);
1002
	return ret;
1003 1004
}

C
Christoph Hellwig 已提交
1005 1006 1007 1008 1009 1010
int nvme_set_queue_count(struct nvme_ctrl *ctrl, int *count)
{
	u32 q_count = (*count - 1) | ((*count - 1) << 16);
	u32 result;
	int status, nr_io_queues;

1011
	status = nvme_set_features(ctrl, NVME_FEAT_NUM_QUEUES, q_count, NULL, 0,
C
Christoph Hellwig 已提交
1012
			&result);
1013
	if (status < 0)
C
Christoph Hellwig 已提交
1014 1015
		return status;

1016 1017 1018 1019 1020 1021
	/*
	 * Degraded controllers might return an error when setting the queue
	 * count.  We still want to be able to bring them online and offer
	 * access to the admin queue, as that might be only way to fix them up.
	 */
	if (status > 0) {
1022
		dev_err(ctrl->device, "Could not set queue count (%d)\n", status);
1023 1024 1025 1026 1027 1028
		*count = 0;
	} else {
		nr_io_queues = min(result & 0xffff, result >> 16) + 1;
		*count = min(*count, nr_io_queues);
	}

C
Christoph Hellwig 已提交
1029 1030
	return 0;
}
1031
EXPORT_SYMBOL_GPL(nvme_set_queue_count);
C
Christoph Hellwig 已提交
1032

1033 1034 1035 1036 1037 1038 1039 1040 1041
static int nvme_submit_io(struct nvme_ns *ns, struct nvme_user_io __user *uio)
{
	struct nvme_user_io io;
	struct nvme_command c;
	unsigned length, meta_len;
	void __user *metadata;

	if (copy_from_user(&io, uio, sizeof(io)))
		return -EFAULT;
1042 1043
	if (io.flags)
		return -EINVAL;
1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065 1066 1067 1068

	switch (io.opcode) {
	case nvme_cmd_write:
	case nvme_cmd_read:
	case nvme_cmd_compare:
		break;
	default:
		return -EINVAL;
	}

	length = (io.nblocks + 1) << ns->lba_shift;
	meta_len = (io.nblocks + 1) * ns->ms;
	metadata = (void __user *)(uintptr_t)io.metadata;

	if (ns->ext) {
		length += meta_len;
		meta_len = 0;
	} else if (meta_len) {
		if ((io.metadata & 3) || !io.metadata)
			return -EINVAL;
	}

	memset(&c, 0, sizeof(c));
	c.rw.opcode = io.opcode;
	c.rw.flags = io.flags;
C
Christoph Hellwig 已提交
1069
	c.rw.nsid = cpu_to_le32(ns->head->ns_id);
1070 1071 1072 1073 1074 1075 1076 1077
	c.rw.slba = cpu_to_le64(io.slba);
	c.rw.length = cpu_to_le16(io.nblocks);
	c.rw.control = cpu_to_le16(io.control);
	c.rw.dsmgmt = cpu_to_le32(io.dsmgmt);
	c.rw.reftag = cpu_to_le32(io.reftag);
	c.rw.apptag = cpu_to_le16(io.apptag);
	c.rw.appmask = cpu_to_le16(io.appmask);

1078
	return nvme_submit_user_cmd(ns->queue, &c,
1079 1080 1081 1082
			(void __user *)(uintptr_t)io.addr, length,
			metadata, meta_len, io.slba, NULL, 0);
}

1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095 1096 1097 1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112
static u32 nvme_known_admin_effects(u8 opcode)
{
	switch (opcode) {
	case nvme_admin_format_nvm:
		return NVME_CMD_EFFECTS_CSUPP | NVME_CMD_EFFECTS_LBCC |
					NVME_CMD_EFFECTS_CSE_MASK;
	case nvme_admin_sanitize_nvm:
		return NVME_CMD_EFFECTS_CSE_MASK;
	default:
		break;
	}
	return 0;
}

static u32 nvme_passthru_start(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
								u8 opcode)
{
	u32 effects = 0;

	if (ns) {
		if (ctrl->effects)
			effects = le32_to_cpu(ctrl->effects->iocs[opcode]);
		if (effects & ~NVME_CMD_EFFECTS_CSUPP)
			dev_warn(ctrl->device,
				 "IO command:%02x has unhandled effects:%08x\n",
				 opcode, effects);
		return 0;
	}

	if (ctrl->effects)
1113
		effects = le32_to_cpu(ctrl->effects->acs[opcode]);
1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
	else
		effects = nvme_known_admin_effects(opcode);

	/*
	 * For simplicity, IO to all namespaces is quiesced even if the command
	 * effects say only one namespace is affected.
	 */
	if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK)) {
		nvme_start_freeze(ctrl);
		nvme_wait_freeze(ctrl);
	}
	return effects;
}

static void nvme_update_formats(struct nvme_ctrl *ctrl)
{
1130 1131
	struct nvme_ns *ns, *next;
	LIST_HEAD(rm_list);
1132

1133
	down_write(&ctrl->namespaces_rwsem);
1134
	list_for_each_entry(ns, &ctrl->namespaces, list) {
1135 1136 1137
		if (ns->disk && nvme_revalidate_disk(ns->disk)) {
			list_move_tail(&ns->list, &rm_list);
		}
1138
	}
1139
	up_write(&ctrl->namespaces_rwsem);
1140 1141 1142

	list_for_each_entry_safe(ns, next, &rm_list, list)
		nvme_ns_remove(ns);
1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154 1155 1156 1157 1158 1159 1160 1161
}

static void nvme_passthru_end(struct nvme_ctrl *ctrl, u32 effects)
{
	/*
	 * Revalidate LBA changes prior to unfreezing. This is necessary to
	 * prevent memory corruption if a logical block size was changed by
	 * this command.
	 */
	if (effects & NVME_CMD_EFFECTS_LBCC)
		nvme_update_formats(ctrl);
	if (effects & (NVME_CMD_EFFECTS_LBCC | NVME_CMD_EFFECTS_CSE_MASK))
		nvme_unfreeze(ctrl);
	if (effects & NVME_CMD_EFFECTS_CCC)
		nvme_init_identify(ctrl);
	if (effects & (NVME_CMD_EFFECTS_NIC | NVME_CMD_EFFECTS_NCC))
		nvme_queue_scan(ctrl);
}

1162
static int nvme_user_cmd(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
1163 1164 1165 1166 1167
			struct nvme_passthru_cmd __user *ucmd)
{
	struct nvme_passthru_cmd cmd;
	struct nvme_command c;
	unsigned timeout = 0;
1168
	u32 effects;
1169 1170 1171 1172 1173 1174
	int status;

	if (!capable(CAP_SYS_ADMIN))
		return -EACCES;
	if (copy_from_user(&cmd, ucmd, sizeof(cmd)))
		return -EFAULT;
1175 1176
	if (cmd.flags)
		return -EINVAL;
1177 1178 1179 1180 1181 1182 1183 1184 1185 1186 1187 1188 1189 1190 1191 1192 1193

	memset(&c, 0, sizeof(c));
	c.common.opcode = cmd.opcode;
	c.common.flags = cmd.flags;
	c.common.nsid = cpu_to_le32(cmd.nsid);
	c.common.cdw2[0] = cpu_to_le32(cmd.cdw2);
	c.common.cdw2[1] = cpu_to_le32(cmd.cdw3);
	c.common.cdw10[0] = cpu_to_le32(cmd.cdw10);
	c.common.cdw10[1] = cpu_to_le32(cmd.cdw11);
	c.common.cdw10[2] = cpu_to_le32(cmd.cdw12);
	c.common.cdw10[3] = cpu_to_le32(cmd.cdw13);
	c.common.cdw10[4] = cpu_to_le32(cmd.cdw14);
	c.common.cdw10[5] = cpu_to_le32(cmd.cdw15);

	if (cmd.timeout_ms)
		timeout = msecs_to_jiffies(cmd.timeout_ms);

1194
	effects = nvme_passthru_start(ctrl, ns, cmd.opcode);
1195
	status = nvme_submit_user_cmd(ns ? ns->queue : ctrl->admin_q, &c,
1196
			(void __user *)(uintptr_t)cmd.addr, cmd.data_len,
1197 1198
			(void __user *)(uintptr_t)cmd.metadata, cmd.metadata,
			0, &cmd.result, timeout);
1199 1200
	nvme_passthru_end(ctrl, effects);

1201 1202 1203 1204 1205 1206 1207 1208
	if (status >= 0) {
		if (put_user(cmd.result, &ucmd->result))
			return -EFAULT;
	}

	return status;
}

1209 1210 1211 1212 1213 1214
/*
 * Issue ioctl requests on the first available path.  Note that unlike normal
 * block layer requests we will not retry failed request on another controller.
 */
static struct nvme_ns *nvme_get_ns_from_disk(struct gendisk *disk,
		struct nvme_ns_head **head, int *srcu_idx)
1215
{
1216 1217 1218 1219 1220 1221 1222 1223 1224 1225 1226
#ifdef CONFIG_NVME_MULTIPATH
	if (disk->fops == &nvme_ns_head_ops) {
		*head = disk->private_data;
		*srcu_idx = srcu_read_lock(&(*head)->srcu);
		return nvme_find_path(*head);
	}
#endif
	*head = NULL;
	*srcu_idx = -1;
	return disk->private_data;
}
1227

1228 1229 1230 1231 1232
static void nvme_put_ns_from_disk(struct nvme_ns_head *head, int idx)
{
	if (head)
		srcu_read_unlock(&head->srcu, idx);
}
1233

1234 1235
static int nvme_ns_ioctl(struct nvme_ns *ns, unsigned cmd, unsigned long arg)
{
1236 1237 1238
	switch (cmd) {
	case NVME_IOCTL_ID:
		force_successful_syscall_return();
C
Christoph Hellwig 已提交
1239
		return ns->head->ns_id;
1240 1241 1242 1243 1244 1245 1246
	case NVME_IOCTL_ADMIN_CMD:
		return nvme_user_cmd(ns->ctrl, NULL, (void __user *)arg);
	case NVME_IOCTL_IO_CMD:
		return nvme_user_cmd(ns->ctrl, ns, (void __user *)arg);
	case NVME_IOCTL_SUBMIT_IO:
		return nvme_submit_io(ns, (void __user *)arg);
	default:
1247 1248 1249 1250
#ifdef CONFIG_NVM
		if (ns->ndev)
			return nvme_nvm_ioctl(ns, cmd, arg);
#endif
1251
		if (is_sed_ioctl(cmd))
1252
			return sed_ioctl(ns->ctrl->opal_dev, cmd,
1253
					 (void __user *) arg);
1254 1255 1256 1257
		return -ENOTTY;
	}
}

1258 1259
static int nvme_ioctl(struct block_device *bdev, fmode_t mode,
		unsigned int cmd, unsigned long arg)
1260
{
1261 1262 1263 1264 1265 1266 1267 1268 1269 1270 1271
	struct nvme_ns_head *head = NULL;
	struct nvme_ns *ns;
	int srcu_idx, ret;

	ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
	if (unlikely(!ns))
		ret = -EWOULDBLOCK;
	else
		ret = nvme_ns_ioctl(ns, cmd, arg);
	nvme_put_ns_from_disk(head, srcu_idx);
	return ret;
1272 1273 1274 1275
}

static int nvme_open(struct block_device *bdev, fmode_t mode)
{
C
Christoph Hellwig 已提交
1276 1277
	struct nvme_ns *ns = bdev->bd_disk->private_data;

1278 1279 1280
#ifdef CONFIG_NVME_MULTIPATH
	/* should never be called due to GENHD_FL_HIDDEN */
	if (WARN_ON_ONCE(ns->head->disk))
1281
		goto fail;
1282
#endif
C
Christoph Hellwig 已提交
1283
	if (!kref_get_unless_zero(&ns->kref))
1284 1285 1286 1287
		goto fail;
	if (!try_module_get(ns->ctrl->ops->module))
		goto fail_put_ns;

C
Christoph Hellwig 已提交
1288
	return 0;
1289 1290 1291 1292 1293

fail_put_ns:
	nvme_put_ns(ns);
fail:
	return -ENXIO;
1294 1295 1296 1297
}

static void nvme_release(struct gendisk *disk, fmode_t mode)
{
1298 1299 1300 1301
	struct nvme_ns *ns = disk->private_data;

	module_put(ns->ctrl->ops->module);
	nvme_put_ns(ns);
1302 1303 1304 1305 1306 1307 1308 1309 1310 1311 1312 1313
}

static int nvme_getgeo(struct block_device *bdev, struct hd_geometry *geo)
{
	/* some standard values */
	geo->heads = 1 << 6;
	geo->sectors = 1 << 5;
	geo->cylinders = get_capacity(bdev->bd_disk) >> 11;
	return 0;
}

#ifdef CONFIG_BLK_DEV_INTEGRITY
1314
static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
1315 1316 1317
{
	struct blk_integrity integrity;

1318
	memset(&integrity, 0, sizeof(integrity));
1319
	switch (pi_type) {
1320 1321
	case NVME_NS_DPS_PI_TYPE3:
		integrity.profile = &t10_pi_type3_crc;
1322 1323
		integrity.tag_size = sizeof(u16) + sizeof(u32);
		integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1324 1325 1326 1327
		break;
	case NVME_NS_DPS_PI_TYPE1:
	case NVME_NS_DPS_PI_TYPE2:
		integrity.profile = &t10_pi_type1_crc;
1328 1329
		integrity.tag_size = sizeof(u16);
		integrity.flags |= BLK_INTEGRITY_DEVICE_CAPABLE;
1330 1331 1332 1333 1334
		break;
	default:
		integrity.profile = NULL;
		break;
	}
1335 1336 1337
	integrity.tuple_size = ms;
	blk_integrity_register(disk, &integrity);
	blk_queue_max_integrity_segments(disk->queue, 1);
1338 1339
}
#else
1340
static void nvme_init_integrity(struct gendisk *disk, u16 ms, u8 pi_type)
1341 1342 1343 1344
{
}
#endif /* CONFIG_BLK_DEV_INTEGRITY */

1345 1346 1347 1348 1349 1350
static void nvme_set_chunk_size(struct nvme_ns *ns)
{
	u32 chunk_size = (((u32)ns->noiob) << (ns->lba_shift - 9));
	blk_queue_chunk_sectors(ns->queue, rounddown_pow_of_two(chunk_size));
}

1351
static void nvme_config_discard(struct nvme_ns *ns)
1352
{
1353 1354
	struct nvme_ctrl *ctrl = ns->ctrl;
	struct request_queue *queue = ns->queue;
1355 1356
	u32 size = queue_logical_block_size(queue);

1357 1358 1359 1360 1361 1362 1363
	if (!(ctrl->oncs & NVME_CTRL_ONCS_DSM)) {
		blk_queue_flag_clear(QUEUE_FLAG_DISCARD, queue);
		return;
	}

	if (ctrl->nr_streams && ns->sws && ns->sgs)
		size *= ns->sws * ns->sgs;
1364

1365 1366 1367
	BUILD_BUG_ON(PAGE_SIZE / sizeof(struct nvme_dsm_range) <
			NVME_DSM_MAX_RANGES);

1368
	queue->limits.discard_alignment = 0;
1369
	queue->limits.discard_granularity = size;
1370

1371 1372 1373 1374
	/* If discard is already enabled, don't reset queue limits */
	if (blk_queue_flag_test_and_set(QUEUE_FLAG_DISCARD, queue))
		return;

1375 1376
	blk_queue_max_discard_sectors(queue, UINT_MAX);
	blk_queue_max_discard_segments(queue, NVME_DSM_MAX_RANGES);
1377 1378

	if (ctrl->quirks & NVME_QUIRK_DEALLOCATE_ZEROES)
1379
		blk_queue_max_write_zeroes_sectors(queue, UINT_MAX);
1380 1381
}

1382
static void nvme_report_ns_ids(struct nvme_ctrl *ctrl, unsigned int nsid,
1383
		struct nvme_id_ns *id, struct nvme_ns_ids *ids)
1384
{
1385 1386
	memset(ids, 0, sizeof(*ids));

1387
	if (ctrl->vs >= NVME_VS(1, 1, 0))
1388
		memcpy(ids->eui64, id->eui64, sizeof(id->eui64));
1389
	if (ctrl->vs >= NVME_VS(1, 2, 0))
1390
		memcpy(ids->nguid, id->nguid, sizeof(id->nguid));
1391
	if (ctrl->vs >= NVME_VS(1, 3, 0)) {
1392 1393 1394
		 /* Don't treat error as fatal we potentially
		  * already have a NGUID or EUI-64
		  */
1395
		if (nvme_identify_ns_descs(ctrl, nsid, ids))
1396
			dev_warn(ctrl->device,
1397 1398
				 "%s: Identify Descriptors failed\n", __func__);
	}
1399 1400
}

C
Christoph Hellwig 已提交
1401 1402 1403 1404 1405 1406 1407
static bool nvme_ns_ids_valid(struct nvme_ns_ids *ids)
{
	return !uuid_is_null(&ids->uuid) ||
		memchr_inv(ids->nguid, 0, sizeof(ids->nguid)) ||
		memchr_inv(ids->eui64, 0, sizeof(ids->eui64));
}

1408 1409 1410 1411 1412 1413 1414
static bool nvme_ns_ids_equal(struct nvme_ns_ids *a, struct nvme_ns_ids *b)
{
	return uuid_equal(&a->uuid, &b->uuid) &&
		memcmp(&a->nguid, &b->nguid, sizeof(a->nguid)) == 0 &&
		memcmp(&a->eui64, &b->eui64, sizeof(a->eui64)) == 0;
}

1415 1416 1417 1418
static void nvme_update_disk_info(struct gendisk *disk,
		struct nvme_ns *ns, struct nvme_id_ns *id)
{
	sector_t capacity = le64_to_cpup(&id->nsze) << (ns->lba_shift - 9);
1419
	unsigned short bs = 1 << ns->lba_shift;
1420 1421 1422 1423

	blk_mq_freeze_queue(disk->queue);
	blk_integrity_unregister(disk);

1424 1425 1426 1427
	blk_queue_logical_block_size(disk->queue, bs);
	blk_queue_physical_block_size(disk->queue, bs);
	blk_queue_io_min(disk->queue, bs);

1428 1429 1430
	if (ns->ms && !ns->ext &&
	    (ns->ctrl->ops->flags & NVME_F_METADATA_SUPPORTED))
		nvme_init_integrity(disk, ns->ms, ns->pi_type);
1431
	if (ns->ms && !nvme_ns_has_pi(ns) && !blk_get_integrity(disk))
1432 1433
		capacity = 0;

1434 1435
	set_capacity(disk, capacity);
	nvme_config_discard(ns);
1436 1437 1438
	blk_mq_unfreeze_queue(disk->queue);
}

1439 1440 1441
static void __nvme_revalidate_disk(struct gendisk *disk, struct nvme_id_ns *id)
{
	struct nvme_ns *ns = disk->private_data;
1442 1443 1444 1445 1446

	/*
	 * If identify namespace failed, use default 512 byte block size so
	 * block layer can use before failing read/write for 0 capacity.
	 */
1447
	ns->lba_shift = id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ds;
1448 1449
	if (ns->lba_shift == 0)
		ns->lba_shift = 9;
1450
	ns->noiob = le16_to_cpu(id->noiob);
1451 1452 1453 1454 1455 1456 1457
	ns->ext = ns->ms && (id->flbas & NVME_NS_FLBAS_META_EXT);
	ns->ms = le16_to_cpu(id->lbaf[id->flbas & NVME_NS_FLBAS_LBA_MASK].ms);
	/* the PI implementation requires metadata equal t10 pi tuple size */
	if (ns->ms == sizeof(struct t10_pi_tuple))
		ns->pi_type = id->dps & NVME_NS_DPS_PI_MASK;
	else
		ns->pi_type = 0;
1458

1459 1460
	if (ns->noiob)
		nvme_set_chunk_size(ns);
1461
	nvme_update_disk_info(disk, ns, id);
1462 1463
	if (ns->ndev)
		nvme_nvm_update_nvm_info(ns);
1464 1465 1466 1467
#ifdef CONFIG_NVME_MULTIPATH
	if (ns->head->disk)
		nvme_update_disk_info(ns->head->disk, ns, id);
#endif
1468
}
1469

1470 1471 1472
static int nvme_revalidate_disk(struct gendisk *disk)
{
	struct nvme_ns *ns = disk->private_data;
1473 1474
	struct nvme_ctrl *ctrl = ns->ctrl;
	struct nvme_id_ns *id;
1475
	struct nvme_ns_ids ids;
1476
	int ret = 0;
1477 1478 1479 1480 1481 1482

	if (test_bit(NVME_NS_DEAD, &ns->flags)) {
		set_capacity(disk, 0);
		return -ENODEV;
	}

C
Christoph Hellwig 已提交
1483
	id = nvme_identify_ns(ctrl, ns->head->ns_id);
1484 1485
	if (!id)
		return -ENODEV;
1486

1487 1488 1489 1490
	if (id->ncap == 0) {
		ret = -ENODEV;
		goto out;
	}
1491

1492
	__nvme_revalidate_disk(disk, id);
C
Christoph Hellwig 已提交
1493 1494
	nvme_report_ns_ids(ctrl, ns->head->ns_id, id, &ids);
	if (!nvme_ns_ids_equal(&ns->head->ids, &ids)) {
1495
		dev_err(ctrl->device,
C
Christoph Hellwig 已提交
1496
			"identifiers changed for nsid %d\n", ns->head->ns_id);
1497 1498 1499
		ret = -ENODEV;
	}

1500 1501 1502
out:
	kfree(id);
	return ret;
1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527
}

static char nvme_pr_type(enum pr_type type)
{
	switch (type) {
	case PR_WRITE_EXCLUSIVE:
		return 1;
	case PR_EXCLUSIVE_ACCESS:
		return 2;
	case PR_WRITE_EXCLUSIVE_REG_ONLY:
		return 3;
	case PR_EXCLUSIVE_ACCESS_REG_ONLY:
		return 4;
	case PR_WRITE_EXCLUSIVE_ALL_REGS:
		return 5;
	case PR_EXCLUSIVE_ACCESS_ALL_REGS:
		return 6;
	default:
		return 0;
	}
};

static int nvme_pr_command(struct block_device *bdev, u32 cdw10,
				u64 key, u64 sa_key, u8 op)
{
1528 1529
	struct nvme_ns_head *head = NULL;
	struct nvme_ns *ns;
1530
	struct nvme_command c;
1531
	int srcu_idx, ret;
1532 1533
	u8 data[16] = { 0, };

1534 1535 1536 1537
	ns = nvme_get_ns_from_disk(bdev->bd_disk, &head, &srcu_idx);
	if (unlikely(!ns))
		return -EWOULDBLOCK;

1538 1539 1540 1541 1542
	put_unaligned_le64(key, &data[0]);
	put_unaligned_le64(sa_key, &data[8]);

	memset(&c, 0, sizeof(c));
	c.common.opcode = op;
1543
	c.common.nsid = cpu_to_le32(ns->head->ns_id);
1544 1545
	c.common.cdw10[0] = cpu_to_le32(cdw10);

1546
	ret = nvme_submit_sync_cmd(ns->queue, &c, data, 16);
1547 1548
	nvme_put_ns_from_disk(head, srcu_idx);
	return ret;
1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580
}

static int nvme_pr_register(struct block_device *bdev, u64 old,
		u64 new, unsigned flags)
{
	u32 cdw10;

	if (flags & ~PR_FL_IGNORE_KEY)
		return -EOPNOTSUPP;

	cdw10 = old ? 2 : 0;
	cdw10 |= (flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0;
	cdw10 |= (1 << 30) | (1 << 31); /* PTPL=1 */
	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_register);
}

static int nvme_pr_reserve(struct block_device *bdev, u64 key,
		enum pr_type type, unsigned flags)
{
	u32 cdw10;

	if (flags & ~PR_FL_IGNORE_KEY)
		return -EOPNOTSUPP;

	cdw10 = nvme_pr_type(type) << 8;
	cdw10 |= ((flags & PR_FL_IGNORE_KEY) ? 1 << 3 : 0);
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_acquire);
}

static int nvme_pr_preempt(struct block_device *bdev, u64 old, u64 new,
		enum pr_type type, bool abort)
{
1581
	u32 cdw10 = nvme_pr_type(type) << 8 | (abort ? 2 : 1);
1582 1583 1584 1585 1586
	return nvme_pr_command(bdev, cdw10, old, new, nvme_cmd_resv_acquire);
}

static int nvme_pr_clear(struct block_device *bdev, u64 key)
{
1587
	u32 cdw10 = 1 | (key ? 1 << 3 : 0);
1588 1589 1590 1591 1592
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_register);
}

static int nvme_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
{
1593
	u32 cdw10 = nvme_pr_type(type) << 8 | (key ? 1 << 3 : 0);
1594 1595 1596 1597 1598 1599 1600 1601 1602 1603 1604
	return nvme_pr_command(bdev, cdw10, key, 0, nvme_cmd_resv_release);
}

static const struct pr_ops nvme_pr_ops = {
	.pr_register	= nvme_pr_register,
	.pr_reserve	= nvme_pr_reserve,
	.pr_release	= nvme_pr_release,
	.pr_preempt	= nvme_pr_preempt,
	.pr_clear	= nvme_pr_clear,
};

1605
#ifdef CONFIG_BLK_SED_OPAL
1606 1607
int nvme_sec_submit(void *data, u16 spsp, u8 secp, void *buffer, size_t len,
		bool send)
1608
{
1609
	struct nvme_ctrl *ctrl = data;
1610 1611 1612 1613 1614 1615 1616 1617 1618 1619 1620 1621 1622 1623 1624 1625 1626
	struct nvme_command cmd;

	memset(&cmd, 0, sizeof(cmd));
	if (send)
		cmd.common.opcode = nvme_admin_security_send;
	else
		cmd.common.opcode = nvme_admin_security_recv;
	cmd.common.nsid = 0;
	cmd.common.cdw10[0] = cpu_to_le32(((u32)secp) << 24 | ((u32)spsp) << 8);
	cmd.common.cdw10[1] = cpu_to_le32(len);

	return __nvme_submit_sync_cmd(ctrl->admin_q, &cmd, NULL, buffer, len,
				      ADMIN_TIMEOUT, NVME_QID_ANY, 1, 0);
}
EXPORT_SYMBOL_GPL(nvme_sec_submit);
#endif /* CONFIG_BLK_SED_OPAL */

1627
static const struct block_device_operations nvme_fops = {
1628 1629
	.owner		= THIS_MODULE,
	.ioctl		= nvme_ioctl,
1630
	.compat_ioctl	= nvme_ioctl,
1631 1632 1633 1634 1635 1636 1637
	.open		= nvme_open,
	.release	= nvme_release,
	.getgeo		= nvme_getgeo,
	.revalidate_disk= nvme_revalidate_disk,
	.pr_ops		= &nvme_pr_ops,
};

1638 1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649 1650 1651 1652 1653 1654 1655 1656 1657 1658 1659 1660 1661 1662 1663
#ifdef CONFIG_NVME_MULTIPATH
static int nvme_ns_head_open(struct block_device *bdev, fmode_t mode)
{
	struct nvme_ns_head *head = bdev->bd_disk->private_data;

	if (!kref_get_unless_zero(&head->ref))
		return -ENXIO;
	return 0;
}

static void nvme_ns_head_release(struct gendisk *disk, fmode_t mode)
{
	nvme_put_ns_head(disk->private_data);
}

const struct block_device_operations nvme_ns_head_ops = {
	.owner		= THIS_MODULE,
	.open		= nvme_ns_head_open,
	.release	= nvme_ns_head_release,
	.ioctl		= nvme_ioctl,
	.compat_ioctl	= nvme_ioctl,
	.getgeo		= nvme_getgeo,
	.pr_ops		= &nvme_pr_ops,
};
#endif /* CONFIG_NVME_MULTIPATH */

1664 1665 1666 1667 1668 1669 1670 1671
static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled)
{
	unsigned long timeout =
		((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies;
	u32 csts, bit = enabled ? NVME_CSTS_RDY : 0;
	int ret;

	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
K
Keith Busch 已提交
1672 1673
		if (csts == ~0)
			return -ENODEV;
1674 1675 1676 1677 1678 1679 1680
		if ((csts & NVME_CSTS_RDY) == bit)
			break;

		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
1681
			dev_err(ctrl->device,
1682 1683 1684 1685 1686 1687 1688 1689 1690 1691 1692 1693 1694 1695 1696 1697 1698 1699 1700 1701 1702 1703 1704 1705 1706
				"Device not ready; aborting %s\n", enabled ?
						"initialisation" : "reset");
			return -ENODEV;
		}
	}

	return ret;
}

/*
 * If the device has been passed off to us in an enabled state, just clear
 * the enabled bit.  The spec says we should set the 'shutdown notification
 * bits', but doing so may cause the device to complete commands to the
 * admin queue ... and we don't know what memory that might be pointing at!
 */
int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
{
	int ret;

	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
	ctrl->ctrl_config &= ~NVME_CC_ENABLE;

	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
	if (ret)
		return ret;
1707

1708
	if (ctrl->quirks & NVME_QUIRK_DELAY_BEFORE_CHK_RDY)
1709 1710
		msleep(NVME_QUIRK_DELAY_AMOUNT);

1711 1712
	return nvme_wait_ready(ctrl, cap, false);
}
1713
EXPORT_SYMBOL_GPL(nvme_disable_ctrl);
1714 1715 1716 1717 1718 1719 1720 1721 1722 1723 1724 1725

int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap)
{
	/*
	 * Default to a 4K page size, with the intention to update this
	 * path in the future to accomodate architectures with differing
	 * kernel and IO page sizes.
	 */
	unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12;
	int ret;

	if (page_shift < dev_page_min) {
1726
		dev_err(ctrl->device,
1727 1728 1729 1730 1731 1732 1733 1734 1735
			"Minimum device page size %u too large for host (%u)\n",
			1 << dev_page_min, 1 << page_shift);
		return -ENODEV;
	}

	ctrl->page_size = 1 << page_shift;

	ctrl->ctrl_config = NVME_CC_CSS_NVM;
	ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT;
1736
	ctrl->ctrl_config |= NVME_CC_AMS_RR | NVME_CC_SHN_NONE;
1737 1738 1739 1740 1741 1742 1743 1744
	ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES;
	ctrl->ctrl_config |= NVME_CC_ENABLE;

	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
	if (ret)
		return ret;
	return nvme_wait_ready(ctrl, cap, true);
}
1745
EXPORT_SYMBOL_GPL(nvme_enable_ctrl);
1746 1747 1748

int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl)
{
1749
	unsigned long timeout = jiffies + (ctrl->shutdown_timeout * HZ);
1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767
	u32 csts;
	int ret;

	ctrl->ctrl_config &= ~NVME_CC_SHN_MASK;
	ctrl->ctrl_config |= NVME_CC_SHN_NORMAL;

	ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config);
	if (ret)
		return ret;

	while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) {
		if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT)
			break;

		msleep(100);
		if (fatal_signal_pending(current))
			return -EINTR;
		if (time_after(jiffies, timeout)) {
1768
			dev_err(ctrl->device,
1769 1770 1771 1772 1773 1774 1775
				"Device shutdown incomplete; abort shutdown\n");
			return -ENODEV;
		}
	}

	return ret;
}
1776
EXPORT_SYMBOL_GPL(nvme_shutdown_ctrl);
1777

1778 1779 1780
static void nvme_set_queue_limits(struct nvme_ctrl *ctrl,
		struct request_queue *q)
{
1781 1782
	bool vwc = false;

1783
	if (ctrl->max_hw_sectors) {
1784 1785 1786
		u32 max_segments =
			(ctrl->max_hw_sectors / (ctrl->page_size >> 9)) + 1;

1787
		blk_queue_max_hw_sectors(q, ctrl->max_hw_sectors);
1788
		blk_queue_max_segments(q, min_t(u32, max_segments, USHRT_MAX));
1789
	}
1790 1791
	if ((ctrl->quirks & NVME_QUIRK_STRIPE_SIZE) &&
	    is_power_of_2(ctrl->max_hw_sectors))
K
Keith Busch 已提交
1792
		blk_queue_chunk_sectors(q, ctrl->max_hw_sectors);
1793
	blk_queue_virt_boundary(q, ctrl->page_size - 1);
1794 1795 1796
	if (ctrl->vwc & NVME_CTRL_VWC_PRESENT)
		vwc = true;
	blk_queue_write_cache(q, vwc, vwc);
1797 1798
}

1799 1800 1801 1802 1803 1804 1805 1806 1807 1808 1809 1810 1811 1812 1813 1814 1815
static int nvme_configure_timestamp(struct nvme_ctrl *ctrl)
{
	__le64 ts;
	int ret;

	if (!(ctrl->oncs & NVME_CTRL_ONCS_TIMESTAMP))
		return 0;

	ts = cpu_to_le64(ktime_to_ms(ktime_get_real()));
	ret = nvme_set_features(ctrl, NVME_FEAT_TIMESTAMP, 0, &ts, sizeof(ts),
			NULL);
	if (ret)
		dev_warn_once(ctrl->device,
			"could not set timestamp (%d)\n", ret);
	return ret;
}

1816
static int nvme_configure_apst(struct nvme_ctrl *ctrl)
1817 1818 1819 1820 1821 1822 1823 1824
{
	/*
	 * APST (Autonomous Power State Transition) lets us program a
	 * table of power state transitions that the controller will
	 * perform automatically.  We configure it with a simple
	 * heuristic: we are willing to spend at most 2% of the time
	 * transitioning between power states.  Therefore, when running
	 * in any given state, we will enter the next lower-power
A
Andy Lutomirski 已提交
1825
	 * non-operational state after waiting 50 * (enlat + exlat)
1826
	 * microseconds, as long as that state's exit latency is under
1827 1828 1829 1830 1831 1832 1833 1834 1835
	 * the requested maximum latency.
	 *
	 * We will not autonomously enter any non-operational state for
	 * which the total latency exceeds ps_max_latency_us.  Users
	 * can set ps_max_latency_us to zero to turn off APST.
	 */

	unsigned apste;
	struct nvme_feat_auto_pst *table;
1836 1837
	u64 max_lat_us = 0;
	int max_ps = -1;
1838 1839 1840 1841 1842 1843 1844
	int ret;

	/*
	 * If APST isn't supported or if we haven't been initialized yet,
	 * then don't do anything.
	 */
	if (!ctrl->apsta)
1845
		return 0;
1846 1847 1848

	if (ctrl->npss > 31) {
		dev_warn(ctrl->device, "NPSS is invalid; not using APST\n");
1849
		return 0;
1850 1851 1852 1853
	}

	table = kzalloc(sizeof(*table), GFP_KERNEL);
	if (!table)
1854
		return 0;
1855

1856
	if (!ctrl->apst_enabled || ctrl->ps_max_latency_us == 0) {
1857 1858
		/* Turn off APST. */
		apste = 0;
1859
		dev_dbg(ctrl->device, "APST disabled\n");
1860 1861 1862 1863 1864 1865 1866 1867 1868 1869 1870
	} else {
		__le64 target = cpu_to_le64(0);
		int state;

		/*
		 * Walk through all states from lowest- to highest-power.
		 * According to the spec, lower-numbered states use more
		 * power.  NPSS, despite the name, is the index of the
		 * lowest-power state, not the number of states.
		 */
		for (state = (int)ctrl->npss; state >= 0; state--) {
1871
			u64 total_latency_us, exit_latency_us, transition_ms;
1872 1873 1874 1875

			if (target)
				table->entries[state] = target;

1876 1877 1878 1879 1880 1881 1882 1883
			/*
			 * Don't allow transitions to the deepest state
			 * if it's quirked off.
			 */
			if (state == ctrl->npss &&
			    (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS))
				continue;

1884 1885 1886 1887 1888 1889 1890 1891
			/*
			 * Is this state a useful non-operational state for
			 * higher-power states to autonomously transition to?
			 */
			if (!(ctrl->psd[state].flags &
			      NVME_PS_FLAGS_NON_OP_STATE))
				continue;

1892 1893 1894
			exit_latency_us =
				(u64)le32_to_cpu(ctrl->psd[state].exit_lat);
			if (exit_latency_us > ctrl->ps_max_latency_us)
1895 1896
				continue;

1897 1898 1899 1900
			total_latency_us =
				exit_latency_us +
				le32_to_cpu(ctrl->psd[state].entry_lat);

1901 1902 1903 1904 1905 1906 1907 1908 1909 1910 1911
			/*
			 * This state is good.  Use it as the APST idle
			 * target for higher power states.
			 */
			transition_ms = total_latency_us + 19;
			do_div(transition_ms, 20);
			if (transition_ms > (1 << 24) - 1)
				transition_ms = (1 << 24) - 1;

			target = cpu_to_le64((state << 3) |
					     (transition_ms << 8));
1912 1913 1914 1915 1916 1917

			if (max_ps == -1)
				max_ps = state;

			if (total_latency_us > max_lat_us)
				max_lat_us = total_latency_us;
1918 1919 1920
		}

		apste = 1;
1921 1922 1923 1924 1925 1926 1927

		if (max_ps == -1) {
			dev_dbg(ctrl->device, "APST enabled but no non-operational states are available\n");
		} else {
			dev_dbg(ctrl->device, "APST enabled: max PS = %d, max round-trip latency = %lluus, table = %*phN\n",
				max_ps, max_lat_us, (int)sizeof(*table), table);
		}
1928 1929 1930 1931 1932 1933 1934 1935
	}

	ret = nvme_set_features(ctrl, NVME_FEAT_AUTO_PST, apste,
				table, sizeof(*table), NULL);
	if (ret)
		dev_err(ctrl->device, "failed to set APST feature (%d)\n", ret);

	kfree(table);
1936
	return ret;
1937 1938 1939 1940 1941 1942 1943 1944 1945 1946 1947 1948 1949 1950 1951 1952 1953 1954 1955 1956 1957 1958 1959
}

static void nvme_set_latency_tolerance(struct device *dev, s32 val)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	u64 latency;

	switch (val) {
	case PM_QOS_LATENCY_TOLERANCE_NO_CONSTRAINT:
	case PM_QOS_LATENCY_ANY:
		latency = U64_MAX;
		break;

	default:
		latency = val;
	}

	if (ctrl->ps_max_latency_us != latency) {
		ctrl->ps_max_latency_us = latency;
		nvme_configure_apst(ctrl);
	}
}

1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971 1972
struct nvme_core_quirk_entry {
	/*
	 * NVMe model and firmware strings are padded with spaces.  For
	 * simplicity, strings in the quirk table are padded with NULLs
	 * instead.
	 */
	u16 vid;
	const char *mn;
	const char *fr;
	unsigned long quirks;
};

static const struct nvme_core_quirk_entry core_quirks[] = {
1973
	{
1974 1975 1976 1977 1978 1979
		/*
		 * This Toshiba device seems to die using any APST states.  See:
		 * https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1678184/comments/11
		 */
		.vid = 0x1179,
		.mn = "THNSF5256GPUK TOSHIBA",
1980
		.quirks = NVME_QUIRK_NO_APST,
1981
	}
1982 1983 1984 1985 1986 1987 1988 1989 1990 1991 1992 1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008 2009 2010 2011 2012
};

/* match is null-terminated but idstr is space-padded. */
static bool string_matches(const char *idstr, const char *match, size_t len)
{
	size_t matchlen;

	if (!match)
		return true;

	matchlen = strlen(match);
	WARN_ON_ONCE(matchlen > len);

	if (memcmp(idstr, match, matchlen))
		return false;

	for (; matchlen < len; matchlen++)
		if (idstr[matchlen] != ' ')
			return false;

	return true;
}

static bool quirk_matches(const struct nvme_id_ctrl *id,
			  const struct nvme_core_quirk_entry *q)
{
	return q->vid == le16_to_cpu(id->vid) &&
		string_matches(id->mn, q->mn, sizeof(id->mn)) &&
		string_matches(id->fr, q->fr, sizeof(id->fr));
}

C
Christoph Hellwig 已提交
2013 2014
static void nvme_init_subnqn(struct nvme_subsystem *subsys, struct nvme_ctrl *ctrl,
		struct nvme_id_ctrl *id)
2015 2016 2017 2018 2019 2020
{
	size_t nqnlen;
	int off;

	nqnlen = strnlen(id->subnqn, NVMF_NQN_SIZE);
	if (nqnlen > 0 && nqnlen < NVMF_NQN_SIZE) {
C
Christoph Hellwig 已提交
2021
		strncpy(subsys->subnqn, id->subnqn, NVMF_NQN_SIZE);
2022 2023 2024 2025 2026 2027 2028
		return;
	}

	if (ctrl->vs >= NVME_VS(1, 2, 1))
		dev_warn(ctrl->device, "missing or invalid SUBNQN field.\n");

	/* Generate a "fake" NQN per Figure 254 in NVMe 1.3 + ECN 001 */
C
Christoph Hellwig 已提交
2029
	off = snprintf(subsys->subnqn, NVMF_NQN_SIZE,
2030 2031
			"nqn.2014.08.org.nvmexpress:%4x%4x",
			le16_to_cpu(id->vid), le16_to_cpu(id->ssvid));
C
Christoph Hellwig 已提交
2032
	memcpy(subsys->subnqn + off, id->sn, sizeof(id->sn));
2033
	off += sizeof(id->sn);
C
Christoph Hellwig 已提交
2034
	memcpy(subsys->subnqn + off, id->mn, sizeof(id->mn));
2035
	off += sizeof(id->mn);
C
Christoph Hellwig 已提交
2036 2037 2038 2039 2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055 2056 2057 2058
	memset(subsys->subnqn + off, 0, sizeof(subsys->subnqn) - off);
}

static void __nvme_release_subsystem(struct nvme_subsystem *subsys)
{
	ida_simple_remove(&nvme_subsystems_ida, subsys->instance);
	kfree(subsys);
}

static void nvme_release_subsystem(struct device *dev)
{
	__nvme_release_subsystem(container_of(dev, struct nvme_subsystem, dev));
}

static void nvme_destroy_subsystem(struct kref *ref)
{
	struct nvme_subsystem *subsys =
			container_of(ref, struct nvme_subsystem, ref);

	mutex_lock(&nvme_subsystems_lock);
	list_del(&subsys->entry);
	mutex_unlock(&nvme_subsystems_lock);

C
Christoph Hellwig 已提交
2059
	ida_destroy(&subsys->ns_ida);
C
Christoph Hellwig 已提交
2060 2061 2062 2063 2064 2065 2066 2067 2068 2069 2070 2071 2072 2073 2074 2075 2076 2077 2078 2079 2080 2081 2082 2083 2084 2085
	device_del(&subsys->dev);
	put_device(&subsys->dev);
}

static void nvme_put_subsystem(struct nvme_subsystem *subsys)
{
	kref_put(&subsys->ref, nvme_destroy_subsystem);
}

static struct nvme_subsystem *__nvme_find_get_subsystem(const char *subsysnqn)
{
	struct nvme_subsystem *subsys;

	lockdep_assert_held(&nvme_subsystems_lock);

	list_for_each_entry(subsys, &nvme_subsystems, entry) {
		if (strcmp(subsys->subnqn, subsysnqn))
			continue;
		if (!kref_get_unless_zero(&subsys->ref))
			continue;
		return subsys;
	}

	return NULL;
}

2086 2087 2088 2089 2090 2091 2092 2093 2094 2095 2096 2097 2098 2099 2100 2101 2102 2103 2104 2105 2106 2107 2108 2109 2110 2111 2112 2113 2114 2115 2116 2117 2118 2119 2120 2121 2122 2123 2124 2125 2126 2127 2128 2129 2130 2131 2132
#define SUBSYS_ATTR_RO(_name, _mode, _show)			\
	struct device_attribute subsys_attr_##_name = \
		__ATTR(_name, _mode, _show, NULL)

static ssize_t nvme_subsys_show_nqn(struct device *dev,
				    struct device_attribute *attr,
				    char *buf)
{
	struct nvme_subsystem *subsys =
		container_of(dev, struct nvme_subsystem, dev);

	return snprintf(buf, PAGE_SIZE, "%s\n", subsys->subnqn);
}
static SUBSYS_ATTR_RO(subsysnqn, S_IRUGO, nvme_subsys_show_nqn);

#define nvme_subsys_show_str_function(field)				\
static ssize_t subsys_##field##_show(struct device *dev,		\
			    struct device_attribute *attr, char *buf)	\
{									\
	struct nvme_subsystem *subsys =					\
		container_of(dev, struct nvme_subsystem, dev);		\
	return sprintf(buf, "%.*s\n",					\
		       (int)sizeof(subsys->field), subsys->field);	\
}									\
static SUBSYS_ATTR_RO(field, S_IRUGO, subsys_##field##_show);

nvme_subsys_show_str_function(model);
nvme_subsys_show_str_function(serial);
nvme_subsys_show_str_function(firmware_rev);

static struct attribute *nvme_subsys_attrs[] = {
	&subsys_attr_model.attr,
	&subsys_attr_serial.attr,
	&subsys_attr_firmware_rev.attr,
	&subsys_attr_subsysnqn.attr,
	NULL,
};

static struct attribute_group nvme_subsys_attrs_group = {
	.attrs = nvme_subsys_attrs,
};

static const struct attribute_group *nvme_subsys_attrs_groups[] = {
	&nvme_subsys_attrs_group,
	NULL,
};

2133 2134 2135 2136 2137 2138 2139 2140 2141 2142 2143 2144 2145 2146 2147 2148
static int nvme_active_ctrls(struct nvme_subsystem *subsys)
{
	int count = 0;
	struct nvme_ctrl *ctrl;

	mutex_lock(&subsys->lock);
	list_for_each_entry(ctrl, &subsys->ctrls, subsys_entry) {
		if (ctrl->state != NVME_CTRL_DELETING &&
		    ctrl->state != NVME_CTRL_DEAD)
			count++;
	}
	mutex_unlock(&subsys->lock);

	return count;
}

C
Christoph Hellwig 已提交
2149 2150 2151 2152 2153 2154 2155 2156 2157 2158 2159 2160 2161 2162 2163 2164 2165
static int nvme_init_subsystem(struct nvme_ctrl *ctrl, struct nvme_id_ctrl *id)
{
	struct nvme_subsystem *subsys, *found;
	int ret;

	subsys = kzalloc(sizeof(*subsys), GFP_KERNEL);
	if (!subsys)
		return -ENOMEM;
	ret = ida_simple_get(&nvme_subsystems_ida, 0, 0, GFP_KERNEL);
	if (ret < 0) {
		kfree(subsys);
		return ret;
	}
	subsys->instance = ret;
	mutex_init(&subsys->lock);
	kref_init(&subsys->ref);
	INIT_LIST_HEAD(&subsys->ctrls);
C
Christoph Hellwig 已提交
2166
	INIT_LIST_HEAD(&subsys->nsheads);
C
Christoph Hellwig 已提交
2167 2168 2169 2170 2171 2172 2173 2174 2175
	nvme_init_subnqn(subsys, ctrl, id);
	memcpy(subsys->serial, id->sn, sizeof(subsys->serial));
	memcpy(subsys->model, id->mn, sizeof(subsys->model));
	memcpy(subsys->firmware_rev, id->fr, sizeof(subsys->firmware_rev));
	subsys->vendor_id = le16_to_cpu(id->vid);
	subsys->cmic = id->cmic;

	subsys->dev.class = nvme_subsys_class;
	subsys->dev.release = nvme_release_subsystem;
2176
	subsys->dev.groups = nvme_subsys_attrs_groups;
C
Christoph Hellwig 已提交
2177 2178 2179 2180 2181 2182 2183 2184 2185 2186
	dev_set_name(&subsys->dev, "nvme-subsys%d", subsys->instance);
	device_initialize(&subsys->dev);

	mutex_lock(&nvme_subsystems_lock);
	found = __nvme_find_get_subsystem(subsys->subnqn);
	if (found) {
		/*
		 * Verify that the subsystem actually supports multiple
		 * controllers, else bail out.
		 */
2187 2188
		if (!ctrl->opts->discovery_nqn &&
		    nvme_active_ctrls(found) && !(id->cmic & (1 << 1))) {
C
Christoph Hellwig 已提交
2189 2190 2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205
			dev_err(ctrl->device,
				"ignoring ctrl due to duplicate subnqn (%s).\n",
				found->subnqn);
			nvme_put_subsystem(found);
			ret = -EINVAL;
			goto out_unlock;
		}

		__nvme_release_subsystem(subsys);
		subsys = found;
	} else {
		ret = device_add(&subsys->dev);
		if (ret) {
			dev_err(ctrl->device,
				"failed to register subsystem device.\n");
			goto out_unlock;
		}
C
Christoph Hellwig 已提交
2206
		ida_init(&subsys->ns_ida);
C
Christoph Hellwig 已提交
2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218 2219 2220 2221 2222 2223 2224 2225 2226 2227 2228 2229 2230
		list_add_tail(&subsys->entry, &nvme_subsystems);
	}

	ctrl->subsys = subsys;
	mutex_unlock(&nvme_subsystems_lock);

	if (sysfs_create_link(&subsys->dev.kobj, &ctrl->device->kobj,
			dev_name(ctrl->device))) {
		dev_err(ctrl->device,
			"failed to create sysfs link from subsystem.\n");
		/* the transport driver will eventually put the subsystem */
		return -EINVAL;
	}

	mutex_lock(&subsys->lock);
	list_add_tail(&ctrl->subsys_entry, &subsys->ctrls);
	mutex_unlock(&subsys->lock);

	return 0;

out_unlock:
	mutex_unlock(&nvme_subsystems_lock);
	put_device(&subsys->dev);
	return ret;
2231 2232
}

2233
int nvme_get_log_ext(struct nvme_ctrl *ctrl, struct nvme_ns *ns,
2234
		     u8 log_page, void *log,
2235
		     size_t size, u64 offset)
K
Keith Busch 已提交
2236 2237
{
	struct nvme_command c = { };
2238 2239 2240 2241 2242 2243 2244 2245
	unsigned long dwlen = size / 4 - 1;

	c.get_log_page.opcode = nvme_admin_get_log_page;

	if (ns)
		c.get_log_page.nsid = cpu_to_le32(ns->head->ns_id);
	else
		c.get_log_page.nsid = cpu_to_le32(NVME_NSID_ALL);
K
Keith Busch 已提交
2246

2247 2248 2249
	c.get_log_page.lid = log_page;
	c.get_log_page.numdl = cpu_to_le16(dwlen & ((1 << 16) - 1));
	c.get_log_page.numdu = cpu_to_le16(dwlen >> 16);
2250 2251
	c.get_log_page.lpol = cpu_to_le32(lower_32_bits(offset));
	c.get_log_page.lpou = cpu_to_le32(upper_32_bits(offset));
K
Keith Busch 已提交
2252 2253 2254 2255

	return nvme_submit_sync_cmd(ctrl->admin_q, &c, log, size);
}

2256 2257 2258 2259 2260 2261
static int nvme_get_log(struct nvme_ctrl *ctrl, u8 log_page, void *log,
			size_t size)
{
	return nvme_get_log_ext(ctrl, NULL, log_page, log, size, 0);
}

2262 2263 2264 2265 2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278
static int nvme_get_effects_log(struct nvme_ctrl *ctrl)
{
	int ret;

	if (!ctrl->effects)
		ctrl->effects = kzalloc(sizeof(*ctrl->effects), GFP_KERNEL);

	if (!ctrl->effects)
		return 0;

	ret = nvme_get_log(ctrl, NVME_LOG_CMD_EFFECTS, ctrl->effects,
					sizeof(*ctrl->effects));
	if (ret) {
		kfree(ctrl->effects);
		ctrl->effects = NULL;
	}
	return ret;
2279 2280
}

2281 2282 2283 2284 2285 2286 2287 2288 2289 2290
/*
 * Initialize the cached copies of the Identify data and various controller
 * register in our nvme_ctrl structure.  This should be called as soon as
 * the admin queue is fully up and running.
 */
int nvme_init_identify(struct nvme_ctrl *ctrl)
{
	struct nvme_id_ctrl *id;
	u64 cap;
	int ret, page_shift;
2291
	u32 max_hw_sectors;
2292
	bool prev_apst_enabled;
2293

2294 2295
	ret = ctrl->ops->reg_read32(ctrl, NVME_REG_VS, &ctrl->vs);
	if (ret) {
2296
		dev_err(ctrl->device, "Reading VS failed (%d)\n", ret);
2297 2298 2299
		return ret;
	}

2300 2301
	ret = ctrl->ops->reg_read64(ctrl, NVME_REG_CAP, &cap);
	if (ret) {
2302
		dev_err(ctrl->device, "Reading CAP failed (%d)\n", ret);
2303 2304 2305 2306
		return ret;
	}
	page_shift = NVME_CAP_MPSMIN(cap) + 12;

2307
	if (ctrl->vs >= NVME_VS(1, 1, 0))
2308 2309
		ctrl->subsystem = NVME_CAP_NSSRC(cap);

2310 2311
	ret = nvme_identify_ctrl(ctrl, &id);
	if (ret) {
2312
		dev_err(ctrl->device, "Identify Controller failed (%d)\n", ret);
2313 2314 2315
		return -EIO;
	}

2316 2317 2318
	if (id->lpa & NVME_CTRL_LPA_CMD_EFFECTS_LOG) {
		ret = nvme_get_effects_log(ctrl);
		if (ret < 0)
2319
			goto out_free;
2320
	}
2321

2322
	if (!ctrl->identified) {
C
Christoph Hellwig 已提交
2323 2324 2325 2326 2327 2328
		int i;

		ret = nvme_init_subsystem(ctrl, id);
		if (ret)
			goto out_free;

2329 2330 2331 2332 2333 2334 2335 2336 2337 2338 2339 2340 2341 2342
		/*
		 * Check for quirks.  Quirk can depend on firmware version,
		 * so, in principle, the set of quirks present can change
		 * across a reset.  As a possible future enhancement, we
		 * could re-scan for quirks every time we reinitialize
		 * the device, but we'd have to make sure that the driver
		 * behaves intelligently if the quirks change.
		 */
		for (i = 0; i < ARRAY_SIZE(core_quirks); i++) {
			if (quirk_matches(id, &core_quirks[i]))
				ctrl->quirks |= core_quirks[i].quirks;
		}
	}

2343
	if (force_apst && (ctrl->quirks & NVME_QUIRK_NO_DEEPEST_PS)) {
2344
		dev_warn(ctrl->device, "forcibly allowing all power states due to nvme_core.force_apst -- use at your own risk\n");
2345 2346 2347
		ctrl->quirks &= ~NVME_QUIRK_NO_DEEPEST_PS;
	}

2348
	ctrl->oacs = le16_to_cpu(id->oacs);
2349
	ctrl->oncs = le16_to_cpup(&id->oncs);
2350
	atomic_set(&ctrl->abort_limit, id->acl + 1);
2351
	ctrl->vwc = id->vwc;
M
Ming Lin 已提交
2352
	ctrl->cntlid = le16_to_cpup(&id->cntlid);
2353
	if (id->mdts)
2354
		max_hw_sectors = 1 << (id->mdts + page_shift - 9);
2355
	else
2356 2357 2358
		max_hw_sectors = UINT_MAX;
	ctrl->max_hw_sectors =
		min_not_zero(ctrl->max_hw_sectors, max_hw_sectors);
2359

2360
	nvme_set_queue_limits(ctrl, ctrl->admin_q);
2361
	ctrl->sgls = le32_to_cpu(id->sgls);
S
Sagi Grimberg 已提交
2362
	ctrl->kas = le16_to_cpu(id->kas);
2363

2364 2365 2366 2367 2368 2369 2370 2371
	if (id->rtd3e) {
		/* us -> s */
		u32 transition_time = le32_to_cpu(id->rtd3e) / 1000000;

		ctrl->shutdown_timeout = clamp_t(unsigned int, transition_time,
						 shutdown_timeout, 60);

		if (ctrl->shutdown_timeout != shutdown_timeout)
2372
			dev_info(ctrl->device,
2373 2374 2375 2376 2377
				 "Shutdown timeout set to %u seconds\n",
				 ctrl->shutdown_timeout);
	} else
		ctrl->shutdown_timeout = shutdown_timeout;

2378
	ctrl->npss = id->npss;
2379 2380
	ctrl->apsta = id->apsta;
	prev_apst_enabled = ctrl->apst_enabled;
2381 2382
	if (ctrl->quirks & NVME_QUIRK_NO_APST) {
		if (force_apst && id->apsta) {
2383
			dev_warn(ctrl->device, "forcibly allowing APST due to nvme_core.force_apst -- use at your own risk\n");
2384
			ctrl->apst_enabled = true;
2385
		} else {
2386
			ctrl->apst_enabled = false;
2387 2388
		}
	} else {
2389
		ctrl->apst_enabled = id->apsta;
2390
	}
2391 2392
	memcpy(ctrl->psd, id->psd, sizeof(ctrl->psd));

2393
	if (ctrl->ops->flags & NVME_F_FABRICS) {
2394 2395 2396 2397 2398 2399 2400 2401 2402
		ctrl->icdoff = le16_to_cpu(id->icdoff);
		ctrl->ioccsz = le32_to_cpu(id->ioccsz);
		ctrl->iorcsz = le32_to_cpu(id->iorcsz);
		ctrl->maxcmd = le16_to_cpu(id->maxcmd);

		/*
		 * In fabrics we need to verify the cntlid matches the
		 * admin connect
		 */
2403
		if (ctrl->cntlid != le16_to_cpu(id->cntlid)) {
2404
			ret = -EINVAL;
2405 2406
			goto out_free;
		}
S
Sagi Grimberg 已提交
2407 2408

		if (!ctrl->opts->discovery_nqn && !ctrl->kas) {
2409
			dev_err(ctrl->device,
S
Sagi Grimberg 已提交
2410 2411
				"keep-alive support is mandatory for fabrics\n");
			ret = -EINVAL;
2412
			goto out_free;
S
Sagi Grimberg 已提交
2413
		}
2414 2415
	} else {
		ctrl->cntlid = le16_to_cpu(id->cntlid);
2416 2417
		ctrl->hmpre = le32_to_cpu(id->hmpre);
		ctrl->hmmin = le32_to_cpu(id->hmmin);
2418 2419
		ctrl->hmminds = le32_to_cpu(id->hmminds);
		ctrl->hmmaxd = le16_to_cpu(id->hmmaxd);
2420
	}
2421

2422
	kfree(id);
2423

2424
	if (ctrl->apst_enabled && !prev_apst_enabled)
2425
		dev_pm_qos_expose_latency_tolerance(ctrl->device);
2426
	else if (!ctrl->apst_enabled && prev_apst_enabled)
2427 2428
		dev_pm_qos_hide_latency_tolerance(ctrl->device);

2429 2430 2431
	ret = nvme_configure_apst(ctrl);
	if (ret < 0)
		return ret;
2432 2433 2434 2435
	
	ret = nvme_configure_timestamp(ctrl);
	if (ret < 0)
		return ret;
2436 2437 2438 2439

	ret = nvme_configure_directives(ctrl);
	if (ret < 0)
		return ret;
2440

2441
	ctrl->identified = true;
2442

2443 2444 2445 2446
	return 0;

out_free:
	kfree(id);
2447
	return ret;
2448
}
2449
EXPORT_SYMBOL_GPL(nvme_init_identify);
2450

2451
static int nvme_dev_open(struct inode *inode, struct file *file)
2452
{
2453 2454
	struct nvme_ctrl *ctrl =
		container_of(inode->i_cdev, struct nvme_ctrl, cdev);
2455

2456 2457 2458 2459 2460
	switch (ctrl->state) {
	case NVME_CTRL_LIVE:
	case NVME_CTRL_ADMIN_ONLY:
		break;
	default:
2461
		return -EWOULDBLOCK;
2462 2463
	}

2464
	file->private_data = ctrl;
2465 2466 2467
	return 0;
}

2468 2469 2470 2471 2472
static int nvme_dev_user_cmd(struct nvme_ctrl *ctrl, void __user *argp)
{
	struct nvme_ns *ns;
	int ret;

2473
	down_read(&ctrl->namespaces_rwsem);
2474 2475 2476 2477 2478 2479 2480
	if (list_empty(&ctrl->namespaces)) {
		ret = -ENOTTY;
		goto out_unlock;
	}

	ns = list_first_entry(&ctrl->namespaces, struct nvme_ns, list);
	if (ns != list_last_entry(&ctrl->namespaces, struct nvme_ns, list)) {
2481
		dev_warn(ctrl->device,
2482 2483 2484 2485 2486
			"NVME_IOCTL_IO_CMD not supported when multiple namespaces present!\n");
		ret = -EINVAL;
		goto out_unlock;
	}

2487
	dev_warn(ctrl->device,
2488 2489
		"using deprecated NVME_IOCTL_IO_CMD ioctl on the char device!\n");
	kref_get(&ns->kref);
2490
	up_read(&ctrl->namespaces_rwsem);
2491 2492 2493 2494 2495 2496

	ret = nvme_user_cmd(ctrl, ns, argp);
	nvme_put_ns(ns);
	return ret;

out_unlock:
2497
	up_read(&ctrl->namespaces_rwsem);
2498 2499 2500
	return ret;
}

2501 2502 2503 2504 2505 2506 2507 2508 2509 2510
static long nvme_dev_ioctl(struct file *file, unsigned int cmd,
		unsigned long arg)
{
	struct nvme_ctrl *ctrl = file->private_data;
	void __user *argp = (void __user *)arg;

	switch (cmd) {
	case NVME_IOCTL_ADMIN_CMD:
		return nvme_user_cmd(ctrl, NULL, argp);
	case NVME_IOCTL_IO_CMD:
2511
		return nvme_dev_user_cmd(ctrl, argp);
2512
	case NVME_IOCTL_RESET:
2513
		dev_warn(ctrl->device, "resetting controller\n");
2514
		return nvme_reset_ctrl_sync(ctrl);
2515 2516
	case NVME_IOCTL_SUBSYS_RESET:
		return nvme_reset_subsystem(ctrl);
K
Keith Busch 已提交
2517 2518 2519
	case NVME_IOCTL_RESCAN:
		nvme_queue_scan(ctrl);
		return 0;
2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537 2538
	default:
		return -ENOTTY;
	}
}

static const struct file_operations nvme_dev_fops = {
	.owner		= THIS_MODULE,
	.open		= nvme_dev_open,
	.unlocked_ioctl	= nvme_dev_ioctl,
	.compat_ioctl	= nvme_dev_ioctl,
};

static ssize_t nvme_sysfs_reset(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	int ret;

2539
	ret = nvme_reset_ctrl_sync(ctrl);
2540 2541 2542
	if (ret < 0)
		return ret;
	return count;
2543
}
2544
static DEVICE_ATTR(reset_controller, S_IWUSR, NULL, nvme_sysfs_reset);
2545

K
Keith Busch 已提交
2546 2547 2548 2549 2550 2551 2552 2553 2554 2555 2556
static ssize_t nvme_sysfs_rescan(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	nvme_queue_scan(ctrl);
	return count;
}
static DEVICE_ATTR(rescan_controller, S_IWUSR, NULL, nvme_sysfs_rescan);

2557 2558 2559 2560 2561 2562 2563 2564 2565 2566
static inline struct nvme_ns_head *dev_to_ns_head(struct device *dev)
{
	struct gendisk *disk = dev_to_disk(dev);

	if (disk->fops == &nvme_fops)
		return nvme_get_ns_from_dev(dev)->head;
	else
		return disk->private_data;
}

2567
static ssize_t wwid_show(struct device *dev, struct device_attribute *attr,
2568
		char *buf)
2569
{
2570 2571 2572
	struct nvme_ns_head *head = dev_to_ns_head(dev);
	struct nvme_ns_ids *ids = &head->ids;
	struct nvme_subsystem *subsys = head->subsys;
C
Christoph Hellwig 已提交
2573 2574
	int serial_len = sizeof(subsys->serial);
	int model_len = sizeof(subsys->model);
2575

2576 2577
	if (!uuid_is_null(&ids->uuid))
		return sprintf(buf, "uuid.%pU\n", &ids->uuid);
2578

2579 2580
	if (memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
		return sprintf(buf, "eui.%16phN\n", ids->nguid);
2581

2582 2583
	if (memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
		return sprintf(buf, "eui.%8phN\n", ids->eui64);
2584

C
Christoph Hellwig 已提交
2585 2586
	while (serial_len > 0 && (subsys->serial[serial_len - 1] == ' ' ||
				  subsys->serial[serial_len - 1] == '\0'))
2587
		serial_len--;
C
Christoph Hellwig 已提交
2588 2589
	while (model_len > 0 && (subsys->model[model_len - 1] == ' ' ||
				 subsys->model[model_len - 1] == '\0'))
2590 2591
		model_len--;

C
Christoph Hellwig 已提交
2592 2593
	return sprintf(buf, "nvme.%04x-%*phN-%*phN-%08x\n", subsys->vendor_id,
		serial_len, subsys->serial, model_len, subsys->model,
2594
		head->ns_id);
2595
}
J
Joe Perches 已提交
2596
static DEVICE_ATTR_RO(wwid);
2597

2598
static ssize_t nguid_show(struct device *dev, struct device_attribute *attr,
2599
		char *buf)
2600
{
2601
	return sprintf(buf, "%pU\n", dev_to_ns_head(dev)->ids.nguid);
2602
}
J
Joe Perches 已提交
2603
static DEVICE_ATTR_RO(nguid);
2604

2605
static ssize_t uuid_show(struct device *dev, struct device_attribute *attr,
2606
		char *buf)
2607
{
2608
	struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
2609 2610 2611 2612

	/* For backward compatibility expose the NGUID to userspace if
	 * we have no UUID set
	 */
2613
	if (uuid_is_null(&ids->uuid)) {
2614 2615
		printk_ratelimited(KERN_WARNING
				   "No UUID available providing old NGUID\n");
2616
		return sprintf(buf, "%pU\n", ids->nguid);
2617
	}
2618
	return sprintf(buf, "%pU\n", &ids->uuid);
2619
}
J
Joe Perches 已提交
2620
static DEVICE_ATTR_RO(uuid);
2621 2622

static ssize_t eui_show(struct device *dev, struct device_attribute *attr,
2623
		char *buf)
2624
{
2625
	return sprintf(buf, "%8ph\n", dev_to_ns_head(dev)->ids.eui64);
2626
}
J
Joe Perches 已提交
2627
static DEVICE_ATTR_RO(eui);
2628 2629

static ssize_t nsid_show(struct device *dev, struct device_attribute *attr,
2630
		char *buf)
2631
{
2632
	return sprintf(buf, "%d\n", dev_to_ns_head(dev)->ns_id);
2633
}
J
Joe Perches 已提交
2634
static DEVICE_ATTR_RO(nsid);
2635

2636
static struct attribute *nvme_ns_id_attrs[] = {
2637
	&dev_attr_wwid.attr,
2638
	&dev_attr_uuid.attr,
2639
	&dev_attr_nguid.attr,
2640 2641 2642 2643 2644
	&dev_attr_eui.attr,
	&dev_attr_nsid.attr,
	NULL,
};

2645
static umode_t nvme_ns_id_attrs_are_visible(struct kobject *kobj,
2646 2647 2648
		struct attribute *a, int n)
{
	struct device *dev = container_of(kobj, struct device, kobj);
2649
	struct nvme_ns_ids *ids = &dev_to_ns_head(dev)->ids;
2650 2651

	if (a == &dev_attr_uuid.attr) {
2652
		if (uuid_is_null(&ids->uuid) &&
2653
		    !memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
2654 2655 2656
			return 0;
	}
	if (a == &dev_attr_nguid.attr) {
2657
		if (!memchr_inv(ids->nguid, 0, sizeof(ids->nguid)))
2658 2659 2660
			return 0;
	}
	if (a == &dev_attr_eui.attr) {
2661
		if (!memchr_inv(ids->eui64, 0, sizeof(ids->eui64)))
2662 2663 2664 2665 2666
			return 0;
	}
	return a->mode;
}

2667 2668 2669
const struct attribute_group nvme_ns_id_attr_group = {
	.attrs		= nvme_ns_id_attrs,
	.is_visible	= nvme_ns_id_attrs_are_visible,
2670 2671
};

M
Ming Lin 已提交
2672
#define nvme_show_str_function(field)						\
2673 2674 2675 2676
static ssize_t  field##_show(struct device *dev,				\
			    struct device_attribute *attr, char *buf)		\
{										\
        struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
C
Christoph Hellwig 已提交
2677 2678
        return sprintf(buf, "%.*s\n",						\
		(int)sizeof(ctrl->subsys->field), ctrl->subsys->field);		\
2679 2680 2681
}										\
static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);

C
Christoph Hellwig 已提交
2682 2683 2684 2685
nvme_show_str_function(model);
nvme_show_str_function(serial);
nvme_show_str_function(firmware_rev);

M
Ming Lin 已提交
2686 2687 2688 2689 2690 2691 2692 2693 2694 2695
#define nvme_show_int_function(field)						\
static ssize_t  field##_show(struct device *dev,				\
			    struct device_attribute *attr, char *buf)		\
{										\
        struct nvme_ctrl *ctrl = dev_get_drvdata(dev);				\
        return sprintf(buf, "%d\n", ctrl->field);	\
}										\
static DEVICE_ATTR(field, S_IRUGO, field##_show, NULL);

nvme_show_int_function(cntlid);
2696

M
Ming Lin 已提交
2697 2698 2699 2700 2701 2702 2703
static ssize_t nvme_sysfs_delete(struct device *dev,
				struct device_attribute *attr, const char *buf,
				size_t count)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	if (device_remove_file_self(dev, attr))
2704
		nvme_delete_ctrl_sync(ctrl);
M
Ming Lin 已提交
2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715 2716 2717 2718
	return count;
}
static DEVICE_ATTR(delete_controller, S_IWUSR, NULL, nvme_sysfs_delete);

static ssize_t nvme_sysfs_show_transport(struct device *dev,
					 struct device_attribute *attr,
					 char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->ops->name);
}
static DEVICE_ATTR(transport, S_IRUGO, nvme_sysfs_show_transport, NULL);

2719 2720 2721 2722 2723 2724 2725 2726
static ssize_t nvme_sysfs_show_state(struct device *dev,
				     struct device_attribute *attr,
				     char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);
	static const char *const state_name[] = {
		[NVME_CTRL_NEW]		= "new",
		[NVME_CTRL_LIVE]	= "live",
2727
		[NVME_CTRL_ADMIN_ONLY]	= "only-admin",
2728
		[NVME_CTRL_RESETTING]	= "resetting",
2729
		[NVME_CTRL_CONNECTING]	= "connecting",
2730 2731 2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742
		[NVME_CTRL_DELETING]	= "deleting",
		[NVME_CTRL_DEAD]	= "dead",
	};

	if ((unsigned)ctrl->state < ARRAY_SIZE(state_name) &&
	    state_name[ctrl->state])
		return sprintf(buf, "%s\n", state_name[ctrl->state]);

	return sprintf(buf, "unknown state\n");
}

static DEVICE_ATTR(state, S_IRUGO, nvme_sysfs_show_state, NULL);

M
Ming Lin 已提交
2743 2744 2745 2746 2747 2748
static ssize_t nvme_sysfs_show_subsysnqn(struct device *dev,
					 struct device_attribute *attr,
					 char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

C
Christoph Hellwig 已提交
2749
	return snprintf(buf, PAGE_SIZE, "%s\n", ctrl->subsys->subnqn);
M
Ming Lin 已提交
2750 2751 2752 2753 2754 2755 2756 2757 2758 2759 2760 2761 2762
}
static DEVICE_ATTR(subsysnqn, S_IRUGO, nvme_sysfs_show_subsysnqn, NULL);

static ssize_t nvme_sysfs_show_address(struct device *dev,
					 struct device_attribute *attr,
					 char *buf)
{
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

	return ctrl->ops->get_address(ctrl, buf, PAGE_SIZE);
}
static DEVICE_ATTR(address, S_IRUGO, nvme_sysfs_show_address, NULL);

2763 2764
static struct attribute *nvme_dev_attrs[] = {
	&dev_attr_reset_controller.attr,
K
Keith Busch 已提交
2765
	&dev_attr_rescan_controller.attr,
2766 2767 2768
	&dev_attr_model.attr,
	&dev_attr_serial.attr,
	&dev_attr_firmware_rev.attr,
M
Ming Lin 已提交
2769
	&dev_attr_cntlid.attr,
M
Ming Lin 已提交
2770 2771 2772 2773
	&dev_attr_delete_controller.attr,
	&dev_attr_transport.attr,
	&dev_attr_subsysnqn.attr,
	&dev_attr_address.attr,
2774
	&dev_attr_state.attr,
2775 2776 2777
	NULL
};

M
Ming Lin 已提交
2778 2779 2780 2781 2782 2783
static umode_t nvme_dev_attrs_are_visible(struct kobject *kobj,
		struct attribute *a, int n)
{
	struct device *dev = container_of(kobj, struct device, kobj);
	struct nvme_ctrl *ctrl = dev_get_drvdata(dev);

2784 2785 2786 2787
	if (a == &dev_attr_delete_controller.attr && !ctrl->ops->delete_ctrl)
		return 0;
	if (a == &dev_attr_address.attr && !ctrl->ops->get_address)
		return 0;
M
Ming Lin 已提交
2788 2789 2790 2791

	return a->mode;
}

2792
static struct attribute_group nvme_dev_attrs_group = {
M
Ming Lin 已提交
2793 2794
	.attrs		= nvme_dev_attrs,
	.is_visible	= nvme_dev_attrs_are_visible,
2795 2796 2797 2798 2799 2800 2801
};

static const struct attribute_group *nvme_dev_attr_groups[] = {
	&nvme_dev_attrs_group,
	NULL,
};

C
Christoph Hellwig 已提交
2802 2803 2804 2805 2806 2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820 2821 2822 2823 2824 2825
static struct nvme_ns_head *__nvme_find_ns_head(struct nvme_subsystem *subsys,
		unsigned nsid)
{
	struct nvme_ns_head *h;

	lockdep_assert_held(&subsys->lock);

	list_for_each_entry(h, &subsys->nsheads, entry) {
		if (h->ns_id == nsid && kref_get_unless_zero(&h->ref))
			return h;
	}

	return NULL;
}

static int __nvme_check_ids(struct nvme_subsystem *subsys,
		struct nvme_ns_head *new)
{
	struct nvme_ns_head *h;

	lockdep_assert_held(&subsys->lock);

	list_for_each_entry(h, &subsys->nsheads, entry) {
		if (nvme_ns_ids_valid(&new->ids) &&
2826
		    !list_empty(&h->list) &&
C
Christoph Hellwig 已提交
2827 2828 2829 2830 2831 2832 2833 2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847
		    nvme_ns_ids_equal(&new->ids, &h->ids))
			return -EINVAL;
	}

	return 0;
}

static struct nvme_ns_head *nvme_alloc_ns_head(struct nvme_ctrl *ctrl,
		unsigned nsid, struct nvme_id_ns *id)
{
	struct nvme_ns_head *head;
	int ret = -ENOMEM;

	head = kzalloc(sizeof(*head), GFP_KERNEL);
	if (!head)
		goto out;
	ret = ida_simple_get(&ctrl->subsys->ns_ida, 1, 0, GFP_KERNEL);
	if (ret < 0)
		goto out_free_head;
	head->instance = ret;
	INIT_LIST_HEAD(&head->list);
2848 2849 2850
	ret = init_srcu_struct(&head->srcu);
	if (ret)
		goto out_ida_remove;
C
Christoph Hellwig 已提交
2851 2852 2853 2854 2855 2856 2857 2858 2859 2860 2861 2862 2863
	head->subsys = ctrl->subsys;
	head->ns_id = nsid;
	kref_init(&head->ref);

	nvme_report_ns_ids(ctrl, nsid, id, &head->ids);

	ret = __nvme_check_ids(ctrl->subsys, head);
	if (ret) {
		dev_err(ctrl->device,
			"duplicate IDs for nsid %d\n", nsid);
		goto out_cleanup_srcu;
	}

2864 2865 2866 2867
	ret = nvme_mpath_alloc_disk(ctrl, head);
	if (ret)
		goto out_cleanup_srcu;

C
Christoph Hellwig 已提交
2868 2869 2870 2871
	list_add_tail(&head->entry, &ctrl->subsys->nsheads);
	return head;
out_cleanup_srcu:
	cleanup_srcu_struct(&head->srcu);
2872
out_ida_remove:
C
Christoph Hellwig 已提交
2873 2874 2875 2876 2877 2878 2879 2880
	ida_simple_remove(&ctrl->subsys->ns_ida, head->instance);
out_free_head:
	kfree(head);
out:
	return ERR_PTR(ret);
}

static int nvme_init_ns_head(struct nvme_ns *ns, unsigned nsid,
2881
		struct nvme_id_ns *id)
C
Christoph Hellwig 已提交
2882 2883 2884 2885 2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911 2912 2913 2914 2915 2916 2917
{
	struct nvme_ctrl *ctrl = ns->ctrl;
	bool is_shared = id->nmic & (1 << 0);
	struct nvme_ns_head *head = NULL;
	int ret = 0;

	mutex_lock(&ctrl->subsys->lock);
	if (is_shared)
		head = __nvme_find_ns_head(ctrl->subsys, nsid);
	if (!head) {
		head = nvme_alloc_ns_head(ctrl, nsid, id);
		if (IS_ERR(head)) {
			ret = PTR_ERR(head);
			goto out_unlock;
		}
	} else {
		struct nvme_ns_ids ids;

		nvme_report_ns_ids(ctrl, nsid, id, &ids);
		if (!nvme_ns_ids_equal(&head->ids, &ids)) {
			dev_err(ctrl->device,
				"IDs don't match for shared namespace %d\n",
					nsid);
			ret = -EINVAL;
			goto out_unlock;
		}
	}

	list_add_tail(&ns->siblings, &head->list);
	ns->head = head;

out_unlock:
	mutex_unlock(&ctrl->subsys->lock);
	return ret;
}

2918 2919 2920 2921 2922
static int ns_cmp(void *priv, struct list_head *a, struct list_head *b)
{
	struct nvme_ns *nsa = container_of(a, struct nvme_ns, list);
	struct nvme_ns *nsb = container_of(b, struct nvme_ns, list);

C
Christoph Hellwig 已提交
2923
	return nsa->head->ns_id - nsb->head->ns_id;
2924 2925
}

2926
static struct nvme_ns *nvme_find_get_ns(struct nvme_ctrl *ctrl, unsigned nsid)
2927
{
2928
	struct nvme_ns *ns, *ret = NULL;
2929

2930
	down_read(&ctrl->namespaces_rwsem);
2931
	list_for_each_entry(ns, &ctrl->namespaces, list) {
C
Christoph Hellwig 已提交
2932
		if (ns->head->ns_id == nsid) {
2933 2934
			if (!kref_get_unless_zero(&ns->kref))
				continue;
2935 2936 2937
			ret = ns;
			break;
		}
C
Christoph Hellwig 已提交
2938
		if (ns->head->ns_id > nsid)
2939 2940
			break;
	}
2941
	up_read(&ctrl->namespaces_rwsem);
2942
	return ret;
2943 2944
}

2945 2946 2947 2948 2949 2950 2951 2952
static int nvme_setup_streams_ns(struct nvme_ctrl *ctrl, struct nvme_ns *ns)
{
	struct streams_directive_params s;
	int ret;

	if (!ctrl->nr_streams)
		return 0;

C
Christoph Hellwig 已提交
2953
	ret = nvme_get_stream_params(ctrl, &s, ns->head->ns_id);
2954 2955 2956 2957 2958 2959 2960 2961 2962 2963 2964 2965 2966 2967 2968 2969 2970
	if (ret)
		return ret;

	ns->sws = le32_to_cpu(s.sws);
	ns->sgs = le16_to_cpu(s.sgs);

	if (ns->sws) {
		unsigned int bs = 1 << ns->lba_shift;

		blk_queue_io_min(ns->queue, bs * ns->sws);
		if (ns->sgs)
			blk_queue_io_opt(ns->queue, bs * ns->sws * ns->sgs);
	}

	return 0;
}

2971 2972 2973 2974
static void nvme_alloc_ns(struct nvme_ctrl *ctrl, unsigned nsid)
{
	struct nvme_ns *ns;
	struct gendisk *disk;
2975 2976
	struct nvme_id_ns *id;
	char disk_name[DISK_NAME_LEN];
2977
	int node = dev_to_node(ctrl->dev), flags = GENHD_FL_EXT_DEVT;
2978 2979 2980 2981 2982 2983 2984

	ns = kzalloc_node(sizeof(*ns), GFP_KERNEL, node);
	if (!ns)
		return;

	ns->queue = blk_mq_init_queue(ctrl->tagset);
	if (IS_ERR(ns->queue))
C
Christoph Hellwig 已提交
2985
		goto out_free_ns;
2986
	blk_queue_flag_set(QUEUE_FLAG_NONROT, ns->queue);
2987 2988 2989 2990 2991 2992 2993
	ns->queue->queuedata = ns;
	ns->ctrl = ctrl;

	kref_init(&ns->kref);
	ns->lba_shift = 9; /* set to a default value for 512 until disk is validated */

	blk_queue_logical_block_size(ns->queue, 1 << ns->lba_shift);
2994
	nvme_set_queue_limits(ctrl, ns->queue);
2995

2996 2997
	id = nvme_identify_ns(ctrl, nsid);
	if (!id)
2998 2999
		goto out_free_queue;

3000 3001 3002
	if (id->ncap == 0)
		goto out_free_id;

3003
	if (nvme_init_ns_head(ns, nsid, id))
C
Christoph Hellwig 已提交
3004
		goto out_free_id;
3005
	nvme_setup_streams_ns(ctrl, ns);
3006
	nvme_set_disk_name(disk_name, ns, ctrl, &flags);
3007

C
Christoph Hellwig 已提交
3008 3009 3010
	if ((ctrl->quirks & NVME_QUIRK_LIGHTNVM) && id->vs[0] == 0x1) {
		if (nvme_nvm_register(ns, disk_name, node)) {
			dev_warn(ctrl->device, "LightNVM init failure\n");
C
Christoph Hellwig 已提交
3011
			goto out_unlink_ns;
C
Christoph Hellwig 已提交
3012
		}
3013
	}
3014

3015 3016
	disk = alloc_disk_node(0, node);
	if (!disk)
C
Christoph Hellwig 已提交
3017
		goto out_unlink_ns;
3018

3019 3020 3021
	disk->fops = &nvme_fops;
	disk->private_data = ns;
	disk->queue = ns->queue;
3022
	disk->flags = flags;
3023 3024 3025 3026
	memcpy(disk->disk_name, disk_name, DISK_NAME_LEN);
	ns->disk = disk;

	__nvme_revalidate_disk(disk, id);
3027

3028
	down_write(&ctrl->namespaces_rwsem);
3029
	list_add_tail(&ns->list, &ctrl->namespaces);
3030
	up_write(&ctrl->namespaces_rwsem);
3031

3032
	nvme_get_ctrl(ctrl);
3033 3034 3035

	kfree(id);

3036
	device_add_disk(ctrl->device, ns->disk);
3037
	if (sysfs_create_group(&disk_to_dev(ns->disk)->kobj,
3038
					&nvme_ns_id_attr_group))
3039 3040
		pr_warn("%s: failed to create sysfs group for identification\n",
			ns->disk->disk_name);
3041 3042 3043
	if (ns->ndev && nvme_nvm_register_sysfs(ns))
		pr_warn("%s: failed to register lightnvm sysfs group for identification\n",
			ns->disk->disk_name);
3044

3045
	nvme_mpath_add_disk(ns->head);
T
Thomas Tai 已提交
3046
	nvme_fault_inject_init(ns);
3047
	return;
C
Christoph Hellwig 已提交
3048 3049 3050 3051
 out_unlink_ns:
	mutex_lock(&ctrl->subsys->lock);
	list_del_rcu(&ns->siblings);
	mutex_unlock(&ctrl->subsys->lock);
3052 3053
 out_free_id:
	kfree(id);
3054 3055 3056 3057 3058 3059 3060 3061
 out_free_queue:
	blk_cleanup_queue(ns->queue);
 out_free_ns:
	kfree(ns);
}

static void nvme_ns_remove(struct nvme_ns *ns)
{
3062 3063
	if (test_and_set_bit(NVME_NS_REMOVING, &ns->flags))
		return;
3064

T
Thomas Tai 已提交
3065
	nvme_fault_inject_fini(ns);
3066
	if (ns->disk && ns->disk->flags & GENHD_FL_UP) {
3067
		sysfs_remove_group(&disk_to_dev(ns->disk)->kobj,
3068
					&nvme_ns_id_attr_group);
3069 3070
		if (ns->ndev)
			nvme_nvm_unregister_sysfs(ns);
3071 3072
		del_gendisk(ns->disk);
		blk_cleanup_queue(ns->queue);
3073 3074
		if (blk_get_integrity(ns->disk))
			blk_integrity_unregister(ns->disk);
3075
	}
3076

C
Christoph Hellwig 已提交
3077
	mutex_lock(&ns->ctrl->subsys->lock);
3078
	nvme_mpath_clear_current_path(ns);
3079
	list_del_rcu(&ns->siblings);
C
Christoph Hellwig 已提交
3080 3081
	mutex_unlock(&ns->ctrl->subsys->lock);

3082
	down_write(&ns->ctrl->namespaces_rwsem);
3083
	list_del_init(&ns->list);
3084
	up_write(&ns->ctrl->namespaces_rwsem);
3085

3086
	synchronize_srcu(&ns->head->srcu);
3087
	nvme_mpath_check_last_path(ns);
3088 3089 3090
	nvme_put_ns(ns);
}

3091 3092 3093 3094
static void nvme_validate_ns(struct nvme_ctrl *ctrl, unsigned nsid)
{
	struct nvme_ns *ns;

3095
	ns = nvme_find_get_ns(ctrl, nsid);
3096
	if (ns) {
3097
		if (ns->disk && revalidate_disk(ns->disk))
3098
			nvme_ns_remove(ns);
3099
		nvme_put_ns(ns);
3100 3101 3102 3103
	} else
		nvme_alloc_ns(ctrl, nsid);
}

3104 3105 3106 3107
static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl,
					unsigned nsid)
{
	struct nvme_ns *ns, *next;
3108
	LIST_HEAD(rm_list);
3109

3110
	down_write(&ctrl->namespaces_rwsem);
3111
	list_for_each_entry_safe(ns, next, &ctrl->namespaces, list) {
C
Christoph Hellwig 已提交
3112
		if (ns->head->ns_id > nsid)
3113
			list_move_tail(&ns->list, &rm_list);
3114
	}
3115
	up_write(&ctrl->namespaces_rwsem);
3116 3117 3118 3119

	list_for_each_entry_safe(ns, next, &rm_list, list)
		nvme_ns_remove(ns);

3120 3121
}

3122 3123 3124 3125 3126 3127 3128
static int nvme_scan_ns_list(struct nvme_ctrl *ctrl, unsigned nn)
{
	struct nvme_ns *ns;
	__le32 *ns_list;
	unsigned i, j, nsid, prev = 0, num_lists = DIV_ROUND_UP(nn, 1024);
	int ret = 0;

3129
	ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL);
3130 3131 3132 3133 3134 3135
	if (!ns_list)
		return -ENOMEM;

	for (i = 0; i < num_lists; i++) {
		ret = nvme_identify_ns_list(ctrl, prev, ns_list);
		if (ret)
3136
			goto free;
3137 3138 3139 3140 3141 3142 3143 3144 3145

		for (j = 0; j < min(nn, 1024U); j++) {
			nsid = le32_to_cpu(ns_list[j]);
			if (!nsid)
				goto out;

			nvme_validate_ns(ctrl, nsid);

			while (++prev < nsid) {
3146 3147
				ns = nvme_find_get_ns(ctrl, prev);
				if (ns) {
3148
					nvme_ns_remove(ns);
3149 3150
					nvme_put_ns(ns);
				}
3151 3152 3153 3154 3155
			}
		}
		nn -= j;
	}
 out:
3156 3157
	nvme_remove_invalid_namespaces(ctrl, prev);
 free:
3158 3159 3160 3161
	kfree(ns_list);
	return ret;
}

3162
static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl, unsigned nn)
3163 3164 3165
{
	unsigned i;

3166 3167 3168
	for (i = 1; i <= nn; i++)
		nvme_validate_ns(ctrl, i);

3169
	nvme_remove_invalid_namespaces(ctrl, nn);
3170 3171
}

3172
static void nvme_scan_work(struct work_struct *work)
3173
{
3174 3175
	struct nvme_ctrl *ctrl =
		container_of(work, struct nvme_ctrl, scan_work);
3176
	struct nvme_id_ctrl *id;
3177
	unsigned nn;
3178

3179 3180 3181
	if (ctrl->state != NVME_CTRL_LIVE)
		return;

3182 3183
	WARN_ON_ONCE(!ctrl->tagset);

3184 3185
	if (nvme_identify_ctrl(ctrl, &id))
		return;
3186 3187

	nn = le32_to_cpu(id->nn);
3188
	if (ctrl->vs >= NVME_VS(1, 1, 0) &&
3189 3190 3191 3192
	    !(ctrl->quirks & NVME_QUIRK_IDENTIFY_CNS)) {
		if (!nvme_scan_ns_list(ctrl, nn))
			goto done;
	}
3193
	nvme_scan_ns_sequential(ctrl, nn);
3194
 done:
3195
	down_write(&ctrl->namespaces_rwsem);
3196
	list_sort(NULL, &ctrl->namespaces, ns_cmp);
3197
	up_write(&ctrl->namespaces_rwsem);
3198 3199
	kfree(id);
}
3200 3201 3202 3203

void nvme_queue_scan(struct nvme_ctrl *ctrl)
{
	/*
3204
	 * Only new queue scan work when admin and IO queues are both alive
3205 3206
	 */
	if (ctrl->state == NVME_CTRL_LIVE)
3207
		queue_work(nvme_wq, &ctrl->scan_work);
3208 3209
}
EXPORT_SYMBOL_GPL(nvme_queue_scan);
3210

3211 3212 3213 3214 3215
/*
 * This function iterates the namespace list unlocked to allow recovery from
 * controller failure. It is up to the caller to ensure the namespace list is
 * not modified by scan work while this function is executing.
 */
3216 3217 3218
void nvme_remove_namespaces(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns, *next;
3219
	LIST_HEAD(ns_list);
3220

3221 3222 3223 3224 3225 3226 3227 3228 3229
	/*
	 * The dead states indicates the controller was not gracefully
	 * disconnected. In that case, we won't be able to flush any data while
	 * removing the namespaces' disks; fail all the queues now to avoid
	 * potentially having to clean up the failed sync later.
	 */
	if (ctrl->state == NVME_CTRL_DEAD)
		nvme_kill_queues(ctrl);

3230
	down_write(&ctrl->namespaces_rwsem);
3231
	list_splice_init(&ctrl->namespaces, &ns_list);
3232
	up_write(&ctrl->namespaces_rwsem);
3233 3234

	list_for_each_entry_safe(ns, next, &ns_list, list)
3235 3236
		nvme_ns_remove(ns);
}
3237
EXPORT_SYMBOL_GPL(nvme_remove_namespaces);
3238

3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254
static void nvme_aen_uevent(struct nvme_ctrl *ctrl)
{
	char *envp[2] = { NULL, NULL };
	u32 aen_result = ctrl->aen_result;

	ctrl->aen_result = 0;
	if (!aen_result)
		return;

	envp[0] = kasprintf(GFP_KERNEL, "NVME_AEN=%#08x", aen_result);
	if (!envp[0])
		return;
	kobject_uevent_env(&ctrl->device->kobj, KOBJ_CHANGE, envp);
	kfree(envp[0]);
}

3255 3256 3257 3258 3259
static void nvme_async_event_work(struct work_struct *work)
{
	struct nvme_ctrl *ctrl =
		container_of(work, struct nvme_ctrl, async_event_work);

3260
	nvme_aen_uevent(ctrl);
3261
	ctrl->ops->submit_async_event(ctrl);
3262 3263
}

3264 3265 3266 3267 3268 3269 3270 3271 3272 3273 3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285
static bool nvme_ctrl_pp_status(struct nvme_ctrl *ctrl)
{

	u32 csts;

	if (ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts))
		return false;

	if (csts == ~0)
		return false;

	return ((ctrl->ctrl_config & NVME_CC_ENABLE) && (csts & NVME_CSTS_PP));
}

static void nvme_get_fw_slot_info(struct nvme_ctrl *ctrl)
{
	struct nvme_fw_slot_info_log *log;

	log = kmalloc(sizeof(*log), GFP_KERNEL);
	if (!log)
		return;

K
Keith Busch 已提交
3286
	if (nvme_get_log(ctrl, NVME_LOG_FW_SLOT, log, sizeof(*log)))
3287 3288 3289 3290 3291 3292 3293 3294 3295 3296 3297 3298 3299 3300 3301 3302 3303 3304 3305 3306 3307 3308 3309 3310 3311 3312 3313 3314 3315 3316 3317 3318 3319
		dev_warn(ctrl->device,
				"Get FW SLOT INFO log error\n");
	kfree(log);
}

static void nvme_fw_act_work(struct work_struct *work)
{
	struct nvme_ctrl *ctrl = container_of(work,
				struct nvme_ctrl, fw_act_work);
	unsigned long fw_act_timeout;

	if (ctrl->mtfa)
		fw_act_timeout = jiffies +
				msecs_to_jiffies(ctrl->mtfa * 100);
	else
		fw_act_timeout = jiffies +
				msecs_to_jiffies(admin_timeout * 1000);

	nvme_stop_queues(ctrl);
	while (nvme_ctrl_pp_status(ctrl)) {
		if (time_after(jiffies, fw_act_timeout)) {
			dev_warn(ctrl->device,
				"Fw activation timeout, reset controller\n");
			nvme_reset_ctrl(ctrl);
			break;
		}
		msleep(100);
	}

	if (ctrl->state != NVME_CTRL_LIVE)
		return;

	nvme_start_queues(ctrl);
3320
	/* read FW slot information to clear the AER */
3321 3322 3323
	nvme_get_fw_slot_info(ctrl);
}

3324 3325 3326 3327 3328 3329 3330 3331 3332 3333 3334 3335 3336 3337 3338
static void nvme_handle_aen_notice(struct nvme_ctrl *ctrl, u32 result)
{
	switch ((result & 0xff00) >> 8) {
	case NVME_AER_NOTICE_NS_CHANGED:
		dev_info(ctrl->device, "rescanning\n");
		nvme_queue_scan(ctrl);
		break;
	case NVME_AER_NOTICE_FW_ACT_STARTING:
		queue_work(nvme_wq, &ctrl->fw_act_work);
		break;
	default:
		dev_warn(ctrl->device, "async event result %08x\n", result);
	}
}

3339
void nvme_complete_async_event(struct nvme_ctrl *ctrl, __le16 status,
3340
		volatile union nvme_result *res)
3341
{
3342
	u32 result = le32_to_cpu(res->u32);
3343

3344
	if (le16_to_cpu(status) >> 1 != NVME_SC_SUCCESS)
3345 3346
		return;

3347
	switch (result & 0x7) {
3348 3349 3350
	case NVME_AER_NOTICE:
		nvme_handle_aen_notice(ctrl, result);
		break;
3351 3352 3353 3354 3355
	case NVME_AER_ERROR:
	case NVME_AER_SMART:
	case NVME_AER_CSS:
	case NVME_AER_VS:
		ctrl->aen_result = result;
3356 3357 3358
		break;
	default:
		break;
3359
	}
3360
	queue_work(nvme_wq, &ctrl->async_event_work);
3361 3362
}
EXPORT_SYMBOL_GPL(nvme_complete_async_event);
3363

3364
void nvme_stop_ctrl(struct nvme_ctrl *ctrl)
3365
{
3366
	nvme_stop_keep_alive(ctrl);
3367
	flush_work(&ctrl->async_event_work);
3368
	flush_work(&ctrl->scan_work);
3369
	cancel_work_sync(&ctrl->fw_act_work);
3370 3371
	if (ctrl->ops->stop_ctrl)
		ctrl->ops->stop_ctrl(ctrl);
3372 3373 3374 3375 3376 3377 3378 3379 3380 3381
}
EXPORT_SYMBOL_GPL(nvme_stop_ctrl);

void nvme_start_ctrl(struct nvme_ctrl *ctrl)
{
	if (ctrl->kato)
		nvme_start_keep_alive(ctrl);

	if (ctrl->queue_count > 1) {
		nvme_queue_scan(ctrl);
3382
		queue_work(nvme_wq, &ctrl->async_event_work);
3383 3384 3385 3386
		nvme_start_queues(ctrl);
	}
}
EXPORT_SYMBOL_GPL(nvme_start_ctrl);
3387

3388 3389
void nvme_uninit_ctrl(struct nvme_ctrl *ctrl)
{
3390
	cdev_device_del(&ctrl->cdev, ctrl->device);
3391
}
3392
EXPORT_SYMBOL_GPL(nvme_uninit_ctrl);
3393

3394
static void nvme_free_ctrl(struct device *dev)
3395
{
3396 3397
	struct nvme_ctrl *ctrl =
		container_of(dev, struct nvme_ctrl, ctrl_device);
C
Christoph Hellwig 已提交
3398
	struct nvme_subsystem *subsys = ctrl->subsys;
3399

3400
	ida_simple_remove(&nvme_instance_ida, ctrl->instance);
3401
	kfree(ctrl->effects);
3402

C
Christoph Hellwig 已提交
3403 3404 3405 3406 3407 3408
	if (subsys) {
		mutex_lock(&subsys->lock);
		list_del(&ctrl->subsys_entry);
		mutex_unlock(&subsys->lock);
		sysfs_remove_link(&subsys->dev.kobj, dev_name(ctrl->device));
	}
3409 3410 3411

	ctrl->ops->free_ctrl(ctrl);

C
Christoph Hellwig 已提交
3412 3413
	if (subsys)
		nvme_put_subsystem(subsys);
3414 3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425
}

/*
 * Initialize a NVMe controller structures.  This needs to be called during
 * earliest initialization so that we have the initialized structured around
 * during probing.
 */
int nvme_init_ctrl(struct nvme_ctrl *ctrl, struct device *dev,
		const struct nvme_ctrl_ops *ops, unsigned long quirks)
{
	int ret;

3426 3427
	ctrl->state = NVME_CTRL_NEW;
	spin_lock_init(&ctrl->lock);
3428
	INIT_LIST_HEAD(&ctrl->namespaces);
3429
	init_rwsem(&ctrl->namespaces_rwsem);
3430 3431 3432
	ctrl->dev = dev;
	ctrl->ops = ops;
	ctrl->quirks = quirks;
3433
	INIT_WORK(&ctrl->scan_work, nvme_scan_work);
3434
	INIT_WORK(&ctrl->async_event_work, nvme_async_event_work);
3435
	INIT_WORK(&ctrl->fw_act_work, nvme_fw_act_work);
3436
	INIT_WORK(&ctrl->delete_work, nvme_delete_ctrl_work);
3437

3438 3439
	ret = ida_simple_get(&nvme_instance_ida, 0, 0, GFP_KERNEL);
	if (ret < 0)
3440
		goto out;
3441
	ctrl->instance = ret;
3442

3443 3444
	device_initialize(&ctrl->ctrl_device);
	ctrl->device = &ctrl->ctrl_device;
3445
	ctrl->device->devt = MKDEV(MAJOR(nvme_chr_devt), ctrl->instance);
3446 3447 3448 3449 3450 3451 3452
	ctrl->device->class = nvme_class;
	ctrl->device->parent = ctrl->dev;
	ctrl->device->groups = nvme_dev_attr_groups;
	ctrl->device->release = nvme_free_ctrl;
	dev_set_drvdata(ctrl->device, ctrl);
	ret = dev_set_name(ctrl->device, "nvme%d", ctrl->instance);
	if (ret)
3453 3454
		goto out_release_instance;

3455 3456 3457
	cdev_init(&ctrl->cdev, &nvme_dev_fops);
	ctrl->cdev.owner = ops->module;
	ret = cdev_device_add(&ctrl->cdev, ctrl->device);
3458 3459
	if (ret)
		goto out_free_name;
3460

3461 3462 3463 3464 3465 3466 3467 3468
	/*
	 * Initialize latency tolerance controls.  The sysfs files won't
	 * be visible to userspace unless the device actually supports APST.
	 */
	ctrl->device->power.set_latency_tolerance = nvme_set_latency_tolerance;
	dev_pm_qos_update_user_latency_tolerance(ctrl->device,
		min(default_ps_max_latency_us, (unsigned long)S32_MAX));

3469
	return 0;
3470 3471
out_free_name:
	kfree_const(dev->kobj.name);
3472
out_release_instance:
3473
	ida_simple_remove(&nvme_instance_ida, ctrl->instance);
3474 3475 3476
out:
	return ret;
}
3477
EXPORT_SYMBOL_GPL(nvme_init_ctrl);
3478

3479 3480 3481 3482 3483 3484 3485 3486 3487 3488 3489
/**
 * nvme_kill_queues(): Ends all namespace queues
 * @ctrl: the dead controller that needs to end
 *
 * Call this function when the driver determines it is unable to get the
 * controller in a state capable of servicing IO.
 */
void nvme_kill_queues(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns;

3490
	down_read(&ctrl->namespaces_rwsem);
M
Ming Lei 已提交
3491

3492
	/* Forcibly unquiesce queues to avoid blocking dispatch */
3493 3494
	if (ctrl->admin_q)
		blk_mq_unquiesce_queue(ctrl->admin_q);
3495

3496
	list_for_each_entry(ns, &ctrl->namespaces, list) {
3497 3498 3499 3500
		/*
		 * Revalidating a dead namespace sets capacity to 0. This will
		 * end buffered writers dirtying pages that can't be synced.
		 */
3501 3502 3503
		if (!ns->disk || test_and_set_bit(NVME_NS_DEAD, &ns->flags))
			continue;
		revalidate_disk(ns->disk);
3504
		blk_set_queue_dying(ns->queue);
3505

3506 3507
		/* Forcibly unquiesce queues to avoid blocking dispatch */
		blk_mq_unquiesce_queue(ns->queue);
3508
	}
3509
	up_read(&ctrl->namespaces_rwsem);
3510
}
3511
EXPORT_SYMBOL_GPL(nvme_kill_queues);
3512

K
Keith Busch 已提交
3513 3514 3515 3516
void nvme_unfreeze(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns;

3517
	down_read(&ctrl->namespaces_rwsem);
K
Keith Busch 已提交
3518 3519
	list_for_each_entry(ns, &ctrl->namespaces, list)
		blk_mq_unfreeze_queue(ns->queue);
3520
	up_read(&ctrl->namespaces_rwsem);
K
Keith Busch 已提交
3521 3522 3523 3524 3525 3526 3527
}
EXPORT_SYMBOL_GPL(nvme_unfreeze);

void nvme_wait_freeze_timeout(struct nvme_ctrl *ctrl, long timeout)
{
	struct nvme_ns *ns;

3528
	down_read(&ctrl->namespaces_rwsem);
K
Keith Busch 已提交
3529 3530 3531 3532 3533
	list_for_each_entry(ns, &ctrl->namespaces, list) {
		timeout = blk_mq_freeze_queue_wait_timeout(ns->queue, timeout);
		if (timeout <= 0)
			break;
	}
3534
	up_read(&ctrl->namespaces_rwsem);
K
Keith Busch 已提交
3535 3536 3537 3538 3539 3540 3541
}
EXPORT_SYMBOL_GPL(nvme_wait_freeze_timeout);

void nvme_wait_freeze(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns;

3542
	down_read(&ctrl->namespaces_rwsem);
K
Keith Busch 已提交
3543 3544
	list_for_each_entry(ns, &ctrl->namespaces, list)
		blk_mq_freeze_queue_wait(ns->queue);
3545
	up_read(&ctrl->namespaces_rwsem);
K
Keith Busch 已提交
3546 3547 3548 3549 3550 3551 3552
}
EXPORT_SYMBOL_GPL(nvme_wait_freeze);

void nvme_start_freeze(struct nvme_ctrl *ctrl)
{
	struct nvme_ns *ns;

3553
	down_read(&ctrl->namespaces_rwsem);
K
Keith Busch 已提交
3554
	list_for_each_entry(ns, &ctrl->namespaces, list)
3555
		blk_freeze_queue_start(ns->queue);
3556
	up_read(&ctrl->namespaces_rwsem);
K
Keith Busch 已提交
3557 3558 3559
}
EXPORT_SYMBOL_GPL(nvme_start_freeze);

3560
void nvme_stop_queues(struct nvme_ctrl *ctrl)
3561 3562 3563
{
	struct nvme_ns *ns;

3564
	down_read(&ctrl->namespaces_rwsem);
3565
	list_for_each_entry(ns, &ctrl->namespaces, list)
3566
		blk_mq_quiesce_queue(ns->queue);
3567
	up_read(&ctrl->namespaces_rwsem);
3568
}
3569
EXPORT_SYMBOL_GPL(nvme_stop_queues);
3570

3571
void nvme_start_queues(struct nvme_ctrl *ctrl)
3572 3573 3574
{
	struct nvme_ns *ns;

3575
	down_read(&ctrl->namespaces_rwsem);
3576
	list_for_each_entry(ns, &ctrl->namespaces, list)
3577
		blk_mq_unquiesce_queue(ns->queue);
3578
	up_read(&ctrl->namespaces_rwsem);
3579
}
3580
EXPORT_SYMBOL_GPL(nvme_start_queues);
3581

S
Sagi Grimberg 已提交
3582 3583 3584 3585 3586 3587 3588 3589 3590 3591
int nvme_reinit_tagset(struct nvme_ctrl *ctrl, struct blk_mq_tag_set *set)
{
	if (!ctrl->ops->reinit_request)
		return 0;

	return blk_mq_tagset_iter(set, set->driver_data,
			ctrl->ops->reinit_request);
}
EXPORT_SYMBOL_GPL(nvme_reinit_tagset);

3592 3593
int __init nvme_core_init(void)
{
3594
	int result = -ENOMEM;
3595

3596 3597 3598
	nvme_wq = alloc_workqueue("nvme-wq",
			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
	if (!nvme_wq)
3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609
		goto out;

	nvme_reset_wq = alloc_workqueue("nvme-reset-wq",
			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
	if (!nvme_reset_wq)
		goto destroy_wq;

	nvme_delete_wq = alloc_workqueue("nvme-delete-wq",
			WQ_UNBOUND | WQ_MEM_RECLAIM | WQ_SYSFS, 0);
	if (!nvme_delete_wq)
		goto destroy_reset_wq;
3610

3611
	result = alloc_chrdev_region(&nvme_chr_devt, 0, NVME_MINORS, "nvme");
3612
	if (result < 0)
3613
		goto destroy_delete_wq;
3614 3615 3616 3617 3618 3619 3620

	nvme_class = class_create(THIS_MODULE, "nvme");
	if (IS_ERR(nvme_class)) {
		result = PTR_ERR(nvme_class);
		goto unregister_chrdev;
	}

C
Christoph Hellwig 已提交
3621 3622 3623 3624 3625
	nvme_subsys_class = class_create(THIS_MODULE, "nvme-subsystem");
	if (IS_ERR(nvme_subsys_class)) {
		result = PTR_ERR(nvme_subsys_class);
		goto destroy_class;
	}
3626
	return 0;
3627

C
Christoph Hellwig 已提交
3628 3629
destroy_class:
	class_destroy(nvme_class);
3630
unregister_chrdev:
3631
	unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
3632 3633 3634 3635
destroy_delete_wq:
	destroy_workqueue(nvme_delete_wq);
destroy_reset_wq:
	destroy_workqueue(nvme_reset_wq);
3636 3637
destroy_wq:
	destroy_workqueue(nvme_wq);
3638
out:
3639
	return result;
3640 3641 3642 3643
}

void nvme_core_exit(void)
{
C
Christoph Hellwig 已提交
3644 3645
	ida_destroy(&nvme_subsystems_ida);
	class_destroy(nvme_subsys_class);
3646
	class_destroy(nvme_class);
3647
	unregister_chrdev_region(nvme_chr_devt, NVME_MINORS);
3648 3649
	destroy_workqueue(nvme_delete_wq);
	destroy_workqueue(nvme_reset_wq);
3650
	destroy_workqueue(nvme_wq);
3651
}
3652 3653 3654 3655 3656

MODULE_LICENSE("GPL");
MODULE_VERSION("1.0");
module_init(nvme_core_init);
module_exit(nvme_core_exit);