From 5fd4ce1b005bd6ede913763f65efae9af6f7f386 Mon Sep 17 00:00:00 2001 From: Christoph Hellwig Date: Sat, 28 Nov 2015 15:03:49 +0100 Subject: [PATCH] nvme: move nvme_{enable,disable,shutdown}_ctrl to common code Signed-off-by: Christoph Hellwig Signed-off-by: Jens Axboe --- drivers/nvme/host/core.c | 106 +++++++++++++++++++++++++++++++ drivers/nvme/host/nvme.h | 11 ++++ drivers/nvme/host/pci.c | 133 +++++++-------------------------------- 3 files changed, 141 insertions(+), 109 deletions(-) diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index 63ec86a93b83..e3179b33ff81 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -670,6 +671,111 @@ const struct block_device_operations nvme_fops = { .pr_ops = &nvme_pr_ops, }; +static int nvme_wait_ready(struct nvme_ctrl *ctrl, u64 cap, bool enabled) +{ + unsigned long timeout = + ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; + u32 csts, bit = enabled ? NVME_CSTS_RDY : 0; + int ret; + + while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) { + if ((csts & NVME_CSTS_RDY) == bit) + break; + + msleep(100); + if (fatal_signal_pending(current)) + return -EINTR; + if (time_after(jiffies, timeout)) { + dev_err(ctrl->dev, + "Device not ready; aborting %s\n", enabled ? + "initialisation" : "reset"); + return -ENODEV; + } + } + + return ret; +} + +/* + * If the device has been passed off to us in an enabled state, just clear + * the enabled bit. The spec says we should set the 'shutdown notification + * bits', but doing so may cause the device to complete commands to the + * admin queue ... and we don't know what memory that might be pointing at! + */ +int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap) +{ + int ret; + + ctrl->ctrl_config &= ~NVME_CC_SHN_MASK; + ctrl->ctrl_config &= ~NVME_CC_ENABLE; + + ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config); + if (ret) + return ret; + return nvme_wait_ready(ctrl, cap, false); +} + +int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap) +{ + /* + * Default to a 4K page size, with the intention to update this + * path in the future to accomodate architectures with differing + * kernel and IO page sizes. + */ + unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12; + int ret; + + if (page_shift < dev_page_min) { + dev_err(ctrl->dev, + "Minimum device page size %u too large for host (%u)\n", + 1 << dev_page_min, 1 << page_shift); + return -ENODEV; + } + + ctrl->page_size = 1 << page_shift; + + ctrl->ctrl_config = NVME_CC_CSS_NVM; + ctrl->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT; + ctrl->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; + ctrl->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; + ctrl->ctrl_config |= NVME_CC_ENABLE; + + ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config); + if (ret) + return ret; + return nvme_wait_ready(ctrl, cap, true); +} + +int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl) +{ + unsigned long timeout = SHUTDOWN_TIMEOUT + jiffies; + u32 csts; + int ret; + + ctrl->ctrl_config &= ~NVME_CC_SHN_MASK; + ctrl->ctrl_config |= NVME_CC_SHN_NORMAL; + + ret = ctrl->ops->reg_write32(ctrl, NVME_REG_CC, ctrl->ctrl_config); + if (ret) + return ret; + + while ((ret = ctrl->ops->reg_read32(ctrl, NVME_REG_CSTS, &csts)) == 0) { + if ((csts & NVME_CSTS_SHST_MASK) == NVME_CSTS_SHST_CMPLT) + break; + + msleep(100); + if (fatal_signal_pending(current)) + return -EINTR; + if (time_after(jiffies, timeout)) { + dev_err(ctrl->dev, + "Device shutdown incomplete; abort shutdown\n"); + return -ENODEV; + } + } + + return ret; +} + static void nvme_free_ctrl(struct kref *kref) { struct nvme_ctrl *ctrl = container_of(kref, struct nvme_ctrl, kref); diff --git a/drivers/nvme/host/nvme.h b/drivers/nvme/host/nvme.h index f7f16e32104f..b6c5a55ed59f 100644 --- a/drivers/nvme/host/nvme.h +++ b/drivers/nvme/host/nvme.h @@ -27,6 +27,9 @@ extern unsigned char nvme_io_timeout; extern unsigned char admin_timeout; #define ADMIN_TIMEOUT (admin_timeout * HZ) +extern unsigned char shutdown_timeout; +#define SHUTDOWN_TIMEOUT (shutdown_timeout * HZ) + enum { NVME_NS_LBA = 0, NVME_NS_LIGHTNVM = 1, @@ -55,6 +58,10 @@ struct nvme_ctrl { char serial[20]; char model[40]; char firmware_rev[8]; + + u32 ctrl_config; + + u32 page_size; u16 oncs; u16 abort_limit; u8 event_limit; @@ -85,6 +92,7 @@ struct nvme_ns { struct nvme_ctrl_ops { int (*reg_read32)(struct nvme_ctrl *ctrl, u32 off, u32 *val); + int (*reg_write32)(struct nvme_ctrl *ctrl, u32 off, u32 val); void (*free_ctrl)(struct nvme_ctrl *ctrl); }; @@ -165,6 +173,9 @@ static inline int nvme_error_status(u16 status) } } +int nvme_disable_ctrl(struct nvme_ctrl *ctrl, u64 cap); +int nvme_enable_ctrl(struct nvme_ctrl *ctrl, u64 cap); +int nvme_shutdown_ctrl(struct nvme_ctrl *ctrl); void nvme_put_ctrl(struct nvme_ctrl *ctrl); void nvme_put_ns(struct nvme_ns *ns); diff --git a/drivers/nvme/host/pci.c b/drivers/nvme/host/pci.c index 1ae94cd74702..ccb315101a5e 100644 --- a/drivers/nvme/host/pci.c +++ b/drivers/nvme/host/pci.c @@ -52,7 +52,6 @@ #define NVME_AQ_DEPTH 256 #define SQ_SIZE(depth) (depth * sizeof(struct nvme_command)) #define CQ_SIZE(depth) (depth * sizeof(struct nvme_completion)) -#define SHUTDOWN_TIMEOUT (shutdown_timeout * HZ) unsigned char admin_timeout = 60; module_param(admin_timeout, byte, 0644); @@ -62,7 +61,7 @@ unsigned char nvme_io_timeout = 30; module_param_named(io_timeout, nvme_io_timeout, byte, 0644); MODULE_PARM_DESC(io_timeout, "timeout in seconds for I/O"); -static unsigned char shutdown_timeout = 5; +unsigned char shutdown_timeout = 5; module_param(shutdown_timeout, byte, 0644); MODULE_PARM_DESC(shutdown_timeout, "timeout in seconds for controller shutdown"); @@ -122,7 +121,6 @@ struct nvme_dev { unsigned max_qid; int q_depth; u32 db_stride; - u32 ctrl_config; struct msix_entry *entry; void __iomem *bar; struct list_head namespaces; @@ -133,7 +131,6 @@ struct nvme_dev { bool subsystem; u32 max_hw_sectors; u32 stripe_size; - u32 page_size; void __iomem *cmb; dma_addr_t cmb_dma_addr; u64 cmb_size; @@ -225,7 +222,7 @@ struct nvme_cmd_info { * Max size of iod being embedded in the request payload */ #define NVME_INT_PAGES 2 -#define NVME_INT_BYTES(dev) (NVME_INT_PAGES * (dev)->page_size) +#define NVME_INT_BYTES(dev) (NVME_INT_PAGES * (dev)->ctrl.page_size) #define NVME_INT_MASK 0x01 /* @@ -235,7 +232,8 @@ struct nvme_cmd_info { */ static int nvme_npages(unsigned size, struct nvme_dev *dev) { - unsigned nprps = DIV_ROUND_UP(size + dev->page_size, dev->page_size); + unsigned nprps = DIV_ROUND_UP(size + dev->ctrl.page_size, + dev->ctrl.page_size); return DIV_ROUND_UP(8 * nprps, PAGE_SIZE - 8); } @@ -527,7 +525,7 @@ static struct nvme_iod *nvme_alloc_iod(struct request *rq, struct nvme_dev *dev, static void nvme_free_iod(struct nvme_dev *dev, struct nvme_iod *iod) { - const int last_prp = dev->page_size / 8 - 1; + const int last_prp = dev->ctrl.page_size / 8 - 1; int i; __le64 **list = iod_list(iod); dma_addr_t prp_dma = iod->first_dma; @@ -668,7 +666,7 @@ static bool nvme_setup_prps(struct nvme_dev *dev, struct nvme_iod *iod, struct scatterlist *sg = iod->sg; int dma_len = sg_dma_len(sg); u64 dma_addr = sg_dma_address(sg); - u32 page_size = dev->page_size; + u32 page_size = dev->ctrl.page_size; int offset = dma_addr & (page_size - 1); __le64 *prp_list; __le64 **list = iod_list(iod); @@ -1275,11 +1273,12 @@ static int nvme_cmb_qdepth(struct nvme_dev *dev, int nr_io_queues, int entry_size) { int q_depth = dev->q_depth; - unsigned q_size_aligned = roundup(q_depth * entry_size, dev->page_size); + unsigned q_size_aligned = roundup(q_depth * entry_size, + dev->ctrl.page_size); if (q_size_aligned * nr_io_queues > dev->cmb_size) { u64 mem_per_q = div_u64(dev->cmb_size, nr_io_queues); - mem_per_q = round_down(mem_per_q, dev->page_size); + mem_per_q = round_down(mem_per_q, dev->ctrl.page_size); q_depth = div_u64(mem_per_q, entry_size); /* @@ -1298,8 +1297,8 @@ static int nvme_alloc_sq_cmds(struct nvme_dev *dev, struct nvme_queue *nvmeq, int qid, int depth) { if (qid && dev->cmb && use_cmb_sqes && NVME_CMB_SQS(dev->cmbsz)) { - unsigned offset = (qid - 1) * - roundup(SQ_SIZE(depth), dev->page_size); + unsigned offset = (qid - 1) * roundup(SQ_SIZE(depth), + dev->ctrl.page_size); nvmeq->sq_dma_addr = dev->cmb_dma_addr + offset; nvmeq->sq_cmds_io = dev->cmb + offset; } else { @@ -1407,97 +1406,6 @@ static int nvme_create_queue(struct nvme_queue *nvmeq, int qid) return result; } -static int nvme_wait_ready(struct nvme_dev *dev, u64 cap, bool enabled) -{ - unsigned long timeout; - u32 bit = enabled ? NVME_CSTS_RDY : 0; - - timeout = ((NVME_CAP_TIMEOUT(cap) + 1) * HZ / 2) + jiffies; - - while ((readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_RDY) != bit) { - msleep(100); - if (fatal_signal_pending(current)) - return -EINTR; - if (time_after(jiffies, timeout)) { - dev_err(dev->dev, - "Device not ready; aborting %s\n", enabled ? - "initialisation" : "reset"); - return -ENODEV; - } - } - - return 0; -} - -/* - * If the device has been passed off to us in an enabled state, just clear - * the enabled bit. The spec says we should set the 'shutdown notification - * bits', but doing so may cause the device to complete commands to the - * admin queue ... and we don't know what memory that might be pointing at! - */ -static int nvme_disable_ctrl(struct nvme_dev *dev, u64 cap) -{ - dev->ctrl_config &= ~NVME_CC_SHN_MASK; - dev->ctrl_config &= ~NVME_CC_ENABLE; - writel(dev->ctrl_config, dev->bar + NVME_REG_CC); - - return nvme_wait_ready(dev, cap, false); -} - -static int nvme_enable_ctrl(struct nvme_dev *dev, u64 cap) -{ - /* - * Default to a 4K page size, with the intention to update this - * path in the future to accomodate architectures with differing - * kernel and IO page sizes. - */ - unsigned dev_page_min = NVME_CAP_MPSMIN(cap) + 12, page_shift = 12; - - if (page_shift < dev_page_min) { - dev_err(dev->dev, - "Minimum device page size %u too large for host (%u)\n", - 1 << dev_page_min, 1 << page_shift); - return -ENODEV; - } - - dev->page_size = 1 << page_shift; - - dev->ctrl_config = NVME_CC_CSS_NVM; - dev->ctrl_config |= (page_shift - 12) << NVME_CC_MPS_SHIFT; - dev->ctrl_config |= NVME_CC_ARB_RR | NVME_CC_SHN_NONE; - dev->ctrl_config |= NVME_CC_IOSQES | NVME_CC_IOCQES; - dev->ctrl_config |= NVME_CC_ENABLE; - - writel(dev->ctrl_config, dev->bar + NVME_REG_CC); - - return nvme_wait_ready(dev, cap, true); -} - -static int nvme_shutdown_ctrl(struct nvme_dev *dev) -{ - unsigned long timeout; - - dev->ctrl_config &= ~NVME_CC_SHN_MASK; - dev->ctrl_config |= NVME_CC_SHN_NORMAL; - - writel(dev->ctrl_config, dev->bar + NVME_REG_CC); - - timeout = SHUTDOWN_TIMEOUT + jiffies; - while ((readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_SHST_MASK) != - NVME_CSTS_SHST_CMPLT) { - msleep(100); - if (fatal_signal_pending(current)) - return -EINTR; - if (time_after(jiffies, timeout)) { - dev_err(dev->dev, - "Device shutdown incomplete; abort shutdown\n"); - return -ENODEV; - } - } - - return 0; -} - static struct blk_mq_ops nvme_mq_admin_ops = { .queue_rq = nvme_queue_rq, .map_queue = blk_mq_map_queue, @@ -1569,7 +1477,7 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev) (readl(dev->bar + NVME_REG_CSTS) & NVME_CSTS_NSSRO)) writel(NVME_CSTS_NSSRO, dev->bar + NVME_REG_CSTS); - result = nvme_disable_ctrl(dev, cap); + result = nvme_disable_ctrl(&dev->ctrl, cap); if (result < 0) return result; @@ -1587,7 +1495,7 @@ static int nvme_configure_admin_queue(struct nvme_dev *dev) lo_hi_writeq(nvmeq->sq_dma_addr, dev->bar + NVME_REG_ASQ); lo_hi_writeq(nvmeq->cq_dma_addr, dev->bar + NVME_REG_ACQ); - result = nvme_enable_ctrl(dev, cap); + result = nvme_enable_ctrl(&dev->ctrl, cap); if (result) goto free_nvmeq; @@ -1687,13 +1595,13 @@ static void nvme_alloc_ns(struct nvme_dev *dev, unsigned nsid) if (dev->max_hw_sectors) { blk_queue_max_hw_sectors(ns->queue, dev->max_hw_sectors); blk_queue_max_segments(ns->queue, - (dev->max_hw_sectors / (dev->page_size >> 9)) + 1); + (dev->max_hw_sectors / (dev->ctrl.page_size >> 9)) + 1); } if (dev->stripe_size) blk_queue_chunk_sectors(ns->queue, dev->stripe_size >> 9); if (dev->ctrl.vwc & NVME_CTRL_VWC_PRESENT) blk_queue_flush(ns->queue, REQ_FLUSH | REQ_FUA); - blk_queue_virt_boundary(ns->queue, dev->page_size - 1); + blk_queue_virt_boundary(ns->queue, dev->ctrl.page_size - 1); disk->major = nvme_major; disk->first_minor = 0; @@ -2181,7 +2089,7 @@ static void nvme_wait_dq(struct nvme_delq_ctx *dq, struct nvme_dev *dev) * queues than admin tags. */ set_current_state(TASK_RUNNING); - nvme_disable_ctrl(dev, + nvme_disable_ctrl(&dev->ctrl, lo_hi_readq(dev->bar + NVME_REG_CAP)); nvme_clear_queue(dev->queues[0]); flush_kthread_worker(dq->worker); @@ -2367,7 +2275,7 @@ static void nvme_dev_shutdown(struct nvme_dev *dev) } } else { nvme_disable_io_queues(dev); - nvme_shutdown_ctrl(dev); + nvme_shutdown_ctrl(&dev->ctrl); nvme_disable_queue(dev, 0); } nvme_dev_unmap(dev); @@ -2683,8 +2591,15 @@ static int nvme_pci_reg_read32(struct nvme_ctrl *ctrl, u32 off, u32 *val) return 0; } +static int nvme_pci_reg_write32(struct nvme_ctrl *ctrl, u32 off, u32 val) +{ + writel(val, to_nvme_dev(ctrl)->bar + off); + return 0; +} + static const struct nvme_ctrl_ops nvme_pci_ctrl_ops = { .reg_read32 = nvme_pci_reg_read32, + .reg_write32 = nvme_pci_reg_write32, .free_ctrl = nvme_pci_free_ctrl, }; -- GitLab