diff --git a/drivers/scsi/sssraid/sssraid.h b/drivers/scsi/sssraid/sssraid.h index 7fe485524da7a71b2456efb5fe8c780a00b13312..e5c78d3e9f28a1b738191ee3a24585893f5d561c 100644 --- a/drivers/scsi/sssraid/sssraid.h +++ b/drivers/scsi/sssraid/sssraid.h @@ -13,6 +13,7 @@ #define SSSRAID_NAME_LENGTH 32 #define BSG_NAME_SIZE 15 + /* * SSSRAID Vendor ID and Device IDs */ @@ -90,7 +91,7 @@ extern u32 admin_tmout; #define ADMIN_TIMEOUT (admin_tmout * HZ) #define SSSRAID_WAIT_ABNL_CMD_TIMEOUT 6 - +#define SSSRAID_WAIT_RST_IO_TIMEOUT 10 #define SSSRAID_DMA_MSK_BIT_MAX 64 enum { @@ -325,7 +326,7 @@ struct sssraid_ioc { int logging_level; char name[SSSRAID_NAME_LENGTH]; - int cpu_count; + u32 cpu_count; /* * before_affinity_msix_cnt is * min("FW support IO Queue count", num_online_cpus)+1 @@ -363,6 +364,11 @@ struct sssraid_ioc { struct list_head fwevt_list; struct sssraid_fwevt *current_event; + + void *senses; + dma_addr_t sense_dma_addr; + u32 last_qcnt; + u8 hdd_dispatch; }; /* @@ -790,6 +796,7 @@ struct sssraid_squeue { void *sense; dma_addr_t sense_dma_addr; struct dma_pool *prp_small_pool; + atomic_t inflight; }; struct sssraid_cqueue { @@ -813,6 +820,7 @@ struct sssraid_iod { enum sssraid_cmd_state state; int npages; u32 nsge; + u16 cid; u32 length; bool use_sgl; dma_addr_t first_dma; @@ -850,6 +858,11 @@ enum { SSSRAID_NVME_SSD_PD = 0x16, }; +enum { + DISPATCH_BY_CPU, + DISPATCH_BY_DISK, +}; + /* * define the SSSRAID scsi device queue depth */ @@ -929,11 +942,14 @@ struct sssraid_sdev_hostdata { u8 attr; u8 flag; u8 rg_id; - u8 rsvd[3]; + u8 hwq; + u16 pend_count; }; extern unsigned char small_pool_num; extern u32 io_queue_depth; +extern u32 max_hwq_num; +extern bool work_mode; irqreturn_t sssraid_isr_poll(int irq, void *privdata); bool sssraid_poll_cq(struct sssraid_ioc *sdioc, u16 qidx, int cid); void sssraid_submit_cmd(struct sssraid_squeue *sqinfo, const void *cmd); diff --git a/drivers/scsi/sssraid/sssraid_fw.c b/drivers/scsi/sssraid/sssraid_fw.c index 440b50fd5689341ab879731d0872d55ef13d55d1..a4b91c5219b3eac8d7a8d9afaa22da0d90b50754 100644 --- a/drivers/scsi/sssraid/sssraid_fw.c +++ b/drivers/scsi/sssraid/sssraid_fw.c @@ -194,7 +194,7 @@ static int sssraid_alloc_resources(struct sssraid_ioc *sdioc) } /* not num_online_cpus */ - nqueue = num_possible_cpus() + 1; + nqueue = min(num_possible_cpus(), max_hwq_num) + 1; sdioc->cqinfo = kcalloc_node(nqueue, sizeof(struct sssraid_cqueue), GFP_KERNEL, sdioc->numa_node); if (!sdioc->cqinfo) { @@ -282,7 +282,6 @@ static int sssraid_setup_resources(struct sssraid_ioc *sdioc) sdioc->cap = lo_hi_readq(sdioc->bar + SSSRAID_REG_CAP); sdioc->ioq_depth = min_t(u32, SSSRAID_CAP_MQES(sdioc->cap) + 1, io_queue_depth); - sdioc->scsi_qd = sdioc->ioq_depth - SSSRAID_PTCMDS_PERQ; sdioc->db_stride = 1 << SSSRAID_CAP_STRIDE(sdioc->cap); maskbit = SSSRAID_CAP_DMAMASK(sdioc->cap); @@ -357,10 +356,8 @@ static int sssraid_alloc_qpair(struct sssraid_ioc *sdioc, u16 qidx, u16 depth) cqinfo->cqes = dma_alloc_coherent(&sdioc->pdev->dev, CQ_SIZE(depth), &cqinfo->cq_dma_addr, GFP_KERNEL | __GFP_ZERO); - if (!cqinfo->cqes) { - ioc_err(sdioc, "failure at alloc dma space for cqueue.\n"); + if (!cqinfo->cqes) return -ENOMEM; - } sqinfo->sq_cmds = dma_alloc_coherent(&sdioc->pdev->dev, SQ_SIZE(qidx, depth), &sqinfo->sq_dma_addr, GFP_KERNEL); @@ -370,6 +367,13 @@ static int sssraid_alloc_qpair(struct sssraid_ioc *sdioc, u16 qidx, u16 depth) goto free_cqes; } + /* + * if single hw queue, we do not need to alloc sense buffer for every queue, + * we have alloced all on hiraid_alloc_resources. + */ + if (work_mode) + goto initq; + /* alloc sense buffer */ sqinfo->sense = dma_alloc_coherent(&sdioc->pdev->dev, SENSE_SIZE(depth), &sqinfo->sense_dma_addr, GFP_KERNEL | __GFP_ZERO); @@ -379,6 +383,7 @@ static int sssraid_alloc_qpair(struct sssraid_ioc *sdioc, u16 qidx, u16 depth) goto free_sq_cmds; } +initq: spin_lock_init(&sqinfo->sq_lock); spin_lock_init(&cqinfo->cq_lock); cqinfo->sdioc = sdioc; @@ -413,6 +418,8 @@ static void sssraid_init_queue(struct sssraid_ioc *sdioc, u16 qidx) memset((void *)cqinfo->cqes, 0, CQ_SIZE(sqinfo->q_depth)); sqinfo->sq_tail = 0; + atomic_set(&sqinfo->inflight, 0); + cqinfo->cq_head = 0; cqinfo->cq_phase = 1; sqinfo->q_db = &sdioc->dbs[qidx * 2 * sdioc->db_stride]; @@ -819,6 +826,8 @@ int sssraid_init_ctrl_info(struct sssraid_ioc *sdioc) if (sdioc->ctrl_info->aerl > SSSRAID_NR_AEN_CMDS) sdioc->ctrl_info->aerl = SSSRAID_NR_AEN_CMDS; + sdioc->scsi_qd = work_mode ? le16_to_cpu(sdioc->ctrl_info->max_cmds) : + sdioc->ioq_depth - SSSRAID_PTCMDS_PERQ; return 0; } @@ -902,6 +911,8 @@ static int sssraid_create_io_cq(struct sssraid_ioc *sdioc, u16 qidx) if (retval) return retval; + if (!sdioc->last_qcnt) + sdioc->last_qcnt = sdioc->init_done_queue_cnt; /* * cqinfo initialization at sssraid_init_queue */ @@ -1093,9 +1104,18 @@ static int sssraid_setup_io_qpair(struct sssraid_ioc *sdioc) } } - ioc_info(sdioc, "init_done_queue_cnt[%d], intr_info_count[%d] num_queues[%d]", + if (work_mode && !sdioc->senses) { + sdioc->senses = dma_alloc_coherent(&sdioc->pdev->dev, + SENSE_SIZE(SSSRAID_IO_BLK_MQ_DEPTH + max_hwq_num * + SSSRAID_PTCMDS_PERQ), &sdioc->sense_dma_addr, + GFP_KERNEL | __GFP_ZERO); + if (!sdioc->senses) + return -ENOMEM; + } + + ioc_info(sdioc, "init_done_queue_cnt[%d], intr_info_count[%d] num_queues[%d], last_online[%d]", sdioc->init_done_queue_cnt, - sdioc->intr_info_count, num_queues); + sdioc->intr_info_count, num_queues, sdioc->last_qcnt); return retval >= 0 ? 0 : retval; } @@ -1194,10 +1214,8 @@ static int sssraid_disk_list_init(struct sssraid_ioc *sdioc) sdioc->devices = kzalloc_node(nd * sizeof(struct sssraid_dev_info), GFP_KERNEL, sdioc->numa_node); - if (!sdioc->devices) { - ioc_err(sdioc, "err: failed to alloc memory for device info.\n"); + if (!sdioc->devices) return -ENOMEM; - } return 0; } @@ -1230,21 +1248,19 @@ int sssraid_init_ioc(struct sssraid_ioc *sdioc, u8 re_init) retval); goto out_nocleanup; } - } - /* reset need re-setup */ - retval = sssraid_setup_resources(sdioc); - if (retval) { - ioc_err(sdioc, "Err: Failed to setup resources, ret %d\n", - retval); - goto out_failed; - } + /* reset need re-setup */ + retval = sssraid_setup_resources(sdioc); + if (retval) { + ioc_err(sdioc, "Err: Failed to setup resources, ret %d\n", + retval); + goto out_failed; + } - if (!re_init) { retval = sssraid_alloc_admin_cmds(sdioc); if (retval) { ioc_err(sdioc, "Err: Failed to alloc admin cmds, ret %d\n", - retval); + retval); goto out_failed; } /* put here: @@ -1253,17 +1269,14 @@ int sssraid_init_ioc(struct sssraid_ioc *sdioc, u8 re_init) retval = sssraid_alloc_qpair(sdioc, 0, SSSRAID_ADMQ_DEPTH); if (retval) { ioc_err(sdioc, "Err: Failed to alloc admin queue, ret %d\n", - retval); + retval); goto out_failed; } } retval = sssraid_setup_admin_qpair(sdioc); - if (retval) { - ioc_err(sdioc, "Err: Failed to setup admin queue, ret %d\n", - retval); + if (retval) goto out_failed; - } /* 1. unregister all interrupt * 2. admin interrupt registry @@ -1275,14 +1288,16 @@ int sssraid_init_ioc(struct sssraid_ioc *sdioc, u8 re_init) goto out_failed; } - retval = sssraid_init_ctrl_info(sdioc); - if (retval) { - ioc_err(sdioc, "Failed to get ctrl info error %d\n", - retval); - goto out_failed; + if (!re_init) { + retval = sssraid_init_ctrl_info(sdioc); + if (retval) { + ioc_err(sdioc, "Failed to get ctrl info error %d\n", + retval); + goto out_failed; + } } - nr_ioqs = sdioc->cpu_count; + nr_ioqs = min(sdioc->cpu_count, max_hwq_num); retval = sssraid_set_queue_cnt(sdioc, &nr_ioqs); if (retval) { ioc_err(sdioc, "Failed to set queue cnt error %d\n", @@ -1303,19 +1318,19 @@ int sssraid_init_ioc(struct sssraid_ioc *sdioc, u8 re_init) goto out_failed; } - /* remap */ - bar_size = SSSRAID_REG_DBS + ((nr_ioqs + 1) * 8 * sdioc->db_stride); - retval = sssraid_remap_bar(sdioc, bar_size); - if (retval) { - ioc_err(sdioc, "Failed to re-map bar, error %d\n", - retval); - goto out_failed; - } - sdioc->sqinfo[0].q_db = sdioc->dbs; - /* num_vecs no sense, abandon */ if (!re_init) { + /* remap */ + bar_size = SSSRAID_REG_DBS + ((nr_ioqs + 1) * 8 * sdioc->db_stride); + retval = sssraid_remap_bar(sdioc, bar_size); + if (retval) { + ioc_err(sdioc, "Failed to re-map bar, error %d\n", + retval); + goto out_failed; + } + sdioc->sqinfo[0].q_db = sdioc->dbs; + for (i = sdioc->init_done_queue_cnt; i < sdioc->intr_info_count; i++) { retval = sssraid_alloc_qpair(sdioc, i, sdioc->ioq_depth); if (retval) { @@ -1403,6 +1418,15 @@ static void sssraid_free_ioq_ptcmds(struct sssraid_ioc *sdioc) INIT_LIST_HEAD(&sdioc->ioq_pt_list); } +static void sssraid_free_sense_buffer(struct sssraid_ioc *sdioc) +{ + if (sdioc->senses) { + dma_free_coherent(&sdioc->pdev->dev, SENSE_SIZE(SSSRAID_IO_BLK_MQ_DEPTH + + max_hwq_num * SSSRAID_PTCMDS_PERQ), sdioc->senses, sdioc->sense_dma_addr); + sdioc->senses = NULL; + } +} + static void sssraid_delete_io_queues(struct sssraid_ioc *sdioc) { u16 queues = sdioc->init_done_queue_cnt - SSSRAID_ADM_QUEUE_NUM; @@ -1419,6 +1443,7 @@ static void sssraid_delete_io_queues(struct sssraid_ioc *sdioc) return; } + sssraid_free_sense_buffer(sdioc); for (pass = 0; pass < 2; pass++) { for (i = queues; i > 0; i--) if (sssraid_delete_queue(sdioc, opcode, i)) @@ -1516,7 +1541,12 @@ static void sssraid_complete_ioq_cmnd(struct sssraid_ioc *sdioc, u16 qidx, struct request *req; unsigned long elapsed; - tags = sdioc->shost->tag_set.tags[sqinfo->qidx - 1]; + atomic_dec(&sqinfo->inflight); + + if (work_mode) + tags = sdioc->shost->tag_set.tags[0]; + else + tags = sdioc->shost->tag_set.tags[sqinfo->qidx - 1]; req = blk_mq_tag_to_rq(tags, le16_to_cpu(cqe->cmd_id)); if (unlikely(!req || !blk_mq_request_started(req))) { @@ -1556,9 +1586,9 @@ static void sssraid_complete_ioq_cmnd(struct sssraid_ioc *sdioc, u16 qidx, scmd->scsi_done(scmd); } -static void sssraid_process_admin_cq(struct sssraid_ioc *sdioc, - struct sssraid_squeue *sqinfo, - struct sssraid_completion *cqe) +static +void sssraid_process_admin_cq(struct sssraid_ioc *sdioc, struct sssraid_squeue *sqinfo, + struct sssraid_completion *cqe) { struct sssraid_fwevt *fwevt = NULL; u16 cid = le16_to_cpu(cqe->cmd_id), sz; @@ -1584,9 +1614,8 @@ static void sssraid_process_admin_cq(struct sssraid_ioc *sdioc, } } -static void sssraid_process_io_cq(struct sssraid_ioc *sdioc, - struct sssraid_squeue *sqinfo, - struct sssraid_completion *cqe) +static void sssraid_process_io_cq(struct sssraid_ioc *sdioc, struct sssraid_squeue *sqinfo, + struct sssraid_completion *cqe) { u16 cid = le16_to_cpu(cqe->cmd_id); @@ -1604,7 +1633,7 @@ static inline void sssraid_handle_cqe(struct sssraid_ioc *sdioc, u16 mdix, u16 d struct sssraid_completion *cqe = &cqinfo->cqes[didx]; u16 cid = le16_to_cpu(cqe->cmd_id); - if (unlikely(cid >= sqinfo->q_depth)) { + if (unlikely(!work_mode && (cid >= sqinfo->q_depth))) { ioc_err(sdioc, "Err: invalid command id[%d] completed on queue %d\n", cid, cqe->sq_id); return; @@ -1670,8 +1699,9 @@ static void sssraid_free_all_queues(struct sssraid_ioc *sdioc) (void *)cqinfo->cqes, cqinfo->cq_dma_addr); dma_free_coherent(&sdioc->pdev->dev, SQ_SIZE(sqinfo->qidx, sqinfo->q_depth), sqinfo->sq_cmds, sqinfo->sq_dma_addr); - dma_free_coherent(&sdioc->pdev->dev, SENSE_SIZE(sqinfo->q_depth), - sqinfo->sense, sqinfo->sense_dma_addr); + if (!work_mode) + dma_free_coherent(&sdioc->pdev->dev, SENSE_SIZE(sqinfo->q_depth), + sqinfo->sense, sqinfo->sense_dma_addr); } sdioc->init_done_queue_cnt = 0; @@ -1732,11 +1762,11 @@ int sssraid_soft_reset_handler(struct sssraid_ioc *sdioc) * i.e. sssraid_cleanup_ioc(1) */ if (sdioc->ctrl_config & SSSRAID_CC_ENABLE) { - ioc_info(sdioc, "start disable admin queue\n"); + ioc_info(sdioc, "\n"); retval = sssraid_disable_admin_queue(sdioc, 0); } - sssraid_cleanup_resources(sdioc); + sssraid_cleanup_isr(sdioc); /* realize above here: * sssraid_dev_disable -> sssraid_back_all_io @@ -1747,14 +1777,13 @@ int sssraid_soft_reset_handler(struct sssraid_ioc *sdioc) goto host_reset_failed; retval = sssraid_init_ioc(sdioc, 1); - if (retval) - goto cleanup_resources; + if (retval || sdioc->last_qcnt != sdioc->init_done_queue_cnt) + goto host_reset_failed; + sssraid_scan_disk(sdioc); sssraid_change_host_state(sdioc, SSSRAID_LIVE); return 0; -cleanup_resources: - sssraid_cleanup_resources(sdioc); host_reset_failed: sssraid_change_host_state(sdioc, SSSRAID_DEAD); ioc_err(sdioc, "err, host reset failed\n"); diff --git a/drivers/scsi/sssraid/sssraid_os.c b/drivers/scsi/sssraid/sssraid_os.c index 6b8f143190d17f68f7363dc14fed6ee258bfa7c2..d442f0550362e7fc8ac48bdee71c98e772a5a63b 100644 --- a/drivers/scsi/sssraid/sssraid_os.c +++ b/drivers/scsi/sssraid/sssraid_os.c @@ -38,6 +38,9 @@ #include "sssraid.h" #include "sssraid_debug.h" +#define MAX_IO_QUEUES 128 +#define MIN_IO_QUEUES 1 + u32 admin_tmout = 60; module_param(admin_tmout, uint, 0644); MODULE_PARM_DESC(admin_tmout, "admin commands timeout (seconds)"); @@ -50,6 +53,31 @@ static u32 scmd_tmout_vd = 180; module_param(scmd_tmout_vd, uint, 0644); MODULE_PARM_DESC(scmd_tmout_vd, "scsi commands timeout for vd(seconds)"); +bool work_mode; +module_param(work_mode, bool, 0444); +MODULE_PARM_DESC(work_mode, "work mode switch, default false for multi hw queues"); + +static int ioq_num_set(const char *val, const struct kernel_param *kp) +{ + int n = 0; + int ret; + + ret = kstrtoint(val, 10, &n); + if (ret != 0 || n < MIN_IO_QUEUES || n > MAX_IO_QUEUES) + return -EINVAL; + + return param_set_int(val, kp); +} + +static const struct kernel_param_ops max_hwq_num_ops = { + .set = ioq_num_set, + .get = param_get_uint, +}; + +u32 max_hwq_num = 128; +module_param_cb(max_hwq_num, &max_hwq_num_ops, &max_hwq_num, 0444); +MODULE_PARM_DESC(max_hwq_num, "max num of hw io queues, should >= 1, default 128"); + static int ioq_depth_set(const char *val, const struct kernel_param *kp); static const struct kernel_param_ops ioq_depth_ops = { .set = ioq_depth_set, @@ -124,6 +152,11 @@ enum FW_STAT_CODE { FW_STAT_NEED_RETRY }; +enum { + FW_EH_OK = 0, + FW_EH_DEV_NONE = 0x701 +}; + static const char * const raid_levels[] = {"0", "1", "5", "6", "10", "50", "60", "NA"}; static const char * const raid_states[] = { @@ -270,6 +303,19 @@ static struct sssraid_fwevt *sssraid_dequeue_fwevt( return fwevt; } +static bool sssraid_disk_is_hdd(u8 attr) +{ + switch (SSSRAID_DISK_TYPE(attr)) { + case SSSRAID_SAS_HDD_VD: + case SSSRAID_SATA_HDD_VD: + case SSSRAID_SAS_HDD_PD: + case SSSRAID_SATA_HDD_PD: + return true; + default: + return false; + } +} + void sssraid_cleanup_fwevt_list(struct sssraid_ioc *sdioc) { struct sssraid_fwevt *fwevt = NULL; @@ -566,8 +612,8 @@ static void sssraid_shost_init(struct sssraid_ioc *sdioc) bus = pdev->bus->number; dev_func = pdev->devfn; - sdioc->shost->nr_hw_queues = SSSRAID_NR_HW_QUEUES; - sdioc->shost->can_queue = (sdioc->ioq_depth - SSSRAID_PTCMDS_PERQ); + sdioc->shost->nr_hw_queues = work_mode ? 1 : SSSRAID_NR_HW_QUEUES; + sdioc->shost->can_queue = SSSRAID_IO_BLK_MQ_DEPTH; sdioc->shost->sg_tablesize = le16_to_cpu(sdioc->ctrl_info->max_num_sge); /* 512B per sector */ @@ -583,11 +629,22 @@ static void sssraid_shost_init(struct sssraid_ioc *sdioc) sdioc->shost->hostt->cmd_size = sssraid_cmd_size(sdioc); } -static inline void sssraid_get_tag_from_scmd(struct scsi_cmnd *scmd, u16 *qidx, u16 *cid) +static +inline void sssraid_get_tag_from_scmd(struct scsi_cmnd *scmd, u16 *qidx, + struct sssraid_ioc *sdioc, u16 *cid, + struct sssraid_sdev_hostdata *hostdata) { u32 tag = blk_mq_unique_tag(scmd->request); - *qidx = blk_mq_unique_tag_to_hwq(tag) + 1; + if (work_mode) { + if ((sdioc->hdd_dispatch == DISPATCH_BY_DISK) && (hostdata->hwq != 0)) + *qidx = hostdata->hwq; + else + *qidx = raw_smp_processor_id() % (sdioc->init_done_queue_cnt - 1) + 1; + } else { + *qidx = blk_mq_unique_tag_to_hwq(tag) + 1; + } + *cid = blk_mq_unique_tag_to_tag(tag); } @@ -971,54 +1028,97 @@ static bool sssraid_check_scmd_completed(struct scsi_cmnd *scmd) struct sssraid_ioc *sdioc = shost_priv(scmd->device->host); struct sssraid_iod *iod = scsi_cmd_priv(scmd); struct sssraid_squeue *sqinfo; - u16 hwq, cid; - sssraid_get_tag_from_scmd(scmd, &hwq, &cid); - sqinfo = &sdioc->sqinfo[hwq]; - if (READ_ONCE(iod->state) == SSSRAID_CMDSTAT_COMPLETE || sssraid_poll_cq(sdioc, hwq, cid)) { + sqinfo = iod->sqinfo; + if (!sqinfo) + return false; + + if (READ_ONCE(iod->state) == SSSRAID_CMDSTAT_COMPLETE || + sssraid_poll_cq(sdioc, sqinfo->qidx, iod->cid)) { ioc_warn(sdioc, "warn: cid[%d] qidx[%d] has completed\n", - cid, sqinfo->qidx); + iod->cid, sqinfo->qidx); return true; } return false; } +static bool sssraid_tgt_rst_pending_io_count(struct request *rq, void *data, bool reserved) +{ + unsigned int id = *(unsigned int *)data; + struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq); + struct sssraid_iod *iod; + struct sssraid_sdev_hostdata *hostdata; + + if (scmd) { + iod = scsi_cmd_priv(scmd); + if ((iod->state == SSSRAID_CMDSTAT_FLIGHT) || + (iod->state == SSSRAID_CMDSTAT_TIMEOUT)) { + if ((scmd->device) && (scmd->device->id == id)) { + hostdata = scmd->device->hostdata; + hostdata->pend_count++; + } + } + } + return true; +} + +static int wait_tgt_reset_io_done(struct scsi_cmnd *scmd) +{ + u16 timeout = 0; + struct sssraid_sdev_hostdata *hostdata; + struct sssraid_ioc *sdioc = shost_priv(scmd->device->host); + + hostdata = scmd->device->hostdata; + + do { + hostdata->pend_count = 0; + blk_mq_tagset_busy_iter(&sdioc->shost->tag_set, sssraid_tgt_rst_pending_io_count, + (void *) (&scmd->device->id)); + if (!hostdata->pend_count) + return 0; + + msleep(500); + timeout++; + } while (timeout <= SSSRAID_WAIT_RST_IO_TIMEOUT); + + return -ETIMEDOUT; +} + static int sssraid_scsi_reset(struct scsi_cmnd *scmd, enum sssraid_scsi_rst_type rst) { struct sssraid_ioc *sdioc = shost_priv(scmd->device->host); - struct sssraid_iod *iod = scsi_cmd_priv(scmd); struct sssraid_sdev_hostdata *hostdata; - u16 hwq, cid; int ret; - scsi_print_command(scmd); - - if (sdioc->state != SSSRAID_LIVE || !sssraid_wait_abnl_cmd_done(iod) || - sssraid_check_scmd_completed(scmd)) + if (sdioc->state != SSSRAID_LIVE) return SUCCESS; hostdata = scmd->device->hostdata; - sssraid_get_tag_from_scmd(scmd, &hwq, &cid); - ioc_warn(sdioc, "warn: cid[%d] qidx[%d] timeout, %s reset\n", cid, hwq, - rst ? "bus" : "target"); + ioc_warn(sdioc, "sdev[%d:%d] send %s reset\n", scmd->device->channel, scmd->device->id, + rst ? "bus" : "target"); ret = sssraid_send_reset_cmd(sdioc, rst, hostdata->hdid); - if (ret == 0) { - ret = sssraid_wait_abnl_cmd_done(iod); - if (ret) { - ioc_warn(sdioc, "warn: cid[%d] qidx[%d] %s reset failed, no found\n", - cid, hwq, rst ? "bus" : "target"); + if ((ret == 0) || (ret == FW_EH_DEV_NONE && rst == SSSRAID_RESET_TARGET)) { + if (rst == SSSRAID_RESET_TARGET) { + ret = wait_tgt_reset_io_done(scmd); + if (ret) { + ioc_warn(sdioc, "sdev[%d:%d] target has %d pending comands;" + "target reset failed\n", scmd->device->channel, + scmd->device->id, hostdata->pend_count); return FAILED; + } } - ioc_warn(sdioc, "cid[%d] qidx[%d] %s reset success\n", cid, hwq, - rst ? "bus" : "target"); + ioc_warn(sdioc, "sdev[%d:%d] %s reset success\n", scmd->device->channel, + scmd->device->id, rst ? "bus" : "target"); return SUCCESS; } - ioc_warn(sdioc, "warn: cid[%d] qidx[%d] ret[%d] %s reset failed\n", cid, hwq, ret, - rst ? "bus" : "target"); + ioc_warn(sdioc, "sdev[%d:%d] %s reset failed\n", scmd->device->channel, scmd->device->id, + rst ? "bus" : "target"); + return FAILED; + } bool sssraid_change_host_state(struct sssraid_ioc *sdioc, enum sssraid_state newstate) @@ -1252,8 +1352,17 @@ static int sssraid_submit_ioq_sync_cmd(struct sssraid_ioc *sdioc, struct sssraid sqinfo = &sdioc->sqinfo[pt_cmd->qid]; ret = pt_cmd->cid * SCSI_SENSE_BUFFERSIZE; - sense_addr = sqinfo->sense + ret; - sense_dma = sqinfo->sense_dma_addr + ret; + + if (work_mode) { + ret = ((pt_cmd->qid - 1) * SSSRAID_PTCMDS_PERQ + pt_cmd->cid) * + SCSI_SENSE_BUFFERSIZE; + sense_addr = sdioc->senses + ret; + sense_dma = sdioc->sense_dma_addr + ret; + } else { + ret = pt_cmd->cid * SCSI_SENSE_BUFFERSIZE; + sense_addr = sqinfo->sense + ret; + sense_dma = sqinfo->sense_dma_addr + ret; + } cmd->common.sense_addr = cpu_to_le64(sense_dma); cmd->common.sense_len = cpu_to_le16(SCSI_SENSE_BUFFERSIZE); @@ -1388,51 +1497,15 @@ static int sssraid_bsg_host_dispatch(struct bsg_job *job) return 0; } -static void sssraid_back_fault_cqe(struct sssraid_squeue *sqinfo, struct sssraid_completion *cqe) -{ - struct sssraid_ioc *sdioc = sqinfo->sdioc; - struct blk_mq_tags *tags; - struct scsi_cmnd *scmd; - struct sssraid_iod *iod; - struct request *req; - - tags = sdioc->shost->tag_set.tags[sqinfo->qidx - 1]; - req = blk_mq_tag_to_rq(tags, le16_to_cpu(cqe->cmd_id)); - if (unlikely(!req || !blk_mq_request_started(req))) - return; - - scmd = blk_mq_rq_to_pdu(req); - iod = scsi_cmd_priv(scmd); - - if (READ_ONCE(iod->state) != SSSRAID_CMDSTAT_FLIGHT && - READ_ONCE(iod->state) != SSSRAID_CMDSTAT_TIMEOUT) - return; - - WRITE_ONCE(iod->state, SSSRAID_CMDSTAT_TMO_COMPLETE); - set_host_byte(scmd, DID_NO_CONNECT); - if (iod->nsge) - scsi_dma_unmap(scmd); - sssraid_free_iod_res(sdioc, iod); - scmd->scsi_done(scmd); - ioc_warn(sdioc, "warn: back fault CQE, cid[%d] qidx[%d]\n", - le16_to_cpu(cqe->cmd_id), sqinfo->qidx); -} - +static void sssraid_drain_pending_ios(struct sssraid_ioc *sdioc); void sssraid_back_all_io(struct sssraid_ioc *sdioc) { int i, j; - struct sssraid_squeue *sqinfo; - struct sssraid_completion cqe = { 0 }; scsi_block_requests(sdioc->shost); - for (i = 1; i <= sdioc->shost->nr_hw_queues; i++) { - sqinfo = &sdioc->sqinfo[i]; - for (j = 0; j < sdioc->scsi_qd; j++) { - cqe.cmd_id = cpu_to_le16(j); - sssraid_back_fault_cqe(sqinfo, &cqe); - } - } + sssraid_drain_pending_ios(sdioc); + scsi_unblock_requests(sdioc->shost); @@ -1456,17 +1529,6 @@ void sssraid_back_all_io(struct sssraid_ioc *sdioc) } } -static int sssraid_get_first_sibling(unsigned int cpu) -{ - unsigned int ret; - - ret = cpumask_first(topology_sibling_cpumask(cpu)); - if (ret < nr_cpu_ids) - return ret; - - return cpu; -} - /* * static struct scsi_host_template sssraid_driver_template */ @@ -1495,15 +1557,12 @@ static int sssraid_bus_reset_handler(struct scsi_cmnd *scmd) /* eh_host_reset_handler call back */ static int sssraid_eh_host_reset(struct scsi_cmnd *scmd) { - u16 hwq, cid; struct sssraid_ioc *sdioc = shost_priv(scmd->device->host); - scsi_print_command(scmd); - if (sdioc->state != SSSRAID_LIVE || sssraid_check_scmd_completed(scmd)) + if (sdioc->state != SSSRAID_LIVE) return SUCCESS; - sssraid_get_tag_from_scmd(scmd, &hwq, &cid); - ioc_warn(sdioc, "warn: cid[%d] qidx[%d] host reset\n", cid, hwq); + ioc_warn(sdioc, "sdev[%d:%d] send host reset\n", scmd->device->channel, scmd->device->id); /* It's useless: * old code sssraid_reset_work_sync @@ -1515,11 +1574,13 @@ static int sssraid_eh_host_reset(struct scsi_cmnd *scmd) return FAILED; } if (sssraid_soft_reset_handler(sdioc)) { - ioc_warn(sdioc, "warn: cid[%d] qidx[%d] host reset failed\n", cid, hwq); + ioc_warn(sdioc, "warn: sdev[%d:%d] host reset failed\n", + scmd->device->channel, scmd->device->id); return FAILED; } - ioc_warn(sdioc, "cid[%d] qidx[%d] host reset success\n", cid, hwq); + ioc_warn(sdioc, "sdev[%d:%d] host reset success\n", scmd->device->channel, + scmd->device->id); return SUCCESS; } @@ -1550,11 +1611,10 @@ static int sssraid_map_queues(struct Scsi_Host *shost) struct blk_mq_tag_set *tag_set = &shost->tag_set; struct blk_mq_queue_map *queue_map = &tag_set->map[HCTX_TYPE_DEFAULT]; const struct cpumask *node_mask = NULL; - unsigned int queue_offset = queue_map->queue_offset; unsigned int *map = queue_map->mq_map; unsigned int nr_queues = queue_map->nr_queues; unsigned int node_id, node_id_last = 0xFFFFFFFF; - int cpu, first_sibling, cpu_index = 0; + int cpu, queue = 0; u8 node_count = 0, i; unsigned int node_id_array[100]; @@ -1588,16 +1648,8 @@ static int sssraid_map_queues(struct Scsi_Host *shost) node_mask = cpumask_of_node(node_id_array[i]); dbgprint(sdioc, "NUMA_node = %d\n", node_id_array[i]); for_each_cpu(cpu, node_mask) { - if (cpu_index < nr_queues) { - map[cpu_index++] = queue_offset + (cpu % nr_queues); - } else { - first_sibling = sssraid_get_first_sibling(cpu); - if (first_sibling == cpu) - map[cpu_index++] = queue_offset + (cpu % nr_queues); - else - map[cpu_index++] = map[first_sibling]; - } - dbgprint(sdioc, "map[%d] = %d\n", cpu_index - 1, map[cpu_index - 1]); + map[cpu] = (queue < nr_queues) ? queue++ : 0; + dbgprint(sdioc, "map[%d] = %d\n", cpu, map[cpu]); } } @@ -1611,7 +1663,7 @@ static int sssraid_qcmd(struct Scsi_Host *shost, struct sssraid_iod *iod = scsi_cmd_priv(scmd); struct sssraid_ioc *sdioc = shost_priv(shost); struct scsi_device *sdev = scmd->device; - struct sssraid_sdev_hostdata *hostdata = sdev->hostdata; + struct sssraid_sdev_hostdata *hostdata; u16 hwq, cid; struct sssraid_squeue *sq; struct sssraid_ioq_command ioq_cmd; @@ -1629,10 +1681,15 @@ static int sssraid_qcmd(struct Scsi_Host *shost, if (unlikely(sdioc->logging_level & SSSRAID_DEBUG)) scsi_print_command(scmd); - sssraid_get_tag_from_scmd(scmd, &hwq, &cid); hostdata = sdev->hostdata; + sssraid_get_tag_from_scmd(scmd, &hwq, sdioc, &cid, hostdata); sq = &sdioc->sqinfo[hwq]; + if (unlikely(atomic_inc_return(&sq->inflight) > (sdioc->ioq_depth - SSSRAID_PTCMDS_PERQ))) { + atomic_dec(&sq->inflight); + return SCSI_MLQUEUE_HOST_BUSY; + } + memset(&ioq_cmd, 0, sizeof(ioq_cmd)); ioq_cmd.rw.hdid = cpu_to_le32(hostdata->hdid); ioq_cmd.rw.command_id = cpu_to_le16(cid); @@ -1640,18 +1697,27 @@ static int sssraid_qcmd(struct Scsi_Host *shost, retval = sssraid_setup_ioq_cmd(sdioc, &ioq_cmd, scmd); if (unlikely(retval)) { set_host_byte(scmd, DID_ERROR); + atomic_dec(&sq->inflight); scmd->scsi_done(scmd); return 0; } - iod->sense = sq->sense + retval; - iod->sense_dma = sq->sense_dma_addr + retval; + retval = cid * SCSI_SENSE_BUFFERSIZE; + if (work_mode) { + iod->sense = sdioc->senses + retval; + iod->sense_dma = sdioc->sense_dma_addr + retval; + } else { + iod->sense = sq->sense + retval; + iod->sense_dma = sq->sense_dma_addr + retval; + } + ioq_cmd.common.sense_addr = cpu_to_le64(iod->sense_dma); ioq_cmd.common.sense_len = cpu_to_le16(SCSI_SENSE_BUFFERSIZE); sssraid_init_iod(iod); iod->sqinfo = sq; + iod->cid = cid; retval = sssraid_io_map_data(sdioc, iod, scmd, &ioq_cmd); if (unlikely(retval)) { ioc_err(sdioc, "err: io map data fail.\n"); @@ -1667,6 +1733,7 @@ static int sssraid_qcmd(struct Scsi_Host *shost, return 0; deinit_iod: + atomic_dec(&sq->inflight); sssraid_free_iod_res(sdioc, iod); return retval; } @@ -1691,6 +1758,11 @@ static int sssraid_slave_configure(struct scsi_device *sdev) timeout = scmd_tmout_rawdisk * HZ; max_sec = hostdata->max_io_kb << 1; qd = sssraid_get_qd_by_disk(hostdata->attr); + + if (sssraid_disk_is_hdd(hostdata->attr)) + hostdata->hwq = hostdata->hdid % (sdioc->init_done_queue_cnt - 1) + 1; + else + hostdata->hwq = 0; } else { ioc_err(sdioc, "err: scsi dev hostdata is null\n"); } @@ -1710,6 +1782,40 @@ static int sssraid_slave_configure(struct scsi_device *sdev) return 0; } +static bool sssraid_clean_pending_io(struct request *rq, void *data, bool reserved) +{ + struct sssraid_ioc *sdioc = data; + struct scsi_cmnd *scmd; + struct sssraid_iod *iod; + + if (unlikely(!rq || !blk_mq_request_started(rq))) + return true; + + scmd = blk_mq_rq_to_pdu(rq); + iod = scsi_cmd_priv(scmd); + + if ((cmpxchg(&iod->state, SSSRAID_CMDSTAT_FLIGHT, SSSRAID_CMDSTAT_COMPLETE) != + SSSRAID_CMDSTAT_FLIGHT) && + (cmpxchg(&iod->state, SSSRAID_CMDSTAT_TIMEOUT, SSSRAID_CMDSTAT_COMPLETE) != + SSSRAID_CMDSTAT_TIMEOUT)) + return true; + + set_host_byte(scmd, DID_NO_CONNECT); + if (iod->nsge) + scsi_dma_unmap(scmd); + sssraid_free_iod_res(sdioc, iod); + dev_warn_ratelimited(&sdioc->pdev->dev, "back unfinished CQE, cid[%d] qid[%d]\n", + iod->cid, iod->sqinfo->qidx); + scmd->scsi_done(scmd); + + return true; +} + +static void sssraid_drain_pending_ios(struct sssraid_ioc *sdioc) +{ + blk_mq_tagset_busy_iter(&sdioc->shost->tag_set, sssraid_clean_pending_io, (void *)(sdioc)); +} + /* slave_alloc call back */ static int sssraid_slave_alloc(struct scsi_device *sdev) { @@ -1787,7 +1893,8 @@ static int sssraid_abort_handler(struct scsi_cmnd *scmd) return SUCCESS; hostdata = scmd->device->hostdata; - sssraid_get_tag_from_scmd(scmd, &hwq, &cid); + cid = iod->cid; + hwq = iod->sqinfo->qidx; ioc_warn(sdioc, "warn: cid[%d] qidx[%d] timeout, aborting\n", cid, hwq); ret = sssraid_send_abort_cmd(sdioc, hostdata->hdid, hwq, cid); @@ -1866,11 +1973,47 @@ static ssize_t fw_version_show(struct device *cdev, struct device_attribute *att return snprintf(buf, PAGE_SIZE, "%s\n", sdioc->ctrl_info->fr); } +static +ssize_t hdd_dispatch_store(struct device *cdev, struct device_attribute *attr, + const char *buf, size_t count) +{ + int val = 0; + struct Scsi_Host *shost = class_to_shost(cdev); + struct sssraid_ioc *sdioc = shost_priv(shost); + + if (kstrtoint(buf, 0, &val) != 0) + return -EINVAL; + if (val < DISPATCH_BY_CPU || val > DISPATCH_BY_DISK) + return -EINVAL; + sdioc->hdd_dispatch = val; + + return strlen(buf); +} + +static ssize_t hdd_dispatch_show(struct device *cdev, struct device_attribute *attr, char *buf) +{ + struct Scsi_Host *shost = class_to_shost(cdev); + struct sssraid_ioc *sdioc = shost_priv(shost); + + return snprintf(buf, PAGE_SIZE, "%d\n", sdioc->hdd_dispatch); +} + +static ssize_t can_queue_count_show(struct device *dev, + struct device_attribute *attr, char *buf) +{ + struct Scsi_Host *shost = class_to_shost(dev); + struct blk_mq_tag_set *tag_set = &shost->tag_set; + + return snprintf(buf, 20, "%d\n", tag_set->nr_hw_queues + 1); +} + static DEVICE_ATTR_RO(csts_pp); static DEVICE_ATTR_RO(csts_shst); static DEVICE_ATTR_RO(csts_cfs); static DEVICE_ATTR_RO(csts_rdy); static DEVICE_ATTR_RO(fw_version); +static DEVICE_ATTR_RW(hdd_dispatch); +static DEVICE_ATTR_RO(can_queue_count); static struct device_attribute *sssraid_host_attrs[] = { &dev_attr_csts_pp, @@ -1878,6 +2021,8 @@ static struct device_attribute *sssraid_host_attrs[] = { &dev_attr_csts_cfs, &dev_attr_csts_rdy, &dev_attr_fw_version, + &dev_attr_hdd_dispatch, + &dev_attr_can_queue_count, NULL, }; @@ -2041,14 +2186,48 @@ static ssize_t raid_resync_show(struct device *dev, struct device_attribute *att return snprintf(buf, PAGE_SIZE, "%d\n", progress); } +static ssize_t dispatch_hwq_show(struct device *dev, struct device_attribute *attr, char *buf) +{ + struct sssraid_sdev_hostdata *hostdata; + + hostdata = to_scsi_device(dev)->hostdata; + return snprintf(buf, PAGE_SIZE, "%d\n", hostdata->hwq); +} + +static +ssize_t dispatch_hwq_store(struct device *dev, struct device_attribute *attr, + const char *buf, size_t count) +{ + int val; + struct sssraid_ioc *sdioc; + struct scsi_device *sdev; + struct sssraid_sdev_hostdata *hostdata; + + sdev = to_scsi_device(dev); + sdioc = shost_priv(sdev->host); + hostdata = sdev->hostdata; + + if (kstrtoint(buf, 0, &val) != 0) + return -EINVAL; + if (val <= 0 || val >= sdioc->init_done_queue_cnt) + return -EINVAL; + if (!sssraid_disk_is_hdd(hostdata->attr)) + return -EINVAL; + + hostdata->hwq = val; + return strlen(buf); +} + static DEVICE_ATTR_RO(raid_level); static DEVICE_ATTR_RO(raid_state); static DEVICE_ATTR_RO(raid_resync); +static DEVICE_ATTR_RW(dispatch_hwq); static struct device_attribute *sssraid_dev_attrs[] = { &dev_attr_raid_level, &dev_attr_raid_state, &dev_attr_raid_resync, + &dev_attr_dispatch_hwq, NULL, };