diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 31435446051d0a8f09bb7b4e4425d298b16db938..c40cb1d95f7433feb2485089243dd2075a2e73b8 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -16,11 +16,6 @@ enum sec_endian { SEC_64BE }; -enum hisi_sec_error_type { - /* negative compression */ - HSEC_NC_ERR = 0x0d, -}; - struct hisi_sec_ctrl; struct hisi_sec { @@ -28,6 +23,7 @@ struct hisi_sec { struct list_head list; struct hisi_sec_ctrl *ctrl; struct dma_pool *sgl_pool; + int ctx_q_num; }; struct hisi_sec *find_sec_device(int node); diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 7733d19957a74c7199f19a7165e2df444005137f..bbd86e73c101d13b99ae4632620a7724b186b4ae 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0+ #include #include +#include #include #include @@ -15,9 +16,9 @@ #define SEC_ASYNC #define SEC_INVLD_REQ_ID -1 -#define SEC_DEBUG_LOG +#define SEC_DEBUG -#ifdef SEC_DEBUG_LOG +#ifdef SEC_DEBUG #define dbg(msg, ...) pr_info(msg, ##__VA_ARGS__) #else #define dbg(msg, ...) @@ -35,14 +36,17 @@ struct hisi_sec_cipher_req { }; struct hisi_sec_ctx; +struct hisi_sec_qp_ctx; struct hisi_sec_req { struct hisi_sec_sqe sec_sqe; struct hisi_sec_ctx *ctx; + struct hisi_sec_qp_ctx *qp_ctx; struct skcipher_request *sk_req; struct hisi_sec_cipher_req c_req; int err; int req_id; + bool fake_busy; }; struct hisi_sec_cipher_ctx { @@ -53,13 +57,24 @@ struct hisi_sec_cipher_ctx { u8 c_key_len; }; -struct hisi_sec_ctx { +struct hisi_sec_qp_ctx { struct hisi_qp *qp; - struct hisi_sec *sec; - struct device *sec_dev; struct hisi_sec_req **req_list; unsigned long *req_bitmap; spinlock_t req_lock; + atomic_t req_cnt; +}; + +struct hisi_sec_ctx { + struct hisi_sec_qp_ctx *qp_ctx; + struct hisi_sec *sec; + struct device *sec_dev; + atomic_t thread_cnt; + int max_thread_cnt; + int req_fake_limit; + int req_limit; + int q_num; + atomic_t q_id; struct hisi_sec_cipher_ctx c_ctx; }; @@ -106,24 +121,31 @@ static void sec_sg_unmap(struct device *dev, acc_sg_buf_unmap(dev, sk_req->src, creq->c_in, creq->c_in_dma, pool); } -static int hisi_sec_alloc_req_id(struct hisi_sec_req *req) +static int hisi_sec_alloc_req_id(struct hisi_sec_req *req, + struct hisi_sec_qp_ctx *qp_ctx) { struct hisi_sec_ctx *ctx = req->ctx; int req_id; unsigned long flags; - spin_lock_irqsave(&ctx->req_lock, flags); - req_id = find_first_zero_bit(ctx->req_bitmap, QM_Q_DEPTH); - if (req_id >= QM_Q_DEPTH) { - spin_unlock_irqrestore(&ctx->req_lock, flags); + spin_lock_irqsave(&qp_ctx->req_lock, flags); + req_id = find_first_zero_bit(qp_ctx->req_bitmap, ctx->req_limit); + if (req_id >= ctx->req_limit) { + spin_unlock_irqrestore(&qp_ctx->req_lock, flags); + dump_data((uint8_t *)qp_ctx->req_bitmap, ctx->req_limit / 8); + pr_info("[%s][%d] used[%d]\n", __func__, __LINE__, + atomic_read(&qp_ctx->qp->qp_status.used)); dev_err(ctx->sec_dev, "no free req id\n"); - return -EBUSY; + pr_info("[%s][%d] max_thread_cnt[%d]\n", __func__, __LINE__, + ctx->max_thread_cnt); + return -ENOBUFS; } - set_bit(req_id, ctx->req_bitmap); - spin_unlock_irqrestore(&ctx->req_lock, flags); + set_bit(req_id, qp_ctx->req_bitmap); + spin_unlock_irqrestore(&qp_ctx->req_lock, flags); - ctx->req_list[req_id] = req; + qp_ctx->req_list[req_id] = req; req->req_id = req_id; + req->qp_ctx = qp_ctx; return 0; } @@ -132,6 +154,7 @@ static void hisi_sec_free_req_id(struct hisi_sec_req *req) { struct hisi_sec_ctx *ctx = req->ctx; int req_id = req->req_id; + struct hisi_sec_qp_ctx *qp_ctx = req->qp_ctx; unsigned long flags; if (req_id < 0) { @@ -140,45 +163,70 @@ static void hisi_sec_free_req_id(struct hisi_sec_req *req) } req->req_id = SEC_INVLD_REQ_ID; - ctx->req_list[req_id] = NULL; + qp_ctx->req_list[req_id] = NULL; - spin_lock_irqsave(&ctx->req_lock, flags); - bitmap_clear(ctx->req_bitmap, req_id, 1); - spin_unlock_irqrestore(&ctx->req_lock, flags); + spin_lock_irqsave(&qp_ctx->req_lock, flags); + clear_bit(req_id, qp_ctx->req_bitmap); + atomic_dec(&qp_ctx->req_cnt); + spin_unlock_irqrestore(&qp_ctx->req_lock, flags); } -static int hisi_sec_create_qp(struct hisi_qm *qm, struct hisi_sec_ctx *ctx, - int alg_type, int req_type) +static int hisi_sec_create_qp_ctx(struct hisi_qm *qm, struct hisi_sec_ctx *ctx, + int qp_ctx_id, int alg_type, int req_type) { struct hisi_qp *qp; + struct hisi_sec_qp_ctx *qp_ctx; int ret; qp = hisi_qm_create_qp(qm, alg_type); if (IS_ERR(qp)) return PTR_ERR(qp); + qp_ctx = &ctx->qp_ctx[qp_ctx_id]; qp->req_type = req_type; - qp->qp_ctx = ctx; + qp->qp_ctx = qp_ctx; #ifdef SEC_ASYNC qp->req_cb = sec_cipher_cb; #endif - ctx->qp = qp; + qp_ctx->qp = qp; + + spin_lock_init(&qp_ctx->req_lock); + atomic_set(&qp_ctx->req_cnt, 0); + + qp_ctx->req_bitmap = kcalloc(BITS_TO_LONGS(QM_Q_DEPTH), sizeof(long), + GFP_KERNEL); + if (!qp_ctx->req_bitmap) { + ret = -ENOMEM; + goto err_qm_release_qp; + } + + qp_ctx->req_list = kcalloc(QM_Q_DEPTH, sizeof(void *), GFP_KERNEL); + if (!qp_ctx->req_list) { + ret = -ENOMEM; + goto err_free_req_bitmap; + } ret = hisi_qm_start_qp(qp, 0); if (ret < 0) - goto err_qm_release_qp; + goto err_free_req_list; return 0; +err_free_req_list: + kfree(qp_ctx->req_list); +err_free_req_bitmap: + kfree(qp_ctx->req_bitmap); err_qm_release_qp: hisi_qm_release_qp(qp); return ret; } -static void hisi_sec_release_qp(struct hisi_sec_ctx *ctx) +static void hisi_sec_release_qp_ctx(struct hisi_sec_qp_ctx *qp_ctx) { - hisi_qm_stop_qp(ctx->qp); - hisi_qm_release_qp(ctx->qp); + hisi_qm_stop_qp(qp_ctx->qp); + kfree(qp_ctx->req_bitmap); + kfree(qp_ctx->req_list); + hisi_qm_release_qp(qp_ctx->qp); } static int __hisi_sec_ctx_init(struct hisi_sec_ctx *ctx, int qlen) @@ -186,17 +234,11 @@ static int __hisi_sec_ctx_init(struct hisi_sec_ctx *ctx, int qlen) if (!ctx || qlen < 0) return -EINVAL; - spin_lock_init(&ctx->req_lock); - ctx->req_bitmap = kcalloc(BITS_TO_LONGS(qlen), sizeof(long), - GFP_KERNEL); - if (!ctx->req_bitmap) - return -ENOMEM; - - ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); - if (!ctx->req_list) { - kfree(ctx->req_bitmap); - return -ENOMEM; - } + ctx->req_limit = qlen; + ctx->req_fake_limit = qlen / 2; + atomic_set(&ctx->thread_cnt, 0); + ctx->max_thread_cnt = 0; + atomic_set(&ctx->q_id, 0); return 0; } @@ -207,7 +249,7 @@ static int hisi_sec_cipher_ctx_init(struct crypto_skcipher *tfm) struct hisi_qm *qm; struct hisi_sec_cipher_ctx *c_ctx; struct hisi_sec *sec; - int ret; + int i, ret; crypto_skcipher_set_reqsize(tfm, sizeof(struct hisi_sec_req)); @@ -221,9 +263,17 @@ static int hisi_sec_cipher_ctx_init(struct crypto_skcipher *tfm) qm = &sec->qm; ctx->sec_dev = &qm->pdev->dev; - ret = hisi_sec_create_qp(qm, ctx, 0, 0); - if (ret) - return ret; + ctx->q_num = sec->ctx_q_num; + ctx->qp_ctx = kcalloc(ctx->q_num, sizeof(struct hisi_sec_qp_ctx), + GFP_KERNEL); + if (!ctx->qp_ctx) + return -ENOMEM; + + for (i = 0; i < ctx->q_num; i++) { + ret = hisi_sec_create_qp_ctx(qm, ctx, i, 0, 0); + if (ret) + goto err_sec_release_qp_ctx; + } c_ctx = &ctx->c_ctx; c_ctx->c_key = dma_alloc_coherent(ctx->sec_dev, @@ -231,13 +281,17 @@ static int hisi_sec_cipher_ctx_init(struct crypto_skcipher *tfm) if (!ctx->c_ctx.c_key) { ret = -ENOMEM; - goto err_sec_release_qp; + goto err_sec_release_qp_ctx; } return __hisi_sec_ctx_init(ctx, QM_Q_DEPTH); -err_sec_release_qp: - hisi_sec_release_qp(ctx); +err_sec_release_qp_ctx: + i = i - 1; + for (; i >= 0; i--) + hisi_sec_release_qp_ctx(&ctx->qp_ctx[i]); + + kfree(ctx->qp_ctx); return ret; } @@ -245,6 +299,7 @@ static void hisi_sec_cipher_ctx_exit(struct crypto_skcipher *tfm) { struct hisi_sec_ctx *ctx = crypto_skcipher_ctx(tfm); struct hisi_sec_cipher_ctx *c_ctx; + int i = 0; c_ctx = &ctx->c_ctx; @@ -254,13 +309,10 @@ static void hisi_sec_cipher_ctx_exit(struct crypto_skcipher *tfm) c_ctx->c_key = NULL; } - kfree(ctx->req_bitmap); - ctx->req_bitmap = NULL; + for (i = 0; i < ctx->q_num; i++) + hisi_sec_release_qp_ctx(&ctx->qp_ctx[i]); - kfree(ctx->req_list); - ctx->req_list = NULL; - - hisi_sec_release_qp(ctx); + kfree(ctx->qp_ctx); } static int sec_alloc_cipher_req(struct hisi_sec_req *req) @@ -298,12 +350,13 @@ static void sec_cipher_cb(struct hisi_qp *qp, void *resp) { struct hisi_sec_sqe *sec_sqe = (struct hisi_sec_sqe *)resp; u32 req_id = sec_sqe->type2.tag; - struct hisi_sec_ctx *ctx = qp->qp_ctx; - struct dma_pool *pool = ctx->sec->sgl_pool; + struct hisi_sec_qp_ctx *qp_ctx = qp->qp_ctx; + struct dma_pool *pool; struct hisi_sec_req *req; int ret = 0; - req = ctx->req_list[req_id]; + req = qp_ctx->req_list[req_id]; + pool = req->ctx->sec->sgl_pool; if (sec_sqe->type2.done != 0x1 || sec_sqe->type2.flag != 0x2) { ret = sec_sqe->type2.error_type; @@ -318,6 +371,11 @@ static void sec_cipher_cb(struct hisi_qp *qp, void *resp) hisi_sec_free_req_id(req); + if (req->fake_busy) { + req->sk_req->base.complete(&req->sk_req->base, -EINPROGRESS); + req->fake_busy = 0; + } + req->sk_req->base.complete(&req->sk_req->base, ret); } @@ -459,14 +517,22 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, struct hisi_sec_ctx *ctx = crypto_skcipher_ctx(atfm); struct hisi_sec_req *req = skcipher_request_ctx(sk_req); struct hisi_sec_sqe *sec_sqe = &req->sec_sqe; - struct hisi_qp *qp = ctx->qp; - struct device *dev = &qp->qm->pdev->dev; + struct device *dev = ctx->sec_dev; struct hisi_sec_cipher_req *c_req = &req->c_req; struct dma_pool *pool = ctx->sec->sgl_pool; + struct hisi_sec_qp_ctx *qp_ctx; + unsigned long flags; + int req_cnt; + int thread_cnt; + int issue_id; if (!sk_req->src || !sk_req->dst || !sk_req->cryptlen) return -EINVAL; + thread_cnt = atomic_inc_return(&ctx->thread_cnt); + if (thread_cnt > ctx->max_thread_cnt) + ctx->max_thread_cnt = thread_cnt; + req->sk_req = sk_req; req->ctx = ctx; @@ -475,6 +541,7 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, ret = sec_alloc_cipher_req(req); if (ret) { dev_err(dev, "sec alloc cipher request failed\n"); + atomic_dec(&ctx->thread_cnt); return ret; } @@ -512,35 +579,64 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, } else memcpy(c_req->c_ivin, sk_req->iv, crypto_skcipher_ivsize(atfm)); + /* get issue_id */ + issue_id = atomic_fetch_inc(&ctx->q_id) % ctx->q_num; + if (issue_id % ctx->q_num == 0 && ctx->q_id.counter > ctx->q_num) + atomic_sub(ctx->q_num, &ctx->q_id); + + qp_ctx = &ctx->qp_ctx[issue_id]; #ifdef SEC_ASYNC - ret = hisi_sec_alloc_req_id(req); + ret = hisi_sec_alloc_req_id(req, qp_ctx); if (ret) { dev_err(dev, "sec alloc req id failed\n"); goto err_unmap_dst_sg; } + + req->fake_busy = 0; + + req_cnt = atomic_inc_return(&qp_ctx->req_cnt); + if (req_cnt >= ctx->req_fake_limit) + req->fake_busy = 1; + sec_sqe->type2.tag = req->req_id; #endif + spin_lock_irqsave(&qp_ctx->req_lock, flags); + ret = hisi_qp_send(qp_ctx->qp, sec_sqe); + spin_unlock_irqrestore(&qp_ctx->req_lock, flags); - ret = hisi_qp_send(qp, sec_sqe); if (ret < 0) { - dev_err(dev, "hisi_qp_send failed\n"); +#ifdef SEC_ASYNC + if (ret == -EBUSY) + ret = -ENOBUFS; + goto err_free_req_id; +#else goto err_unmap_dst_sg; +#endif } #ifdef SEC_ASYNC - ret = -EINPROGRESS; + if (req_cnt >= ctx->req_fake_limit) + ret = -EBUSY; + else + ret = -EINPROGRESS; #else - ret = hisi_qp_wait(qp); + ret = hisi_qp_wait(qp_ctx->qp); if (ret < 0) goto err_unmap_dst_sg; sec_update_iv(req, sk_req->iv); sec_sg_unmap(dev, sk_req, c_req, pool); + sec_free_cipher_req(req); #endif + atomic_dec(&ctx->thread_cnt); return ret; +#ifdef SEC_ASYNC +err_free_req_id: + hisi_sec_free_req_id(req); +#endif err_unmap_dst_sg: if (sk_req->dst != sk_req->src) acc_sg_buf_unmap(dev, sk_req->dst, @@ -551,6 +647,7 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, err_free_cipher_req: sec_free_cipher_req(req); + atomic_dec(&ctx->thread_cnt); return ret; } diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 94144e151e3440c6a4c8f01a116e8e2a569b3705..7e5c66f07c2a37577955863b8dc54296164e3aee 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -339,15 +339,15 @@ static struct debugfs_reg32 hsec_dfx_regs[] = { {"HSEC_ECC_1BIT_INFO ", 0x301C04}, {"HSEC_ECC_2BIT_CNT ", 0x301C10}, {"HSEC_ECC_2BIT_INFO ", 0x301C14}, - {"HSEC_ECC_BD_SAA0 ", 0x301C20}, - {"HSEC_ECC_BD_SAA1 ", 0x301C24}, - {"HSEC_ECC_BD_SAA2 ", 0x301C28}, - {"HSEC_ECC_BD_SAA3 ", 0x301C2C}, - {"HSEC_ECC_BD_SAA4 ", 0x301C30}, - {"HSEC_ECC_BD_SAA5 ", 0x301C34}, - {"HSEC_ECC_BD_SAA6 ", 0x301C38}, - {"HSEC_ECC_BD_SAA7 ", 0x301C3C}, - {"HSEC_ECC_BD_SAA8 ", 0x301C40}, + {"HSEC_BD_SAA0 ", 0x301C20}, + {"HSEC_BD_SAA1 ", 0x301C24}, + {"HSEC_BD_SAA2 ", 0x301C28}, + {"HSEC_BD_SAA3 ", 0x301C2C}, + {"HSEC_BD_SAA4 ", 0x301C30}, + {"HSEC_BD_SAA5 ", 0x301C34}, + {"HSEC_BD_SAA6 ", 0x301C38}, + {"HSEC_BD_SAA7 ", 0x301C3C}, + {"HSEC_BD_SAA8 ", 0x301C40}, }; static int pf_q_num_set(const char *val, const struct kernel_param *kp) @@ -402,6 +402,9 @@ module_param(uacce_mode, int, 0444); static int enable_sm4_ctr; module_param(enable_sm4_ctr, int, 0444); +static int ctx_q_num = 64; +module_param(ctx_q_num, int, 0444); + static const struct pci_device_id hisi_sec_dev_ids[] = { { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_SEC_PF) }, { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, PCI_DEVICE_ID_SEC_VF) }, @@ -839,6 +842,8 @@ static int hisi_sec_probe(struct pci_dev *pdev, const struct pci_device_id *id) if (!hisi_sec->sgl_pool) return -ENOMEM; + hisi_sec->ctx_q_num = ctx_q_num; + qm = &hisi_sec->qm; qm->pdev = pdev; qm->ver = rev_id; diff --git a/drivers/crypto/hisilicon/sec2/sec_usr_if.h b/drivers/crypto/hisilicon/sec2/sec_usr_if.h index 55ab65158eca29eddd813d07997f4219c35ede35..451a243a225cc900c7f23b1b0a8dc62e08f14428 100644 --- a/drivers/crypto/hisilicon/sec2/sec_usr_if.h +++ b/drivers/crypto/hisilicon/sec2/sec_usr_if.h @@ -188,8 +188,4 @@ enum C_ALG { SM4 = 0x3, }; -#define SEG_MAX_SGE_SIZE (8 * 1024 * 1024) -#define SEC_MAX_SGE_NUM 255 -#define SEC_MAX_SGL_NUM 256 - #endif