From f1ae75f143e2b1a6b3f756f2fad9118bba3c7d4b Mon Sep 17 00:00:00 2001 From: xuzaibo Date: Sat, 17 Nov 2018 11:39:10 +0800 Subject: [PATCH] Add Hi1620 ZIP&HPRE drivers with common QM driver, which are adpated to Crypto and SPIMDEV. Feature or Bugfix:Feature Signed-off-by: xuzaibo --- drivers/crypto/hisilicon/Kconfig | 33 + drivers/crypto/hisilicon/Makefile | 3 + drivers/crypto/hisilicon/hpre/Makefile | 2 + drivers/crypto/hisilicon/hpre/hpre.h | 64 + drivers/crypto/hisilicon/hpre/hpre_crypto.c | 1059 +++++++++++++++ drivers/crypto/hisilicon/hpre/hpre_main.c | 287 ++++ drivers/crypto/hisilicon/qm.c | 1300 +++++++++++++++++++ drivers/crypto/hisilicon/qm.h | 132 ++ drivers/crypto/hisilicon/zip/Makefile | 2 + drivers/crypto/hisilicon/zip/zip.h | 57 + drivers/crypto/hisilicon/zip/zip_crypto.c | 357 +++++ drivers/crypto/hisilicon/zip/zip_crypto.h | 8 + drivers/crypto/hisilicon/zip/zip_main.c | 202 +++ 13 files changed, 3506 insertions(+) create mode 100644 drivers/crypto/hisilicon/hpre/Makefile create mode 100644 drivers/crypto/hisilicon/hpre/hpre.h create mode 100644 drivers/crypto/hisilicon/hpre/hpre_crypto.c create mode 100644 drivers/crypto/hisilicon/hpre/hpre_main.c create mode 100644 drivers/crypto/hisilicon/qm.c create mode 100644 drivers/crypto/hisilicon/qm.h create mode 100644 drivers/crypto/hisilicon/zip/Makefile create mode 100644 drivers/crypto/hisilicon/zip/zip.h create mode 100644 drivers/crypto/hisilicon/zip/zip_crypto.c create mode 100644 drivers/crypto/hisilicon/zip/zip_crypto.h create mode 100644 drivers/crypto/hisilicon/zip/zip_main.c diff --git a/drivers/crypto/hisilicon/Kconfig b/drivers/crypto/hisilicon/Kconfig index 8ca9c503bcb0..a1dae67fd6ec 100644 --- a/drivers/crypto/hisilicon/Kconfig +++ b/drivers/crypto/hisilicon/Kconfig @@ -12,3 +12,36 @@ config CRYPTO_DEV_HISI_SEC To compile this as a module, choose M here: the module will be called hisi_sec. + +config CRYPTO_DEV_HISILICON + tristate "Support for HISILICON CRYPTO ACCELERATOR" + help + Enable this to use Hisilicon Hardware Accelerators + +config CRYPTO_DEV_HISI_SPIMDEV + bool "Enable SPIMDEV interface" + depends on CRYPTO_DEV_HISILICON + select VFIO_SPIMDEV + help + Enable this enable the SPIMDEV, "shared parent IOMMU Mediated Device" + interface for all Hisilicon accelerators if they can. The SPIMDEV + enable the WarpDrive user space accelerator driver to access the + hardware function directly. + +config CRYPTO_DEV_HISI_QM + tristate + depends on ARM64 && PCI + +config CRYPTO_DEV_HISI_ZIP + tristate "Support for HISI ZIP Driver" + depends on ARM64 && CRYPTO_DEV_HISILICON + select CRYPTO_DEV_HISI_QM + help + Support for HiSilicon HIP08 ZIP Driver + +config CRYPTO_DEV_HISI_HPRE + tristate "Support for HISI HPRE Driver" + depends on ARM64 && CRYPTO_DEV_HISILICON + select CRYPTO_DEV_HISI_QM + help + Support for HiSilicon HIP08 HPRE Driver \ No newline at end of file diff --git a/drivers/crypto/hisilicon/Makefile b/drivers/crypto/hisilicon/Makefile index 463f46ace182..254db49d3d2c 100644 --- a/drivers/crypto/hisilicon/Makefile +++ b/drivers/crypto/hisilicon/Makefile @@ -1,2 +1,5 @@ # SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += qm.o +obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/ +obj-$(CONFIG_CRYPTO_DEV_HISI_HPRE) += hpre/ obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += sec/ diff --git a/drivers/crypto/hisilicon/hpre/Makefile b/drivers/crypto/hisilicon/hpre/Makefile new file mode 100644 index 000000000000..4fd32b789e1e --- /dev/null +++ b/drivers/crypto/hisilicon/hpre/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_HISI_HPRE) += hisi_hpre.o +hisi_hpre-objs = hpre_main.o hpre_crypto.o diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h new file mode 100644 index 000000000000..8435aeb9ec66 --- /dev/null +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -0,0 +1,64 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef __HISI_HPRE_H +#define __HISI_HPRE_H + +#include +#include "../qm.h" + +#define HPRE_SQE_SIZE 64 +#define HPRE_SQ_SIZE (HPRE_SQE_SIZE * QM_Q_DEPTH) +#define QM_CQ_SIZE (QM_CQE_SIZE * QM_Q_DEPTH) +#define HPRE_PF_DEF_Q_NUM 64 +#define HPRE_PF_DEF_Q_BASE 0 + +struct hisi_hpre { + struct qm_info qm; + struct list_head list; + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV + struct vfio_spimdev *spimdev; +#endif +}; + +enum hisi_hpre_alg_type { + HPRE_ALG_NC_NCRT = 0x0, + HPRE_ALG_NC_CRT = 0x1, + HPRE_ALG_KG_STD = 0x2, + HPRE_ALG_KG_CRT = 0x3, + HPRE_ALG_DH_G2 = 0x4, + HPRE_ALG_DH = 0x5, + HPRE_ALG_PRIME = 0x6, + HPRE_ALG_MOD = 0x7, + HPRE_ALG_MOD_INV = 0x8, + HPRE_ALG_MUL = 0x9, + HPRE_ALG_COPRIME = 0xA +}; + +struct hisi_hpre_sqe { + __u32 alg : 5; + + /* error type */ + __u32 etype :11; + __u32 resv0 : 14; + __u32 done : 2; + __u32 task_len1 : 8; + __u32 task_len2 : 8; + __u32 mrttest_num : 8; + __u32 resv1 : 8; + __u32 low_key; + __u32 hi_key; + __u32 low_in; + __u32 hi_in; + __u32 low_out; + __u32 hi_out; + __u32 tag :16; + __u32 resv2 :16; + __u32 rsvd1[7]; +}; + +extern struct list_head hisi_hpre_list; + +extern int hpre_algs_register(void); +extern void hpre_algs_unregister(void); + +#endif diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c new file mode 100644 index 000000000000..04946afaf8b0 --- /dev/null +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -0,0 +1,1059 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hpre.h" + +static DEFINE_MUTEX(algs_lock); +static unsigned int active_devs; + +struct hpre_ctx; + +#define GET_DEV(ctx) ((ctx)->qp->qm->pdev->dev) + +#define BITS64_MERGE(low_32, high_32) (((u64)(low_32)) | \ + (((u64)(high_32)) << 32)) +typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); + +struct _rsa_ctx { + /* low address: e--->n */ + char *pubkey; + + /* low address: d--->n */ + char *prikey; + + /* low address: dq->dp->q->p->qinv */ + char *crt_prikey; + dma_addr_t dma_pubkey; + dma_addr_t dma_prikey; + dma_addr_t dma_crt_prikey; + struct crypto_akcipher *soft_tfm; +}; + +struct _dh_ctx { + /* + * If base is g we compute the public key + * ya = g^xa mod p; [RFC2631 sec 2.1.1] + * else if base if the counterpart public key we + * compute the shared secret + * ZZ = yb^xa mod p; [RFC2631 sec 2.1.1] + */ + char *xa_p; /* low address: d--->n */ + char *g; /* m */ + dma_addr_t dma_xa_p; + dma_addr_t dma_g; +}; + +struct hpre_ctx { + struct hisi_qp *qp; + struct hpre_asym_request **req_list; + unsigned long *req_bitmap; + spinlock_t req_lock; + unsigned int key_sz; + bool crt_g2_mode; + union { + struct _rsa_ctx rsa; + struct _dh_ctx dh; + u64 resv[7]; + }; +}; + +struct hpre_asym_request { + char *src_align; + char *dst_align; + struct hisi_hpre_sqe req; + struct hpre_ctx *ctx; + union { + struct akcipher_request *rsa; + struct kpp_request *dh; + } areq; + int err; + int req_id; + hpre_cb cb; +}; + +void hpre_bn_format(unsigned char *buf, int len) +{ + int i = len - 1, j; + + + while (!buf[i] && i >= 0) + i--; + if (i == len - 1) + return; + for (j = len - 1; j >= 0; j--, i--) { + if (i >= 0) + buf[j] = buf[i]; + else + buf[j] = 0; + } +} + +static struct hisi_hpre *find_hpre_device(int node) +{ + struct hisi_hpre *hisi_hpre, *ret = NULL; + struct device *dev; + int min_distance = 100; + int dev_node = 0; + + list_for_each_entry(hisi_hpre, &hisi_hpre_list, list) { + dev = &hisi_hpre->qm.pdev->dev; +#ifdef CONFIG_NUMA + dev_node = dev->numa_node; +#endif + if (node_distance(dev_node, node) < min_distance) { + ret = hisi_hpre; + min_distance = node_distance(dev_node, node); + } + } + + return ret; +} + +static int hpre_alloc_req_id(struct hpre_ctx *ctx) +{ + int id; + unsigned long flags; + + spin_lock_irqsave(&ctx->req_lock, flags); + id = find_first_zero_bit(ctx->req_bitmap, QM_Q_DEPTH); + if (id >= QM_Q_DEPTH) { + spin_unlock_irqrestore(&ctx->req_lock, flags); + pr_err("\nno free req id!"); + return -EBUSY; + } + set_bit(id, ctx->req_bitmap); + spin_unlock_irqrestore(&ctx->req_lock, flags); + + return id; +} + +static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id) +{ + unsigned long flags; + + spin_lock_irqsave(&ctx->req_lock, flags); + bitmap_clear(ctx->req_bitmap, req_id, 1); + spin_unlock_irqrestore(&ctx->req_lock, flags); +} + +static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req) +{ + struct hpre_ctx *ctx; + int id; + + ctx = hpre_req->ctx; + id = hpre_alloc_req_id(ctx); + if (id < 0) + return -EINVAL; + + ctx->req_list[id] = hpre_req; + hpre_req->req_id = id; + + return id; +} + +static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req) +{ + int id = hpre_req->req_id; + struct hpre_ctx *ctx = hpre_req->ctx; + + ctx->req_list[id] = NULL; + hpre_free_req_id(ctx, id); +} + +static struct hisi_qp *hpre_get_qp(void) +{ + struct hisi_qp *qp = NULL; + struct hisi_hpre *hpre; + int ret; + + /* find the proper hpre device */ + hpre = find_hpre_device(cpu_to_node(smp_processor_id())); + if (!hpre) { + pr_err("Can not find proper hpre device!\n"); + return ERR_PTR(-ENODEV); + } + qp = hisi_qm_create_qp(&hpre->qm, 0); + if (!qp) { + dev_err(&hpre->qm.pdev->dev, "Can not create qp!\n"); + return ERR_PTR(-ENODEV); + } + ret = hisi_qm_start_qp(qp, 0); + if (ret) { + hisi_qm_release_qp(qp); + dev_err(&hpre->qm.pdev->dev, "Can not start qp!\n"); + return ERR_PTR(-EINVAL); + } + + return qp; +} + +static int _hw_data_init(struct hpre_asym_request *hpre_req, + struct scatterlist *data, unsigned int len, + int is_src) +{ + struct hisi_hpre_sqe *msg = &hpre_req->req; + struct hpre_ctx *ctx = hpre_req->ctx; + struct device *dev = &GET_DEV(ctx); + enum dma_data_direction dma_dir; + dma_addr_t tmp; + char *ptr; + + if (sg_is_last(data) && len == ctx->key_sz) { + if (is_src) { + hpre_req->src_align = NULL; + dma_dir = DMA_TO_DEVICE; + } else { + hpre_req->dst_align = NULL; + dma_dir = DMA_FROM_DEVICE; + } + tmp = dma_map_single(dev, sg_virt(data), + len, dma_dir); + if (unlikely(dma_mapping_error(dev, tmp))) { + dev_err(dev, "\ndma map data err!"); + return -ENOMEM; + } + } else { + int shift = ctx->key_sz - len; + + ptr = dma_zalloc_coherent(dev, ctx->key_sz, &tmp, GFP_KERNEL); + if (unlikely(!ptr)) { + dev_err(dev, "\ndma alloc data err!"); + return -ENOMEM; + } + if (is_src) { + scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0); + hpre_req->src_align = ptr; + } else { + hpre_req->dst_align = ptr; + } + } + if (is_src) { + msg->low_in = lower_32_bits(tmp); + msg->hi_in = upper_32_bits(tmp); + } else { + msg->low_out = lower_32_bits(tmp); + msg->hi_out = upper_32_bits(tmp); + } + + return 0; +} + +static void _hw_data_clr_all(struct hpre_ctx *ctx, + struct hpre_asym_request *req, + struct scatterlist *dst, struct scatterlist *src) +{ + dma_addr_t tmp; + struct device *dev = &GET_DEV(ctx); + struct hisi_hpre_sqe *sqe = &req->req; + + tmp = BITS64_MERGE(sqe->low_in, sqe->hi_in); + if (src && tmp) { + if (req->src_align) + dma_free_coherent(dev, ctx->key_sz, + req->src_align, tmp); + else + dma_unmap_single(dev, tmp, + ctx->key_sz, DMA_TO_DEVICE); + } + tmp = BITS64_MERGE(sqe->low_out, sqe->hi_out); + if (req->dst_align && tmp) { + if (dst) + scatterwalk_map_and_copy(req->dst_align, dst, 0, + ctx->key_sz, 1); + dma_free_coherent(dev, ctx->key_sz, req->dst_align, tmp); + } else { + dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE); + } +} + +static int _alg_res_post_hf(struct hpre_ctx *ctx, struct hisi_hpre_sqe *sqe, + void **kreq) +{ + struct hpre_asym_request *req; + int err, id; + + id = (int)sqe->tag; + req = ctx->req_list[id]; + hpre_rm_req_from_ctx(req); + *kreq = req; + err = sqe->etype; + err = (err == 0 && sqe->done == 3) ? 0 : -EINVAL; + + return err; +} + +static int _ctx_init(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) +{ + int ret = -ENOMEM; + + if (!ctx || !qp || qlen < 0) + return -EINVAL; + + spin_lock_init(&ctx->req_lock); + ctx->req_bitmap = kcalloc(BITS_TO_LONGS(qlen), sizeof(long), + GFP_KERNEL); + if (!ctx->req_bitmap) + return ret; + ctx->qp = qp; + ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL); + if (!ctx->req_list) { + kfree(ctx->req_bitmap); + return ret; + } + ctx->key_sz = 0; + ctx->crt_g2_mode = false; + + return 0; +} + +static void _ctx_clear(struct hpre_ctx *ctx, int is_exit) +{ + if (is_exit) { + kfree(ctx->req_bitmap); + kfree(ctx->req_list); + hisi_qm_release_qp(ctx->qp); + } + + ctx->crt_g2_mode = false; + ctx->key_sz = 0; +} + +static void _dh_cb(struct hpre_ctx *ctx, void *resp) +{ + struct kpp_request *areq; + struct hpre_asym_request *req; + int ret; + + ret = _alg_res_post_hf(ctx, resp, (void **)&req); + areq = req->areq.dh; + areq->dst_len = ctx->key_sz; + _hw_data_clr_all(ctx, req, areq->dst, areq->src); + kpp_request_complete(areq, ret); +} + +void hpre_alg_cb(struct hisi_qp *qp, void *_resp) +{ + struct hisi_hpre_sqe *sqe = _resp; + struct hpre_asym_request *areq; + struct hpre_ctx *ctx = qp->qp_ctx; + int id = (int)sqe->tag; + + areq = ctx->req_list[id]; + areq->cb(ctx, _resp); +} + +static int hpre_ctx_init(struct hpre_ctx *ctx) +{ + struct hisi_qp *qp; + + qp = hpre_get_qp(); + if (IS_ERR(qp)) + return PTR_ERR(qp); + qp->qp_ctx = ctx; + qp->req_cb = hpre_alg_cb; + + return _ctx_init(ctx, qp, QM_Q_DEPTH); +} + +static int hpre_dh_compute_value(struct kpp_request *req) +{ + struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct hpre_asym_request *hpre_req = + PTR_ALIGN(kpp_request_ctx(req), 64); + struct hisi_hpre_sqe *msg = &hpre_req->req; + int ret = -ENOMEM, ctr = 0, req_id; + + if (unlikely(!ctx->dh.xa_p)) + return -EINVAL; + + if (req->dst_len < ctx->key_sz) { + req->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + memset(msg, '\0', sizeof(*msg)); + + msg->done = 1; + hpre_req->cb = _dh_cb; + hpre_req->ctx = ctx; + hpre_req->areq.dh = req; + msg->task_len1 = (ctx->key_sz >> 3) - 1; + + msg->low_key = lower_32_bits(ctx->dh.dma_xa_p); + msg->hi_key = upper_32_bits(ctx->dh.dma_xa_p); + if (!req->src && !ctx->crt_g2_mode) { + msg->low_in = lower_32_bits(ctx->dh.dma_g); + msg->hi_in = upper_32_bits(ctx->dh.dma_g); + } + + if (req->src) { + (void)hpre_bn_format(sg_virt(req->src), ctx->key_sz); + ret = _hw_data_init(hpre_req, req->src, req->src_len, 1); + if (ret) + return ret; + } + ret = _hw_data_init(hpre_req, req->dst, req->dst_len, 0); + if (ret) { + _hw_data_clr_all(ctx, hpre_req, NULL, req->src); + return ret; + } + if (ctx->crt_g2_mode && !req->src) + msg->alg = HPRE_ALG_DH_G2; + else + msg->alg = HPRE_ALG_DH; + req_id = hpre_add_req_to_ctx(hpre_req); + if (req_id < 0) { + ret = -EBUSY; + goto clear_all; + } + msg->tag = (u16)req_id; + + do { + ret = hisi_qp_send(ctx->qp, (void *)msg); + } while (ret == -EBUSY && ctr++ < 100); + + if (!ret) + return -EINPROGRESS; + + hpre_rm_req_from_ctx(hpre_req); +clear_all: + _hw_data_clr_all(ctx, hpre_req, NULL, req->src); + return ret; +} + +static int hpre_dh_check_params_length(unsigned int key_sz) +{ + switch (key_sz) { + case 768: + case 1024: + case 1536: + case 2048: + case 3072: + case 4096: + return 0; + } + return -EINVAL; +} + +static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) +{ + struct device *dev = &GET_DEV(ctx); + unsigned int sz; + + if (hpre_dh_check_params_length(params->p_size << 3)) + return -EINVAL; + + sz = ctx->key_sz = params->p_size; + ctx->dh.xa_p = dma_zalloc_coherent(dev, sz << 1, + &ctx->dh.dma_xa_p, GFP_KERNEL); + if (!ctx->dh.xa_p) + return -ENOMEM; + memcpy(ctx->dh.xa_p + sz, params->p, sz); + hpre_bn_format((unsigned char *)ctx->dh.xa_p + sz, sz); + + /* If g equals 2 don't copy it */ + if (params->g_size == 1 && *(char *)params->g == 0x02) { + ctx->crt_g2_mode = true; + return 0; + } + + ctx->dh.g = dma_zalloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL); + if (!ctx->dh.g) + return -ENOMEM; + memcpy(ctx->dh.g + (sz - params->g_size), params->g, + params->g_size); + hpre_bn_format(ctx->dh.g, ctx->key_sz); + + return 0; +} + +static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, int is_exit) +{ + unsigned int sz = ctx->key_sz; + struct device *dev = &GET_DEV(ctx); + + if (ctx->dh.g) { + dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g); + ctx->dh.g = NULL; + } + if (ctx->dh.xa_p) { + dma_free_coherent(dev, sz << 1, ctx->dh.xa_p, + ctx->dh.dma_xa_p); + ctx->dh.xa_p = NULL; + } + _ctx_clear(ctx, is_exit); +} + +static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf, + unsigned int len) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + struct dh params; + int ret; + + if (crypto_dh_decode_key(buf, len, ¶ms) < 0) + return -EINVAL; + + /* Free old secret if any */ + hpre_dh_clear_ctx(ctx, 0); + + ret = hpre_dh_set_params(ctx, ¶ms); + if (ret < 0) + goto err_clear_ctx; + + memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key, + params.key_size); + hpre_bn_format((unsigned char *)ctx->dh.xa_p, ctx->key_sz); + + return 0; + +err_clear_ctx: + hpre_dh_clear_ctx(ctx, 0); + return ret; +} + +static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + return ctx->key_sz; +} + +static int hpre_dh_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + return hpre_ctx_init(ctx); +} + +static void hpre_dh_exit_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + hpre_dh_clear_ctx(ctx, 1); +} + +static void _rsa_cb(struct hpre_ctx *ctx, void *resp) +{ + struct akcipher_request *areq; + struct hpre_asym_request *req; + int ret; + + ret = _alg_res_post_hf(ctx, resp, (void **)&req); + areq = req->areq.rsa; + areq->dst_len = ctx->key_sz; + _hw_data_clr_all(ctx, req, areq->dst, areq->src); + akcipher_request_complete(areq, ret); +} + +static unsigned long hpre_rsa_key_size_check(unsigned int len) +{ + unsigned int bitslen = len << 3; + + switch (bitslen) { + /* 512bits is not supported by HPRE now! */ + case 512: + case 1024: + case 2048: + case 3072: + case 4096: + return 0; + default: + return -1; + }; +} + +static int hpre_rsa_enc(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + struct hpre_asym_request *hpre_req = + PTR_ALIGN(akcipher_request_ctx(req), 64); + struct hisi_hpre_sqe *msg = &hpre_req->req; + int ret = -ENOMEM, ctr = 0, req_id; + + if (ctx->key_sz == 64 && ctx->rsa.soft_tfm) { + akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); + ret = crypto_akcipher_encrypt(req); + akcipher_request_set_tfm(req, tfm); + return ret; + } + if (unlikely(!ctx->rsa.pubkey)) + return -EINVAL; + + if (req->dst_len < ctx->key_sz) { + req->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + memset(msg, '\0', sizeof(*msg)); + msg->done = 1; + msg->task_len1 = (ctx->key_sz >> 3) - 1; + msg->low_key = lower_32_bits(ctx->rsa.dma_pubkey); + msg->hi_key = upper_32_bits(ctx->rsa.dma_pubkey); + hpre_req->cb = _rsa_cb; + hpre_req->ctx = ctx; + hpre_req->areq.rsa = req; + ret = _hw_data_init(hpre_req, req->src, req->src_len, 1); + if (ret) + return ret; + ret = _hw_data_init(hpre_req, req->dst, req->dst_len, 0); + if (ret) { + _hw_data_clr_all(ctx, hpre_req, NULL, req->src); + return ret; + } + msg->alg = HPRE_ALG_NC_NCRT; + req_id = hpre_add_req_to_ctx(hpre_req); + if (req_id < 0) { + ret = -EBUSY; + goto clear_all; + } + msg->tag = (u16)req_id; + do { + ret = hisi_qp_send(ctx->qp, (void *)msg); + } while (ret == -EBUSY && ctr++ < 100); + + if (!ret) + return -EINPROGRESS; + + hpre_rm_req_from_ctx(hpre_req); +clear_all: + _hw_data_clr_all(ctx, hpre_req, NULL, req->src); + return ret; +} + +static int hpre_rsa_dec(struct akcipher_request *req) +{ + struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + struct hpre_asym_request *hpre_req = + PTR_ALIGN(akcipher_request_ctx(req), 64); + struct hisi_hpre_sqe *msg = &hpre_req->req; + int ret = -ENOMEM, ctr = 0, req_id; + + if (ctx->key_sz == 64 && ctx->rsa.soft_tfm) { + akcipher_request_set_tfm(req, ctx->rsa.soft_tfm); + ret = crypto_akcipher_decrypt(req); + akcipher_request_set_tfm(req, tfm); + return ret; + } + if (unlikely(!ctx->rsa.prikey)) + return -EINVAL; + + if (req->dst_len < ctx->key_sz) { + req->dst_len = ctx->key_sz; + return -EOVERFLOW; + } + memset(msg, '\0', sizeof(*msg)); + msg->task_len1 = (ctx->key_sz >> 3) - 1; + msg->done = 1; + if (ctx->crt_g2_mode) { + msg->low_key = lower_32_bits(ctx->rsa.dma_crt_prikey); + msg->hi_key = upper_32_bits(ctx->rsa.dma_crt_prikey); + } else { + msg->low_key = lower_32_bits(ctx->rsa.dma_prikey); + msg->hi_key = upper_32_bits(ctx->rsa.dma_prikey); + } + + hpre_req->cb = _rsa_cb; + hpre_req->ctx = ctx; + hpre_req->areq.rsa = req; + ret = _hw_data_init(hpre_req, req->src, req->src_len, 1); + if (ret) + return ret; + ret = _hw_data_init(hpre_req, req->dst, req->dst_len, 0); + if (ret) { + _hw_data_clr_all(ctx, hpre_req, NULL, req->src); + return ret; + } + if (ctx->crt_g2_mode) + msg->alg = HPRE_ALG_NC_CRT; + else + msg->alg = HPRE_ALG_NC_NCRT; + req_id = hpre_add_req_to_ctx(hpre_req); + if (req_id < 0) { + ret = -EBUSY; + goto clear_all; + } + msg->tag = (u16)req_id; + do { + ret = hisi_qp_send(ctx->qp, (void *)msg); + } while (ret == -EBUSY && ctr++ < 100); + + if (!ret) + return -EINPROGRESS; + + hpre_rm_req_from_ctx(hpre_req); +clear_all: + _hw_data_clr_all(ctx, hpre_req, NULL, req->src); + return ret; +} + +static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, + size_t vlen, bool private) +{ + const char *ptr = value; + int ret = -EINVAL; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + ctx->key_sz = vlen; + + /* invalid key size provided */ + if (hpre_rsa_key_size_check(ctx->key_sz)) + goto err; + if (private) { + ctx->rsa.prikey = dma_zalloc_coherent(&GET_DEV(ctx), + vlen << 1, + &ctx->rsa.dma_prikey, + GFP_KERNEL); + if (!ctx->rsa.prikey) + return -ENOMEM; + } + ctx->rsa.pubkey = dma_zalloc_coherent(&GET_DEV(ctx), + vlen << 1, + &ctx->rsa.dma_pubkey, + GFP_KERNEL); + if (!ctx->rsa.pubkey) + return -ENOMEM; + memcpy(ctx->rsa.pubkey + vlen, ptr, vlen); + hpre_bn_format((unsigned char *)ctx->rsa.pubkey + vlen, vlen); + if (ctx->rsa.prikey) { + memcpy(ctx->rsa.prikey + vlen, ptr, vlen); + hpre_bn_format((unsigned char *)ctx->rsa.prikey + vlen, vlen); + } + + return 0; +err: + ctx->key_sz = 0; + + return ret; +} + +static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value, + size_t vlen) +{ + const char *ptr = value; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) { + ctx->rsa.pubkey = NULL; + return -EINVAL; + } + + memcpy(ctx->rsa.pubkey, ptr, vlen); + hpre_bn_format((unsigned char *)ctx->rsa.pubkey, ctx->key_sz); + + return 0; +} + +static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value, + size_t vlen) +{ + const char *ptr = value; + int ret = -EINVAL; + + while (!*ptr && vlen) { + ptr++; + vlen--; + } + if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) + goto err; + + memcpy(ctx->rsa.prikey, ptr, vlen); + hpre_bn_format((unsigned char *)ctx->rsa.prikey, ctx->key_sz); + return 0; +err: + ctx->rsa.prikey = NULL; + return ret; +} + +static void hpre_rsa_drop_leading_zeros(const char **ptr, unsigned int *len) +{ + while (!**ptr && *len) { + (*ptr)++; + (*len)--; + } +} + +static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key) +{ + struct device *dev = &GET_DEV(ctx); + unsigned int half_key_sz = ctx->key_sz / 2; + const char *ptr; + unsigned int len; + int ret = -EINVAL; + + ctx->rsa.crt_prikey = dma_zalloc_coherent(dev, half_key_sz * 5, + &ctx->rsa.dma_crt_prikey, + GFP_KERNEL); + if (!ctx->rsa.crt_prikey) + return -ENOMEM; + + /* dq */ + ptr = rsa_key->dq; + len = rsa_key->dq_sz; + hpre_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_key; + + memcpy(ctx->rsa.crt_prikey, ptr, len); + hpre_bn_format((unsigned char *)ctx->rsa.crt_prikey, half_key_sz); + + /* dp */ + ptr = rsa_key->dp; + len = rsa_key->dp_sz; + hpre_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_key; + memcpy(ctx->rsa.crt_prikey + half_key_sz, ptr, len); + hpre_bn_format((unsigned char *)ctx->rsa.crt_prikey + half_key_sz, + half_key_sz); + + /* q */ + ptr = rsa_key->q; + len = rsa_key->q_sz; + hpre_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_key; + memcpy(ctx->rsa.crt_prikey + (half_key_sz * 2), ptr, len); + hpre_bn_format((unsigned char *)ctx->rsa.crt_prikey + half_key_sz * 2, + half_key_sz); + + /* p */ + ptr = rsa_key->p; + len = rsa_key->p_sz; + hpre_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_key; + memcpy(ctx->rsa.crt_prikey + half_key_sz * 3, ptr, len); + hpre_bn_format((unsigned char *)ctx->rsa.crt_prikey + half_key_sz * 3, + half_key_sz); + + /* qinv */ + ptr = rsa_key->qinv; + len = rsa_key->qinv_sz; + hpre_rsa_drop_leading_zeros(&ptr, &len); + if (!len) + goto free_key; + memcpy(ctx->rsa.crt_prikey + (half_key_sz * 4), ptr, len); + hpre_bn_format((unsigned char *)ctx->rsa.crt_prikey + half_key_sz * 4, + half_key_sz); + ctx->crt_g2_mode = true; + + return 0; +free_key: + memset(ctx->rsa.crt_prikey + half_key_sz * 5, '\0', half_key_sz); + dma_free_coherent(dev, half_key_sz * 5, ctx->rsa.crt_prikey, + ctx->rsa.dma_crt_prikey); + ctx->rsa.crt_prikey = NULL; + + ctx->crt_g2_mode = false; + + return ret; +} + +static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, int is_exit) +{ + unsigned int half_key_sz = ctx->key_sz >> 1; + struct device *dev = &GET_DEV(ctx); + + if (ctx->rsa.pubkey) { + dma_free_coherent(dev, ctx->key_sz << 1, + ctx->rsa.pubkey, ctx->rsa.dma_pubkey); + ctx->rsa.pubkey = NULL; + } + if (ctx->rsa.crt_prikey) { + memset(ctx->rsa.crt_prikey, '\0', half_key_sz * 5); + dma_free_coherent(dev, half_key_sz * 5, + ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey); + ctx->rsa.crt_prikey = NULL; + } + if (ctx->rsa.prikey) { + memset(ctx->rsa.prikey, '\0', ctx->key_sz); + dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey, + ctx->rsa.dma_prikey); + ctx->rsa.prikey = NULL; + } + + _ctx_clear(ctx, is_exit); +} + +static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key, + unsigned int keylen, bool private) +{ + struct rsa_key rsa_key; + int ret; + + hpre_rsa_clear_ctx(ctx, 0); + + if (private) + ret = rsa_parse_priv_key(&rsa_key, key, keylen); + else + ret = rsa_parse_pub_key(&rsa_key, key, keylen); + if (ret < 0) + goto free; + + ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private); + if (ret < 0) + goto free; + + if (private) { + ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz); + if (ret < 0) + goto free; + hpre_rsa_setkey_crt(ctx, &rsa_key); + } + ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz); + if (ret < 0) + goto free; + + if (!ctx->rsa.pubkey) { + /* invalid key provided */ + ret = -EINVAL; + goto free; + } + if (private && !ctx->rsa.prikey) { + /* invalid private key provided */ + ret = -EINVAL; + goto free; + } + + return 0; +free: + hpre_rsa_clear_ctx(ctx, 0); + return ret; +} + +static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen); + if (ret) + return ret; + return hpre_rsa_setkey(ctx, key, keylen, false); +} + +static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key, + unsigned int keylen) +{ + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + int ret; + + ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen); + if (ret) + return ret; + return hpre_rsa_setkey(ctx, key, keylen, true); +} + +static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm) +{ + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + + if (ctx->rsa.soft_tfm && ctx->key_sz == 64) + return crypto_akcipher_maxsize(ctx->rsa.soft_tfm); + return ctx->key_sz; +} + +static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm) +{ + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + + ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0); + if (IS_ERR(ctx->rsa.soft_tfm)) { + pr_err("Can not alloc_akcipher!\n"); + return PTR_ERR(tfm); + } + + return hpre_ctx_init(ctx); +} + +static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm) +{ + struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm); + + hpre_rsa_clear_ctx(ctx, 1); + if (ctx->rsa.soft_tfm) + crypto_free_akcipher(ctx->rsa.soft_tfm); +} + +static struct akcipher_alg rsa = { + .encrypt = hpre_rsa_enc, + .decrypt = hpre_rsa_dec, + .sign = hpre_rsa_dec, + .verify = hpre_rsa_enc, + .set_pub_key = hpre_rsa_setpubkey, + .set_priv_key = hpre_rsa_setprivkey, + .max_size = hpre_rsa_max_size, + .init = hpre_rsa_init_tfm, + .exit = hpre_rsa_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + 64, + .base = { + .cra_name = "rsa", + .cra_driver_name = "hpre-rsa", + .cra_priority = 1000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct hpre_ctx), + }, +}; + +static struct kpp_alg dh = { + .set_secret = hpre_dh_set_secret, + .generate_public_key = hpre_dh_compute_value, + .compute_shared_secret = hpre_dh_compute_value, + .max_size = hpre_dh_max_size, + .init = hpre_dh_init_tfm, + .exit = hpre_dh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + 64, + .base = { + .cra_name = "dh", + .cra_driver_name = "hpre-dh", + .cra_priority = 1000, + .cra_module = THIS_MODULE, + .cra_ctxsize = sizeof(struct hpre_ctx), + }, +}; + +int hpre_algs_register(void) +{ + int ret = 0; + + mutex_lock(&algs_lock); + if (++active_devs == 1) { + rsa.base.cra_flags = 0; + ret = crypto_register_akcipher(&rsa); + if (ret) + goto unlock; + ret = crypto_register_kpp(&dh); + } +unlock: + mutex_unlock(&algs_lock); + return ret; +} + +void hpre_algs_unregister(void) +{ + mutex_lock(&algs_lock); + if (--active_devs == 0) { + crypto_unregister_akcipher(&rsa); + crypto_unregister_kpp(&dh); + } + mutex_unlock(&algs_lock); +} diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c new file mode 100644 index 000000000000..3b7385a570b1 --- /dev/null +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include +#include +#include +#include "hpre.h" + +#define HPRE_VF_NUM 63 +#define HPRE_QUEUE_NUM_V1 4096 +#define HPRE_QUEUE_NUM_V2 1024 + +#define HPRE_COMM_CNT_CLR_CE 0x0 +#define HPRE_FSM_MAX_CNT 0x301008 +#define HPRE_VFG_AXQOS 0x30100c +#define HPRE_VFG_AXCACHE 0x301010 +#define HPRE_RDCHN_INI_CFG 0x301014 +#define HPRE_BD_ENDIAN 0x301020 +#define HPRE_ECC_BYPASS 0x301024 +#define HPRE_POISON_BYPASS 0x30102c +#define HPRE_ARUSR_CFG 0x301030 +#define HPRE_AWUSR_CFG 0x301034 +#define HPRE_INT_MASK 0x301400 +#define HPRE_RAS_ECC_1BIT_TH 0x30140c + +#define HPRE_TYPES_ENB 0x301038 +#define HPRE_PORT_ARCA_CHE_0 0x301040 +#define HPRE_PORT_ARCA_CHE_1 0x301044 +#define HPRE_PORT_AWCA_CHE_0 0x301060 +#define HPRE_PORT_AWCA_CHE_1 0x301064 + +#define HPRE_BD_RUSER_32_63 0x301110 +#define HPRE_SGL_RUSER_32_63 0x30111c +#define HPRE_DATA_RUSER_32_63 0x301128 +#define HPRE_DATA_WUSER_32_63 0x301134 +#define HPRE_BD_WUSER_32_63 0x301140 +#define HPRE_RDCHN_INI_ST 0x301a00 +#define HPRE_CORE_ENB 0x302004 +#define HPRE_CORE_INI_CFG 0x302020 +#define HPRE_CORE_INI_STATUS 0x302080 + + +LIST_HEAD(hisi_hpre_list); +DEFINE_MUTEX(hisi_hpre_list_lock); + +static const struct pci_device_id hisi_hpre_dev_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa258) }, + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa259) }, + { 0, } +}; + + +static inline void hisi_hpre_add_to_list(struct hisi_hpre *hisi_hpre) +{ + mutex_lock(&hisi_hpre_list_lock); + list_add_tail(&hisi_hpre->list, &hisi_hpre_list); + mutex_unlock(&hisi_hpre_list_lock); +} + +static inline void hisi_hpre_remove_from_list(struct hisi_hpre *hisi_hpre) +{ + mutex_lock(&hisi_hpre_list_lock); + list_del(&hisi_hpre->list); + mutex_unlock(&hisi_hpre_list_lock); +} + +static int hisi_hpre_set_user_domain_and_cache(struct hisi_hpre *hisi_hpre) +{ + int ret; + u32 val; + + writel(0x1, hisi_hpre->qm.io_base + HPRE_TYPES_ENB); + writel(0x0, hisi_hpre->qm.io_base + HPRE_VFG_AXQOS); + writel(0xff, hisi_hpre->qm.io_base + HPRE_VFG_AXCACHE); + writel(0x0, hisi_hpre->qm.io_base + HPRE_BD_ENDIAN); + writel(0x0, hisi_hpre->qm.io_base + HPRE_INT_MASK); + writel(0x0, hisi_hpre->qm.io_base + HPRE_RAS_ECC_1BIT_TH); + writel(0x0, hisi_hpre->qm.io_base + HPRE_POISON_BYPASS); + writel(0x0, hisi_hpre->qm.io_base + HPRE_COMM_CNT_CLR_CE); + writel(0x0, hisi_hpre->qm.io_base + HPRE_ECC_BYPASS); +#ifndef CONFIG_ARM_SMMU_V3 + writel(0x1, hisi_hpre->qm.io_base + HPRE_ARUSR_CFG); + writel(0x1, hisi_hpre->qm.io_base + HPRE_AWUSR_CFG); +#else + writel(0x203, hisi_hpre->qm.io_base + HPRE_ARUSR_CFG); + writel(0x203, hisi_hpre->qm.io_base + HPRE_AWUSR_CFG); +#endif + writel(0x1, hisi_hpre->qm.io_base + HPRE_RDCHN_INI_CFG); + ret = readl_relaxed_poll_timeout(hisi_hpre->qm.io_base + + HPRE_RDCHN_INI_ST, val, val & BIT(0), 10, 1000); + if (ret) { + pr_err("\nHPRE:INI ST TIMEOUT"); + return -ETIMEDOUT; + } + /* First cluster initiating */ + writel(0xf, hisi_hpre->qm.io_base + HPRE_CORE_ENB); + writel(0x1, hisi_hpre->qm.io_base + HPRE_CORE_INI_CFG); + ret = readl_relaxed_poll_timeout(hisi_hpre->qm.io_base + + HPRE_CORE_INI_STATUS, + val, ((val & 0xf) == 0xf), 10, 1000); + if (ret) { + pr_err("\nHPRE:CLUSTER 1 INI ST STATUS timeout"); + return -ETIMEDOUT; + } + /* Second cluster initiating, reg's address is 0x1000 more*/ + /* writel(0xf, hpre->io_base + 0x1000 + HPRE_CORE_ENB);*/ + writel(0x0, hisi_hpre->qm.io_base + 0x1000 + HPRE_CORE_ENB); + writel(0x1, hisi_hpre->qm.io_base + 0x1000 + HPRE_CORE_INI_CFG); + ret = readl_relaxed_poll_timeout(hisi_hpre->qm.io_base + 0x1000 + + HPRE_CORE_INI_STATUS, + val, ((val & 0xf) == 0xf), 10, 1000); + if (ret) { + pr_err("\nHPRE:CLUSTER 2 INI ST STATUS timeout"); + return -ETIMEDOUT; + } + + return ret; +} + +static int hisi_hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct hisi_hpre *hisi_hpre; + struct qm_info *qm; + int ret; + u8 rev_id = 0; + +#ifdef CONFIG_ARM_SMMU_V3 + u32 val; +#endif + + hisi_hpre = devm_kzalloc(&pdev->dev, sizeof(*hisi_hpre), GFP_KERNEL); + if (!hisi_hpre) + return -ENOMEM; + hisi_hpre_add_to_list(hisi_hpre); + + qm = &hisi_hpre->qm; + qm->pdev = pdev; + pci_read_config_byte(pdev, PCI_REVISION_ID, &rev_id); + if (rev_id == 0x20) + qm->ver = QM_HW_V1; + else if (rev_id == 0x21) + qm->ver = QM_HW_V2; + qm->sqe_size = HPRE_SQE_SIZE; + ret = hisi_qm_init(qm, HPRE); + if (ret) + goto err_with_hisi_hpre; + +#define HPRE_ADDR(offset) QM_ADDR(qm, offset) + + if (pdev->is_physfn) { + /* user domain */ + writel(0x40000070, HPRE_ADDR(QM_ARUSER_M_CFG_1)); + writel(0x007ffffc, HPRE_ADDR(QM_ARUSER_M_CFG_ENABLE)); + writel(0x40000070, HPRE_ADDR(QM_AWUSER_M_CFG_1)); + writel(0x007ffffc, HPRE_ADDR(QM_AWUSER_M_CFG_ENABLE)); + writel(0x00000001, HPRE_ADDR(QM_WUSER_M_CFG_ENABLE)); + writel(0x1833, HPRE_ADDR(QM_CACHE_CTL)); + writel(0x00400001, HPRE_ADDR(QM_PEH_AXUSER_CFG)); +#ifdef CONFIG_ARM_SMMU_V3 + writel(0x40000070, HPRE_ADDR(QM_ARUSER_M_CFG_1)); + writel(0xfffffffe, HPRE_ADDR(QM_ARUSER_M_CFG_ENABLE)); + writel(0x40000070, HPRE_ADDR(QM_AWUSER_M_CFG_1)); + writel(0xfffffffe, HPRE_ADDR(QM_AWUSER_M_CFG_ENABLE)); + + val = readl_relaxed(HPRE_ADDR(QM_ARUSER_M_CFG_1)); + val &= ~GENMASK(14, 12); + val |= (1 << 12); + writel(val, HPRE_ADDR(QM_ARUSER_M_CFG_1)); + + val = readl_relaxed(HPRE_ADDR(QM_AWUSER_M_CFG_1)); + val &= ~GENMASK(14, 12); + val |= (1 << 12); + writel(val, HPRE_ADDR(QM_AWUSER_M_CFG_1)); + + val = readl_relaxed(HPRE_ADDR(QM_ARUSER_M_CFG_ENABLE)); + val &= ~0x1; + writel(val, HPRE_ADDR(QM_ARUSER_M_CFG_ENABLE)); + + val = readl_relaxed(HPRE_ADDR(QM_AWUSER_M_CFG_ENABLE)); + val &= ~0x1; + writel(val, HPRE_ADDR(QM_AWUSER_M_CFG_ENABLE)); +#endif + /* cache */ + writel_relaxed(0x0303, /* 0xffff IT */ + HPRE_ADDR(QM_AXI_M_CFG)); + writel_relaxed(0xf, + HPRE_ADDR(QM_AXI_M_CFG_ENABLE)); + writel_relaxed(0x7f, + HPRE_ADDR(QM_PEH_AXUSER_CFG_ENABLE)); +#ifdef CONFIG_ARM_SMMU_V3 + writel_relaxed(0xffff, + HPRE_ADDR(QM_AXI_M_CFG)); + writel_relaxed(0xffffffff, + HPRE_ADDR(QM_AXI_M_CFG_ENABLE)); + writel_relaxed(0xffffffff, + HPRE_ADDR(QM_PEH_AXUSER_CFG_ENABLE)); +#endif + ret = hisi_qm_mem_start(qm); + if (ret) + goto err_with_qm_init; + ret = hisi_hpre_set_user_domain_and_cache(hisi_hpre); + if (ret) + return ret; + qm->qp_base = HPRE_PF_DEF_Q_BASE; + qm->qp_num = HPRE_PF_DEF_Q_NUM; + qm->free_qp = qm->qp_num; + } + + ret = hisi_qm_start(qm); + if (ret) + goto err_with_qm_init; + + /* todo: exception irq handler register, ES did not support */ + + return 0; + +err_with_qm_init: + hisi_qm_uninit(qm); + +err_with_hisi_hpre: + hisi_hpre_remove_from_list(hisi_hpre); + kfree(hisi_hpre); + + return ret; +} + +static void hisi_hpre_remove(struct pci_dev *pdev) +{ + struct hisi_hpre *hisi_hpre = pci_get_drvdata(pdev); + struct qm_info *qm = &hisi_hpre->qm; + + hisi_qm_stop(qm); + hisi_qm_uninit(qm); + hisi_hpre_remove_from_list(hisi_hpre); + kfree(hisi_hpre); +} + +static int hisi_hpre_pci_sriov_configure(struct pci_dev *pdev, int num_vfs) +{ + /* todo: set queue number for VFs */ + + return 0; +} + +static struct pci_driver hisi_hpre_pci_driver = { + .name = "hisi_hpre", + .id_table = hisi_hpre_dev_ids, + .probe = hisi_hpre_probe, + .remove = hisi_hpre_remove, + .sriov_configure = hisi_hpre_pci_sriov_configure +}; + +static int __init hisi_hpre_init(void) +{ + int ret; + + ret = pci_register_driver(&hisi_hpre_pci_driver); + if (ret < 0) { + pr_err("hpre: can't register hisi hpre driver.\n"); + return ret; + } + + ret = hpre_algs_register(); + if (ret < 0) { + pr_err("hpre: can't register hisi hpre to crypto.\n"); + pci_unregister_driver(&hisi_hpre_pci_driver); + return ret; + } + + return 0; +} + +static void __exit hisi_hpre_exit(void) +{ + hpre_algs_unregister(); + pci_unregister_driver(&hisi_hpre_pci_driver); +} + +module_init(hisi_hpre_init); +module_exit(hisi_hpre_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Zaibo Xu "); +MODULE_DESCRIPTION("Driver for HiSilicon HPRE accelerator"); +MODULE_DEVICE_TABLE(pci, hisi_hpre_dev_ids); diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c new file mode 100644 index 000000000000..68a79759cd84 --- /dev/null +++ b/drivers/crypto/hisilicon/qm.c @@ -0,0 +1,1300 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include +#include +#include "qm.h" + +#define QM_DEF_Q_NUM 128 + +/* eq/aeq irq enable */ +#define QM_VF_AEQ_INT_SOURCE 0x0 +#define QM_VF_AEQ_INT_MASK 0x4 +#define QM_VF_EQ_INT_SOURCE 0x8 +#define QM_VF_EQ_INT_MASK 0xc + +/* mailbox */ +#define MAILBOX_CMD_SQC 0x0 +#define MAILBOX_CMD_CQC 0x1 +#define MAILBOX_CMD_EQC 0x2 +#define MAILBOX_CMD_SQC_BT 0x4 +#define MAILBOX_CMD_CQC_BT 0x5 + +#define MAILBOX_CMD_SEND_BASE 0x300 +#define MAILBOX_EVENT_SHIFT 8 +#define MAILBOX_STATUS_SHIFT 9 +#define MAILBOX_BUSY_SHIFT 13 +#define MAILBOX_OP_SHIFT 14 +#define MAILBOX_QUEUE_SHIFT 16 + +/* sqc shift */ +#define SQ_HEAD_SHIFT 0 +#define SQ_TAIL_SHIFI 16 +#define SQ_HOP_NUM_SHIFT 0 +#define SQ_PAGE_SIZE_SHIFT 4 +#define SQ_BUF_SIZE_SHIFT 8 +#define SQ_SQE_SIZE_SHIFT 12 +#define SQ_HEAD_IDX_SIG_SHIFT 0 +#define SQ_TAIL_IDX_SIG_SHIFT 0 +#define SQ_CQN_SHIFT 0 +#define SQ_PRIORITY_SHIFT 0 +#define SQ_ORDERS_SHIFT 4 +#define SQ_TYPE_SHIFT 8 + +#define SQ_TYPE_MASK 0xf + +/* cqc shift */ +#define CQ_HEAD_SHIFT 0 +#define CQ_TAIL_SHIFI 16 +#define CQ_HOP_NUM_SHIFT 0 +#define CQ_PAGE_SIZE_SHIFT 4 +#define CQ_BUF_SIZE_SHIFT 8 +#define CQ_SQE_SIZE_SHIFT 12 +#define CQ_PASID 0 +#define CQ_HEAD_IDX_SIG_SHIFT 0 +#define CQ_TAIL_IDX_SIG_SHIFT 0 +#define CQ_CQN_SHIFT 0 +#define CQ_PRIORITY_SHIFT 16 +#define CQ_ORDERS_SHIFT 0 +#define CQ_TYPE_SHIFT 0 +#define CQ_PHASE_SHIFT 0 +#define CQ_FLAG_SHIFT 1 + +#define CQC_HEAD_INDEX(cqc) ((cqc)->cq_head) +#define CQC_PHASE(cqc) (((cqc)->dw6) & 0x1) +#define CQC_CQ_ADDRESS(cqc) (((u64)((cqc)->cq_base_h) << 32) | \ + ((cqc)->cq_base_l)) +#define CQC_PHASE_BIT 0x1 + +/* eqc shift */ +#define MB_EQC_EQE_SHIFT 12 +#define MB_EQC_PHASE_SHIFT 16 + +#define EQC_HEAD_INDEX(eqc) ((eqc)->eq_head) +#define EQC_TAIL_INDEX(eqc) ((eqc)->eq_tail) +#define EQC_PHASE(eqc) ((((eqc)->dw6) >> 16) & 0x1) + +#define EQC_PHASE_BIT 0x00010000 + +/* cqe shift */ +#define CQE_PHASE(cqe) ((cqe)->w7 & 0x1) +#define CQE_SQ_NUM(cqe) ((cqe)->sq_num) +#define CQE_SQ_HEAD_INDEX(cqe) ((cqe)->sq_head) + +/* eqe shift */ +#define EQE_PHASE(eqe) (((eqe)->dw0 >> 16) & 0x1) +#define EQE_CQN(eqe) (((eqe)->dw0) & 0xffff) + +#define QM_EQE_CQN_MASK 0xffff + +/* doorbell */ +#define DOORBELL_CMD_SQ 0 +#define DOORBELL_CMD_CQ 1 +#define DOORBELL_CMD_EQ 2 +#define DOORBELL_CMD_AEQ 3 + +#define DOORBELL_CMD_SEND_BASE_V1 0x340 +#define DOORBELL_CMD_SEND_BASE_V2 0x1000 +#define QM_MEM_START_INIT 0x100040 +#define QM_MEM_INIT_DONE 0x100044 +#define QM_VFT_CFG_RDY 0x10006c +#define QM_VFT_CFG_OP_WR 0x100058 +#define QM_VFT_CFG_TYPE 0x10005c +#define QM_SQC_VFT 0x0 +#define QM_CQC_VFT 0x1 +#define QM_VFT_CFG_ADDRESS 0x100060 +#define QM_VFT_CFG_OP_ENABLE 0x100054 + +#define QM_VFT_CFG_DATA_L 0x100064 +#define QM_VFT_CFG_DATA_H 0x100068 +#define QM_SQC_VFT_BUF_SIZE (7ULL << 8) +#define QM_SQC_VFT_SQC_SIZE (5ULL << 12) +#define QM_SQC_VFT_INDEX_NUMBER (1ULL << 16) +#define QM_SQC_VFT_BT_INDEX_SHIFT 22 +#define QM_SQC_VFT_START_SQN_SHIFT 28 +#define QM_SQC_VFT_VALID (1ULL << 44) +#define QM_SQC_VFT_SQN_SHIFT 45 +#define QM_CQC_VFT_BUF_SIZE (7ULL << 8) +#define QM_CQC_VFT_SQC_SIZE (5ULL << 12) +#define QM_CQC_VFT_INDEX_NUMBER (1ULL << 16) +#define QM_CQC_VFT_BT_INDEX_SHIFT 22 +#define QM_CQC_VFT_VALID (1ULL << 28) + +struct cqe { + __le32 rsvd0; + __le16 cmd_id; + __le16 rsvd1; + __le16 sq_head; + __le16 sq_num; + __le16 rsvd2; + __le16 w7; +}; + +struct eqe { + __le32 dw0; +}; + +struct sqc { + __le16 head; + __le16 tail; + __le32 base_l; + __le32 base_h; + __le32 dw3; + __le16 w8; + __le16 rsvd0; + __le16 pasid; + __le16 w11; + __le16 cq_num; + __le16 w13; + __le32 rsvd1; +}; + +struct cqc { + __le16 head; + __le16 tail; + __le32 base_l; + __le32 base_h; + __le32 dw3; + __le16 w8; + __le16 rsvd0; + __le16 pasid; + __le16 w11; + __le32 dw6; + __le32 rsvd1; +}; + +#define INIT_QC(qc, base) do { \ + (qc)->head = 0; \ + (qc)->tail = 0; \ + (qc)->base_l = lower_32_bits(base); \ + (qc)->base_h = upper_32_bits(base); \ + (qc)->pasid = 0; \ + (qc)->w11 = 0; \ + (qc)->rsvd1 = 0; \ +} while (0) + +struct eqc { + __le16 head; + __le16 tail; + __le32 base_l; + __le32 base_h; + __le32 dw3; + __le32 rsvd[2]; + __le32 dw6; +}; + +struct mailbox { + __le16 w0; + __le16 queue_num; + __le32 base_l; + __le32 base_h; + __le32 rsvd; +}; + +struct doorbell { + __le16 queue_num; + __le16 cmd; + __le16 index; + __le16 priority; +}; + +#define QM_DMA_BUF(p, buf) ((struct buf *)(p)->buf.addr) +#define QM_SQC(p) QM_DMA_BUF(p, sqc) +#define QM_CQC(p) QM_DMA_BUF(p, cqc) +#define QM_EQC(p) QM_DMA_BUF(p, eqc) +#define QM_EQE(p) QM_DMA_BUF(p, eqe) + +#define QP_SQE_DMA(qp) ((qp)->scqe.dma) +#define QP_CQE(qp) ((struct cqe *)((qp)->scqe.addr + \ + qp->qm->sqe_size * QM_Q_DEPTH)) +#define QP_CQE_DMA(qp) ((qp)->scqe.dma + qp->qm->sqe_size * QM_Q_DEPTH) + +static inline void qm_writel(struct qm_info *qm, u32 val, u32 offset) +{ + writel(val, qm->io_base + offset); +} + +struct qm_info; + +struct hisi_acc_qm_hw_ops { + int (*vft_config)(struct qm_info *qm, u16 base, u32 number); + int (*qm_start_qp)(struct hisi_qp *qp, unsigned long arg); + void (*qm_db)(struct qm_info *qm, u16 qn, + u8 cmd, u16 index, u8 priority); +}; + +static inline int hacc_qm_mb_is_busy(struct qm_info *qm) +{ + u32 val; + + return readl_relaxed_poll_timeout(QM_ADDR(qm, MAILBOX_CMD_SEND_BASE), + val, !((val >> MAILBOX_BUSY_SHIFT) & 0x1), 10, 1000); +} + +static inline void qm_mb_write(struct qm_info *qm, void *src) +{ + void __iomem *fun_base = QM_ADDR(qm, MAILBOX_CMD_SEND_BASE); + unsigned long tmp0 = 0, tmp1 = 0; + + asm volatile("ldp %0, %1, %3\n" + "stp %0, %1, %2\n" + "dsb sy\n" + : "=&r" (tmp0), + "=&r" (tmp1), + "+Q" (*((char *)fun_base)) + : "Q" (*((char *)src)) + : "memory"); +} + +static int qm_mb(struct qm_info *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, + bool op, bool event) +{ + struct mailbox mailbox; + int i = 0; + int ret = 0; + + memset(&mailbox, 0, sizeof(struct mailbox)); + + mailbox.w0 = cmd | + (event ? 0x1 << MAILBOX_EVENT_SHIFT : 0) | + (op ? 0x1 << MAILBOX_OP_SHIFT : 0) | + (0x1 << MAILBOX_BUSY_SHIFT); + mailbox.queue_num = queue; + mailbox.base_l = lower_32_bits(dma_addr); + mailbox.base_h = upper_32_bits(dma_addr); + mailbox.rsvd = 0; + + mutex_lock(&qm->mailbox_lock); + + while (hacc_qm_mb_is_busy(qm) && i < 10) + i++; + if (i >= 10) { + ret = -EBUSY; + dev_err(&qm->pdev->dev, "QM mail box is busy!"); + goto busy_unlock; + } + qm_mb_write(qm, &mailbox); + i = 0; + while (hacc_qm_mb_is_busy(qm) && i < 10) + i++; + if (i >= 10) { + ret = -EBUSY; + dev_err(&qm->pdev->dev, "QM mail box is still busy!"); + goto busy_unlock; + } + +busy_unlock: + mutex_unlock(&qm->mailbox_lock); + + return ret; +} + +static void qm_db_v1(struct qm_info *qm, u16 qn, u8 cmd, u16 index, u8 priority) +{ + u64 doorbell = 0; + + doorbell = (u64)qn | ((u64)cmd << 16); + doorbell |= ((u64)index | ((u64)priority << 16)) << 32; + + writeq(doorbell, QM_ADDR(qm, DOORBELL_CMD_SEND_BASE_V1)); +} + +static void qm_db_v2(struct qm_info *qm, u16 qn, u8 cmd, u16 index, u8 priority) +{ + u64 doorbell = 0; + u16 randate = 0; + + doorbell = (u64)qn | ((u64)cmd << 12) | ((u64)randate << 16); + doorbell |= ((u64)index | ((u64)priority << 16)) << 32; + + writeq(doorbell, QM_ADDR(qm, DOORBELL_CMD_SEND_BASE_V2)); +} + +static void qm_db(struct qm_info *qm, u16 qn, u8 cmd, u16 index, u8 priority) +{ + qm->ops->qm_db(qm, qn, cmd, index, priority); +} + +/* @return 0 - cq/eq event, 1 - async event, 2 - abnormal error */ +static u32 qm_get_irq_source(struct qm_info *qm) +{ + return readl(QM_ADDR(qm, QM_VF_EQ_INT_SOURCE)); +} + +static inline struct hisi_qp *to_hisi_qp(struct qm_info *qm, struct eqe *eqe) +{ + u16 cqn = eqe->dw0 & QM_EQE_CQN_MASK; + struct hisi_qp *qp; + + read_lock(&qm->qps_lock); + qp = qm->qp_array[cqn]; + read_unlock(&qm->qps_lock); + + return qp; +} + +static inline void qm_cq_head_update(struct hisi_qp *qp) +{ + if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) { + QM_CQC(qp)->dw6 = QM_CQC(qp)->dw6 ^ CQC_PHASE_BIT; + qp->qp_status.cq_head = 0; + } else { + qp->qp_status.cq_head++; + } +} + +static inline void qm_poll_qp(struct hisi_qp *qp, struct qm_info *qm) +{ + struct cqe *cqe; + + cqe = QP_CQE(qp) + qp->qp_status.cq_head; + + if (qp->req_cb) { + while (CQE_PHASE(cqe) == CQC_PHASE(QM_CQC(qp))) { + dma_rmb(); + qp->req_cb(qp, QP_SQE_ADDR(qp) + + qm->sqe_size * + CQE_SQ_HEAD_INDEX(cqe)); + qm_cq_head_update(qp); + cqe = QP_CQE(qp) + qp->qp_status.cq_head; + } + } else if (qp->event_cb) { + dma_rmb(); + qp->event_cb(qp); + qm_cq_head_update(qp); + cqe = QP_CQE(qp) + qp->qp_status.cq_head; + } + + qm_db(qm, qp->queue_id, DOORBELL_CMD_CQ, qp->qp_status.cq_head, 0); + + /* set c_flag */ + qm_db(qm, qp->queue_id, DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1); +} + +static irqreturn_t qm_irq_thread(int irq, void *data) +{ + struct qm_info *qm = data; + struct eqe *eqe = QM_EQE(qm) + qm->eq_head; + struct eqc *eqc = QM_EQC(qm); + struct hisi_qp *qp; + + while (EQE_PHASE(eqe) == EQC_PHASE(eqc)) { + qp = to_hisi_qp(qm, eqe); + if (qp) + qm_poll_qp(qp, qm); + + if (qm->eq_head == QM_Q_DEPTH - 1) { + eqc->dw6 = eqc->dw6 ^ EQC_PHASE_BIT; + eqe = QM_EQE(qm); + qm->eq_head = 0; + } else { + eqe++; + qm->eq_head++; + } + + qm_db(qm, 0, DOORBELL_CMD_EQ, qm->eq_head, 0); + } + + return IRQ_HANDLED; +} + +static void qm_init_qp_status(struct hisi_qp *qp) +{ + struct hisi_acc_qp_status *qp_status = &qp->qp_status; + + qp_status->sq_tail = 0; + qp_status->sq_head = 0; + qp_status->cq_head = 0; + qp_status->cqc_phase = 1; + qp_status->is_sq_full = 0; +} + +/* check if bit in regs is 1 */ +static inline int qm_acc_check(struct qm_info *qm, u32 offset, u32 bit) +{ + int val; + + return readl_relaxed_poll_timeout(QM_ADDR(qm, offset), val, + val & BIT(bit), 10, 1000); +} + +static inline int qm_init_q_buffer(struct device *dev, size_t size, + struct qm_dma_buffer *db) +{ + if (db->addr && db->size) { + memset(db->addr, 0, db->size); + return 0; + } + db->size = size; + db->addr = dma_zalloc_coherent(dev, size, &db->dma, GFP_KERNEL); + if (!db->addr) + return -ENOMEM; + + return 0; +} + +static inline void qm_uninit_q_buffer(struct device *dev, + struct qm_dma_buffer *db) +{ + dma_free_coherent(dev, db->size, db->addr, db->dma); +} + +static inline int qm_init_bt(struct qm_info *qm, struct device *dev, + size_t size, struct qm_dma_buffer *db, int mb_cmd) +{ + int ret; + + ret = qm_init_q_buffer(dev, size, db); + if (ret) + return -ENOMEM; + + ret = qm_mb(qm, mb_cmd, db->dma, 0, 0, 0); + if (ret) { + qm_uninit_q_buffer(dev, db); + return ret; + } + + return 0; +} + +/* the config should be conducted after hisi_acc_init_qm_mem() */ +static int qm_vft_common_config_v1(struct qm_info *qm, u16 base, u32 number) +{ + u64 tmp; + int ret; + + ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0); + if (ret) + return ret; + qm_writel(qm, 0x0, QM_VFT_CFG_OP_WR); + qm_writel(qm, QM_SQC_VFT, QM_VFT_CFG_TYPE); + qm_writel(qm, qm->pdev->devfn, QM_VFT_CFG_ADDRESS); + + tmp = QM_SQC_VFT_BUF_SIZE | + QM_SQC_VFT_SQC_SIZE | + QM_SQC_VFT_INDEX_NUMBER | + QM_SQC_VFT_VALID | + (u64)base << QM_SQC_VFT_START_SQN_SHIFT; + + qm_writel(qm, tmp & 0xffffffff, QM_VFT_CFG_DATA_L); + qm_writel(qm, tmp >> 32, QM_VFT_CFG_DATA_H); + + qm_writel(qm, 0x0, QM_VFT_CFG_RDY); + qm_writel(qm, 0x1, QM_VFT_CFG_OP_ENABLE); + ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0); + if (ret) + return ret; + tmp = 0; + + qm_writel(qm, 0x0, QM_VFT_CFG_OP_WR); + qm_writel(qm, QM_CQC_VFT, QM_VFT_CFG_TYPE); + qm_writel(qm, qm->pdev->devfn, QM_VFT_CFG_ADDRESS); + + tmp = QM_CQC_VFT_BUF_SIZE | + QM_CQC_VFT_SQC_SIZE | + QM_CQC_VFT_INDEX_NUMBER | + QM_CQC_VFT_VALID; + + qm_writel(qm, tmp & 0xffffffff, QM_VFT_CFG_DATA_L); + qm_writel(qm, tmp >> 32, QM_VFT_CFG_DATA_H); + + qm_writel(qm, 0x0, QM_VFT_CFG_RDY); + qm_writel(qm, 0x1, QM_VFT_CFG_OP_ENABLE); + ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0); + if (ret) + return ret; + return 0; +} + +static int qm_vft_common_config_v2(struct qm_info *qm, u16 base, u32 number) +{ + u64 tmp; + int ret; + + ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0); + if (ret) + return ret; + qm_writel(qm, 0x0, QM_VFT_CFG_OP_WR); + qm_writel(qm, QM_SQC_VFT, QM_VFT_CFG_TYPE); + qm_writel(qm, qm->pdev->devfn, QM_VFT_CFG_ADDRESS); + + tmp = (u64)number << QM_SQC_VFT_SQN_SHIFT | + QM_SQC_VFT_VALID | + (u64)base << QM_SQC_VFT_START_SQN_SHIFT; + + qm_writel(qm, tmp & 0xffffffff, QM_VFT_CFG_DATA_L); + qm_writel(qm, tmp >> 32, QM_VFT_CFG_DATA_H); + + qm_writel(qm, 0x0, QM_VFT_CFG_RDY); + qm_writel(qm, 0x1, QM_VFT_CFG_OP_ENABLE); + ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0); + if (ret) + return ret; + tmp = 0; + + qm_writel(qm, 0x0, QM_VFT_CFG_OP_WR); + qm_writel(qm, QM_CQC_VFT, QM_VFT_CFG_TYPE); + qm_writel(qm, qm->pdev->devfn, QM_VFT_CFG_ADDRESS); + + tmp = QM_CQC_VFT_VALID; + + qm_writel(qm, tmp & 0xffffffff, QM_VFT_CFG_DATA_L); + qm_writel(qm, tmp >> 32, QM_VFT_CFG_DATA_H); + + qm_writel(qm, 0x0, QM_VFT_CFG_RDY); + qm_writel(qm, 0x1, QM_VFT_CFG_OP_ENABLE); + ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0); + if (ret) + return ret; + return 0; +} + +struct hisi_qp *hisi_qm_create_qp(struct qm_info *qm, u8 alg_type) +{ + struct hisi_qp *qp; + int qp_index; + int ret; + + write_lock(&qm->qps_lock); + qp_index = find_first_zero_bit(qm->qp_bitmap, qm->qp_num); + if (qp_index >= qm->qp_num) { + write_unlock(&qm->qps_lock); + return ERR_PTR(-EBUSY); + } + set_bit(qp_index, qm->qp_bitmap); + + qp = kzalloc(sizeof(*qp), GFP_KERNEL); + if (!qp) { + ret = -ENOMEM; + write_unlock(&qm->qps_lock); + goto err_with_bitset; + } + + qp->queue_id = qp_index; + qp->qm = qm; + qp->alg_type = alg_type; + qm_init_qp_status(qp); + write_unlock(&qm->qps_lock); + qm->free_qp--; + return qp; + +err_with_bitset: + clear_bit(qp_index, qm->qp_bitmap); + + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(hisi_qm_create_qp); + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV +static void hisi_qm_reg_user_info(struct hisi_qp *qp, u32 version) +{ + /* Fill qp_index and qm version information here, for user space */ + *(u32 *)qp->scqe.addr = qp->queue_id; + *((u32 *)qp->scqe.addr + 1) = version; + + /* To be fixed */ + +} +#endif + +int hisi_qm_start_qp_v1(struct hisi_qp *qp, unsigned long arg) +{ + struct qm_info *qm = qp->qm; + struct device *dev = &qm->pdev->dev; + struct sqc *sqc; + struct cqc *cqc; + int qp_index = qp->queue_id; + int pasid = arg; + int ret; + + /* set sq and cq context */ + if (!qp->sqc.addr) { + qp->sqc.addr = QM_SQC(qm) + qp_index; + qp->sqc.dma = qm->sqc.dma + qp_index * sizeof(struct sqc); + } + sqc = QM_SQC(qp); + + if (!qp->cqc.addr) { + qp->cqc.addr = QM_CQC(qm) + qp_index; + qp->cqc.dma = qm->cqc.dma + qp_index * sizeof(struct cqc); + } + cqc = QM_CQC(qp); + + /* allocate sq and cq */ + ret = qm_init_q_buffer(dev, + qm->sqe_size * QM_Q_DEPTH + sizeof(struct cqe) * QM_Q_DEPTH, + &qp->scqe); + if (ret) + return ret; + + INIT_QC(sqc, qp->scqe.dma); + sqc->pasid = pasid; + sqc->dw3 = (0 << SQ_HOP_NUM_SHIFT) | + (0 << SQ_PAGE_SIZE_SHIFT) | + (0 << SQ_BUF_SIZE_SHIFT) | + (ilog2(qm->sqe_size) << SQ_SQE_SIZE_SHIFT); + sqc->w8 = QM_Q_DEPTH - 1; + sqc->cq_num = qp_index; + sqc->w13 = 0 << SQ_PRIORITY_SHIFT | + 1 << SQ_ORDERS_SHIFT | + (qp->alg_type & SQ_TYPE_MASK) << SQ_TYPE_SHIFT; + + ret = qm_mb(qm, MAILBOX_CMD_SQC, qp->sqc.dma, qp_index, 0, 0); + if (ret) + return ret; + + INIT_QC(cqc, qp->scqe.dma + qm->sqe_size * QM_Q_DEPTH); + cqc->dw3 = (0 << CQ_HOP_NUM_SHIFT) | + (0 << CQ_PAGE_SIZE_SHIFT) | + (0 << CQ_BUF_SIZE_SHIFT) | + (4 << CQ_SQE_SIZE_SHIFT); + cqc->dw6 = 1 << CQ_PHASE_SHIFT | 1 << CQ_FLAG_SHIFT; + cqc->w8 = QM_Q_DEPTH - 1; + ret = qm_mb(qm, MAILBOX_CMD_CQC, qp->cqc.dma, qp_index, 0, 0); + if (ret) + return ret; + + write_lock(&qm->qps_lock); + qm->qp_array[qp_index] = qp; + init_completion(&qp->completion); + write_unlock(&qm->qps_lock); + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV + hisi_qm_reg_user_info(qp, qm->ver); +#endif + + return qp_index; +} + + +int hisi_qm_start_qp_v2(struct hisi_qp *qp, unsigned long arg) +{ + struct qm_info *qm = qp->qm; + struct device *dev = &qm->pdev->dev; + int ret; + struct sqc *sqc; + struct cqc *cqc; + int qp_index = qp->queue_id; + int pasid = arg; + + /* set sq and cq context */ + if (!qp->sqc.addr) { + qp->sqc.addr = QM_SQC(qm) + qp_index; + qp->sqc.dma = qm->sqc.dma + qp_index * sizeof(struct sqc); + } + sqc = QM_SQC(qp); + + if (!qp->cqc.addr) { + qp->cqc.addr = QM_CQC(qm) + qp_index; + qp->cqc.dma = qm->cqc.dma + qp_index * sizeof(struct cqc); + } + cqc = QM_CQC(qp); + + /* allocate sq and cq */ + ret = qm_init_q_buffer(dev, + qm->sqe_size * QM_Q_DEPTH + sizeof(struct cqe) * QM_Q_DEPTH, + &qp->scqe); + if (ret) + return ret; + + INIT_QC(sqc, qp->scqe.dma); + sqc->pasid = pasid; + sqc->dw3 = (QM_Q_DEPTH - 1) | + (ilog2(qm->sqe_size) << SQ_SQE_SIZE_SHIFT); + sqc->w8 = 0;/*rand_qc*/ + sqc->cq_num = qp_index; + sqc->w13 = 0 << SQ_PRIORITY_SHIFT | + 1 << SQ_ORDERS_SHIFT | + (qp->alg_type & SQ_TYPE_MASK) << SQ_TYPE_SHIFT; + + ret = qm_mb(qm, MAILBOX_CMD_SQC, qp->sqc.dma, qp_index, 0, 0); + if (ret) + return ret; + + INIT_QC(cqc, qp->scqe.dma + qm->sqe_size * QM_Q_DEPTH); + cqc->pasid = pasid; + cqc->dw3 = (QM_Q_DEPTH - 1) | + (4 << CQ_SQE_SIZE_SHIFT); + cqc->w8 = 0;/*rand_qc*/ + cqc->dw6 = 1 << CQ_PHASE_SHIFT | 1 << CQ_FLAG_SHIFT; + + ret = qm_mb(qm, MAILBOX_CMD_CQC, qp->cqc.dma, qp_index, 0, 0); + if (ret) + return ret; + + write_lock(&qm->qps_lock); + qm->qp_array[qp_index] = qp; + init_completion(&qp->completion); + write_unlock(&qm->qps_lock); + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV + hisi_qm_reg_user_info(qp, qm->ver); +#endif + return qp_index; +} + + +int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg) +{ + struct qm_info *qm = qp->qm; + + return qm->ops->qm_start_qp(qp, arg); +} +EXPORT_SYMBOL_GPL(hisi_qm_start_qp); + +static struct hisi_acc_qm_hw_ops qm_hw_ops_v1 = { + .vft_config = qm_vft_common_config_v1, + .qm_start_qp = hisi_qm_start_qp_v1, + .qm_db = qm_db_v1, +}; + +static struct hisi_acc_qm_hw_ops qm_hw_ops_v2 = { + .vft_config = qm_vft_common_config_v2, + .qm_start_qp = hisi_qm_start_qp_v2, + .qm_db = qm_db_v2, +}; + +void hisi_qm_release_qp(struct hisi_qp *qp) +{ + struct qm_info *qm = qp->qm; + struct device *dev = &qm->pdev->dev; + u64 phy = qp->udma_buf.phy_addr; + int order = qp->udma_buf.order; + + if (order > MAX_ORDER || phy & 0xfff) + return; + if (phy) + __free_pages(phys_to_page(phy), order); + + write_lock(&qm->qps_lock); + qm->free_qp++; + qm->qp_array[qp->queue_id] = NULL; + bitmap_clear(qm->qp_bitmap, qp->queue_id, 1); + write_unlock(&qm->qps_lock); + + qm_uninit_q_buffer(dev, &qp->scqe); + kfree(qp); +} +EXPORT_SYMBOL_GPL(hisi_qm_release_qp); + +static void *qm_get_avail_sqe(struct hisi_qp *qp) +{ + struct hisi_acc_qp_status *qp_status = &qp->qp_status; + void *sq_base = QP_SQE_ADDR(qp); + u16 sq_tail = qp_status->sq_tail; + + if (qp_status->is_sq_full == 1) + return NULL; + + return sq_base + sq_tail * qp->qm->sqe_size; +} + +int hisi_qp_send(struct hisi_qp *qp, void *msg) +{ + struct hisi_acc_qp_status *qp_status = &qp->qp_status; + u16 sq_tail = qp_status->sq_tail; + u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH; + unsigned long timeout = 100; + void *sqe = qm_get_avail_sqe(qp); + + if (!sqe) + return -ENOSPC; + + memcpy(sqe, msg, qp->qm->sqe_size); + + qm_db(qp->qm, qp->queue_id, DOORBELL_CMD_SQ, sq_tail_next, 0); + + qp_status->sq_tail = sq_tail_next; + + if (qp_status->sq_tail == qp_status->sq_head) + qp_status->is_sq_full = 1; + + /* wait until job finished */ + wait_for_completion_timeout(&qp->completion, timeout); + + return 0; +} +EXPORT_SYMBOL_GPL(hisi_qp_send); + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV +/* mdev->supported_type_groups */ +static struct attribute *hisi_qm_type_attrs[] = { + &mdev_type_attr_device_api.attr, + &mdev_type_attr_available_instances.attr, + &mdev_type_attr_type.attr, + NULL, +}; + +/* Other supported types algorithms can be added here */ +static struct attribute_group hisi_zlib_type_group = { + .name = "zlib", + .attrs = hisi_qm_type_attrs, +}; + +static struct attribute_group hisi_gzip_type_group = { + .name = "gzip", + .attrs = hisi_qm_type_attrs, +}; + +static struct attribute_group hisi_rsa_type_group = { + .name = "rsa", + .attrs = hisi_qm_type_attrs, +}; + +static struct attribute_group hisi_dh_type_group = { + .name = "dh", + .attrs = hisi_qm_type_attrs, +}; + +static struct attribute_group *mdev_zip_type_groups[] = { + &hisi_zlib_type_group, + &hisi_gzip_type_group, + NULL, +}; + +static struct attribute_group *mdev_hpre_type_groups[] = { + &hisi_rsa_type_group, + &hisi_dh_type_group, + NULL, +}; + +static struct attribute *mdev_dev_attrs[] = { + &dev_attr_pid.attr, + NULL, +}; + +static const struct attribute_group mdev_dev_group = { + .name = VFIO_SPIMDEV_MDEV_ATTRS_GRP, + .attrs = mdev_dev_attrs, +}; + +/* this will be showed under virtual device's directory */ +static const struct attribute_group *mdev_dev_groups[] = { + &mdev_dev_group, + NULL, +}; + +static int hisi_qm_get_queue(struct vfio_spimdev *spimdev, const char *alg, + struct vfio_spimdev_queue **q) +{ + struct qm_info *qm = spimdev->priv; + struct hisi_qp *qp = NULL; + struct vfio_spimdev_queue *wd_q; + u8 alg_type = 0; + int ret = 0; + + /* alg_type can be gotten from alg name */ + qp = hisi_qm_create_qp(qm, alg_type); + if (IS_ERR(qp)) + return PTR_ERR(qp); + + wd_q = kzalloc(sizeof(struct vfio_spimdev_queue), GFP_KERNEL); + if (!wd_q) { + ret = -ENOMEM; + goto err_with_qp; + } + + wd_q->priv = qp; + wd_q->spimdev = spimdev; + wd_q->alg = alg; + *q = wd_q; + qp->spimdev_q = wd_q; + + /* PASID is 0 now, to be fixed */ + ret = qm->ops->qm_start_qp(qp, 0); + if (ret < 0) + goto err_with_wd_q; + wd_q->qid = ret; + return ret; + +err_with_wd_q: + kfree(wd_q); +err_with_qp: + hisi_qm_release_qp(qp); + return ret; +} + +static int hisi_qm_put_queue(struct vfio_spimdev_queue *q) +{ + struct hisi_qp *qp = q->priv; + + /* need to stop hardware, but can not support in v1 */ + hisi_qm_release_qp(qp); + kfree(q); + return 0; +} + +static int hisi_qm_reset_queue(struct vfio_spimdev_queue *q) +{ + struct hisi_qp *qp = q->priv; + + qm_init_qp_status(qp); + return hisi_qm_start_qp(qp, 0); +} + +/* map sq/cq/doorbell to user space */ +static int hisi_qm_mmap(struct vfio_spimdev_queue *q, + struct vm_area_struct *vma) +{ + struct hisi_qp *qp = (struct hisi_qp *)q->priv; + struct qm_info *qm = qp->qm; + struct device *dev = &qm->pdev->dev; + size_t sz = vma->vm_end - vma->vm_start; + u8 region; + + vma->vm_flags |= (VM_IO | VM_LOCKED | VM_DONTEXPAND | VM_DONTDUMP); + region = _VFIO_SPIMDEV_REGION(vma->vm_pgoff); + + switch (region) { + case 0: + if (sz > PAGE_SIZE) + return -EINVAL; + /* + * Warning: This is not safe as multiple queues use the same + * doorbell, v1 hardware interface problem. v2 will fix it + */ + if (qm->ver == QM_HW_V2) + return remap_pfn_range(vma, vma->vm_start, + (qm->phys_base + PAGE_SIZE) >> + PAGE_SHIFT, sz, + pgprot_noncached( + vma->vm_page_prot)); + else if (qm->ver == QM_HW_V1) + return remap_pfn_range(vma, vma->vm_start, + qm->phys_base >> PAGE_SHIFT, + sz, pgprot_noncached(vma->vm_page_prot)); + else + return -ENODEV; + case 1: + vma->vm_pgoff = 0; + if (sz > qp->scqe.size) + return -EINVAL; + + return dma_mmap_coherent(dev, vma, qp->scqe.addr, qp->scqe.dma, + sz); + + default: + return -EINVAL; + } +} + +static int hisi_qm_get_available_qnum(struct vfio_spimdev *spimdev) +{ + int num; + struct qm_info *qm = spimdev->priv; + + read_lock(&qm->qps_lock); + num = qm->free_qp; + read_unlock(&qm->qps_lock); + + return num; +} + +static long qm_ioctl(struct vfio_spimdev_queue *q, unsigned int cmd, + unsigned long arg) +{ + struct vfio_spimdev *spimdev = q->spimdev; + struct hisi_qp *qp = q->priv; + unsigned long long phy; + void *page; + int order; + struct page *page_list; + u64 size; + + switch (cmd) { + /* While supporting NO-IOMMU, we need this */ + case _GET_DMA_PAGES: +#ifdef CONFIG_STRICT_DEVMEM + dev_err(spimdev->dev, "\nSTRICT_DEVMEM is on!"); + return -EPERM; +#endif + if (copy_from_user(&size, (unsigned long long *)arg, + sizeof(u64))) + return -EFAULT; + if (size == 0) + return -EINVAL; + order = get_order(size); + if (order > MAX_ORDER) + return -ENOMEM; + page_list = alloc_pages_node(spimdev->node_id, GFP_DMA, order); + if (!page_list) { + dev_err(spimdev->dev, + "alloc 2^%d pages fail!\n", order); + return -ENOMEM; + } + page = page_address(page_list); + phy = (unsigned long long)virt_to_phys(page); + qp->udma_buf.phy_addr = phy; + qp->udma_buf.order = order; + + if (copy_to_user((unsigned long long *)arg, &phy, + sizeof(unsigned long long))) + return -EFAULT; + break; + case _PUT_DMA_PAGES: + if (copy_from_user(&phy, + (unsigned long long *)arg, + sizeof(unsigned long long))) + return -EFAULT; + phy = phy & PAGE_MASK; + order = qp->udma_buf.order; + if (order > MAX_ORDER || phy != qp->udma_buf.phy_addr) + return -EINVAL; + __free_pages(phys_to_page(phy), order); + qp->udma_buf.phy_addr = 0; + + break; + default: + dev_err(spimdev->dev, + "%s, ioctl cmd (0x%x) is not supported!\n", __func__, cmd); + return -EINVAL; + } + + return 0; +} + +static const struct vfio_spimdev_ops qm_ops = { + .get_queue = hisi_qm_get_queue, + .put_queue = hisi_qm_put_queue, + .mmap = hisi_qm_mmap, + .get_available_instances = hisi_qm_get_available_qnum, + .ioctl = qm_ioctl, + .reset_queue = hisi_qm_reset_queue, +}; + +static int qm_register_spimdev(struct qm_info *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct vfio_spimdev *spimdev = &qm->spimdev; + +#ifdef CONFIG_IOMMU_SVA + spimdev->dma_flag = VFIO_SPIMDEV_DMA_MULTI_PROC_MAP; +#endif + +#ifdef CONFIG_ARM_SMMU_V3 + spimdev->iommu_type = VFIO_TYPE1_IOMMU; + spimdev->dma_flag = VFIO_SPIMDEV_DMA_SINGLE_PROC_MAP; +#else + spimdev->iommu_type = VFIO_SPIMDEV_IOMMU; + spimdev->dma_flag = VFIO_SPIMDEV_DMA_PHY; +#endif + spimdev->owner = THIS_MODULE; + spimdev->name = qm->dev_name; + spimdev->dev = &pdev->dev; + spimdev->is_vf = pdev->is_virtfn; + spimdev->priv = qm; + if (qm->ver == QM_HW_V1) + spimdev->api_ver = "hisi_qm_v1"; + else + spimdev->api_ver = "hisi_qm_v2"; + spimdev->flags = VFIO_SPIMDEV_SAME_ALG_QFLG; + qm->mdev_dev_groups = mdev_dev_groups; + spimdev->mdev_fops.mdev_attr_groups = qm->mdev_dev_groups; + if (qm->type == ZIP) + spimdev->mdev_fops.supported_type_groups = + mdev_zip_type_groups; + else if (qm->type == HPRE) + spimdev->mdev_fops.supported_type_groups = + mdev_hpre_type_groups; + spimdev->ops = &qm_ops; + + return vfio_spimdev_register(spimdev); +} +#endif + +int hisi_qm_init(struct qm_info *qm, enum qm_type type) +{ + struct pci_dev *pdev = qm->pdev; + int ret; + + ret = pci_enable_device_mem(pdev); + if (ret < 0) { + dev_err(&pdev->dev, "Can't enable device mem!\n"); + return ret; + } + qm->type = type; + qm->dev_name = pdev->driver->name; + ret = pci_request_mem_regions(pdev, qm->dev_name); + if (ret < 0) { + dev_err(&pdev->dev, "Can't request mem regions!\n"); + goto err_with_pcidev; + } + + qm->phys_base = pci_resource_start(pdev, 2); + qm->size = pci_resource_len(qm->pdev, 2); + qm->io_base = devm_ioremap(&pdev->dev, qm->phys_base, qm->size); + if (!qm->io_base) { + ret = -EIO; + dev_err(&pdev->dev, "ioremap qm physical base fail!\n"); + goto err_with_mem_regions; + } + + dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64)); + pci_set_master(pdev); + + ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI); + if (ret < 0) { + dev_err(&pdev->dev, "Enable MSI vectors fail!\n"); + goto err_with_mem_regions; + } + + qm->eq_head = 0; + mutex_init(&qm->mailbox_lock); + rwlock_init(&qm->qps_lock); + + if (qm->ver == QM_HW_V1) + qm->ops = &qm_hw_ops_v1; + else if (qm->ver == QM_HW_V2) + qm->ops = &qm_hw_ops_v2; + + return 0; + +err_with_mem_regions: + pci_release_mem_regions(pdev); +err_with_pcidev: + pci_disable_device(pdev); + + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_init); + +void hisi_qm_uninit(struct qm_info *qm) +{ + struct pci_dev *pdev = qm->pdev; + + pci_free_irq_vectors(pdev); + pci_release_mem_regions(pdev); + pci_disable_device(pdev); +} +EXPORT_SYMBOL_GPL(hisi_qm_uninit); + +static irqreturn_t qm_irq(int irq, void *data) +{ + struct qm_info *qm = data; + u32 int_source; + + int_source = qm_get_irq_source(qm); + if (int_source) + return IRQ_WAKE_THREAD; + + dev_err(&qm->pdev->dev, "invalid int source %d\n", int_source); + + return IRQ_HANDLED; +} + +int hisi_qm_start(struct qm_info *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + int ret; + + if (qm->pdev->is_physfn) + qm->ops->vft_config(qm, qm->qp_base, qm->qp_num); + + ret = qm_init_q_buffer(dev, sizeof(struct eqc), &qm->eqc); + if (ret) + goto err_out; + + ret = qm_init_q_buffer(dev, sizeof(struct eqe) * QM_Q_DEPTH, &qm->eqe); + if (ret) + goto err_with_eqc; + + QM_EQC(qm)->base_l = lower_32_bits(qm->eqe.dma); + QM_EQC(qm)->base_h = upper_32_bits(qm->eqe.dma); + QM_EQC(qm)->dw3 = 2 << MB_EQC_EQE_SHIFT; + QM_EQC(qm)->dw6 = (QM_Q_DEPTH - 1) | (1 << MB_EQC_PHASE_SHIFT); + ret = qm_mb(qm, MAILBOX_CMD_EQC, qm->eqc.dma, 0, 0, 0); + if (ret) + goto err_with_eqe; + + qm->qp_bitmap = kcalloc(BITS_TO_LONGS(qm->qp_num), sizeof(long), + GFP_KERNEL); + if (!qm->qp_bitmap) + goto err_with_eqe; + + qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp *), + GFP_KERNEL); + if (!qm->qp_array) + goto err_with_bitmap; + + /* Init sqc_bt */ + ret = qm_init_bt(qm, dev, sizeof(struct sqc) * qm->qp_num, &qm->sqc, + MAILBOX_CMD_SQC_BT); + if (ret) + goto err_with_qp_array; + + /* Init cqc_bt */ + ret = qm_init_bt(qm, dev, sizeof(struct cqc) * qm->qp_num, &qm->cqc, + MAILBOX_CMD_CQC_BT); + if (ret) + goto err_with_sqc; + + ret = request_threaded_irq(pci_irq_vector(pdev, 0), qm_irq, + qm_irq_thread, IRQF_SHARED, qm->dev_name, + qm); + if (ret) + goto err_with_cqc; + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV + ret = qm_register_spimdev(qm); + if (ret) { + dev_err(dev, "Hisilicon QM register to SPIMDEV fail!\n"); + goto err_with_cqc; + } +#endif + + writel(0x0, QM_ADDR(qm, QM_VF_EQ_INT_MASK)); + + return 0; + +err_with_cqc: + qm_uninit_q_buffer(dev, &qm->cqc); +err_with_sqc: + qm_uninit_q_buffer(dev, &qm->sqc); +err_with_qp_array: + kfree(qm->qp_array); +err_with_bitmap: + kfree(qm->qp_bitmap); +err_with_eqe: + qm_uninit_q_buffer(dev, &qm->eqe); +err_with_eqc: + qm_uninit_q_buffer(dev, &qm->eqc); +err_out: + return ret; +} +EXPORT_SYMBOL_GPL(hisi_qm_start); + +void hisi_qm_stop(struct qm_info *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct device *dev = &pdev->dev; + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV + vfio_spimdev_unregister(&qm->spimdev); +#endif + + free_irq(pci_irq_vector(pdev, 0), qm); + qm_uninit_q_buffer(dev, &qm->cqc); + kfree(qm->qp_array); + kfree(qm->qp_bitmap); + qm_uninit_q_buffer(dev, &qm->eqe); + qm_uninit_q_buffer(dev, &qm->eqc); +} +EXPORT_SYMBOL_GPL(hisi_qm_stop); + +/* put qm into init state, so the acce config become available */ +int hisi_qm_mem_start(struct qm_info *qm) +{ + u32 val; + + qm_writel(qm, 0x1, QM_MEM_START_INIT); + return readl_relaxed_poll_timeout(QM_ADDR(qm, QM_MEM_INIT_DONE), val, + val & BIT(0), 10, 1000); +} +EXPORT_SYMBOL_GPL(hisi_qm_mem_start); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Zhou Wang "); +MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver"); diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h new file mode 100644 index 000000000000..9b99e42d32b3 --- /dev/null +++ b/drivers/crypto/hisilicon/qm.h @@ -0,0 +1,132 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef HISI_ACC_QM_H +#define HISI_ACC_QM_H + +#include +#include +#include +#include +#include + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV +#include +#endif + +#define QM_HW_V1 1 +#define QM_HW_V2 2 + +#define QM_CQE_SIZE 16 +/* default queue depth for sq/cq/eq */ +#define QM_Q_DEPTH 1024 + +/* qm user domain */ +#define QM_ARUSER_M_CFG_1 0x100088 +#define QM_ARUSER_M_CFG_ENABLE 0x100090 +#define QM_AWUSER_M_CFG_1 0x100098 +#define QM_AWUSER_M_CFG_ENABLE 0x1000a0 +#define QM_WUSER_M_CFG_ENABLE 0x1000a8 + +/* qm cache */ +#define QM_CACHE_CTL 0x100050 +#define QM_AXI_M_CFG 0x1000ac +#define QM_AXI_M_CFG_ENABLE 0x1000b0 +#define QM_PEH_AXUSER_CFG 0x1000cc +#define QM_PEH_AXUSER_CFG_ENABLE 0x1000d0 + +#define QP_SQE_ADDR(qp) ((qp)->scqe.addr) +#define _GET_DMA_PAGES _IOW('d', 3, unsigned long long) +#define _PUT_DMA_PAGES _IOW('d', 4, unsigned long long) +enum qm_type { + ZIP = 1, + HPRE, + SEC, +}; + +struct qm_dma_buffer { + int size; + void *addr; + dma_addr_t dma; +}; + +struct qm_info { + int ver; + enum qm_type type; + const char *dev_name; + struct pci_dev *pdev; + resource_size_t phys_base; + resource_size_t size; + void __iomem *io_base; + u32 sqe_size; + u32 qp_base; + u32 qp_num; + u32 free_qp; + struct qm_dma_buffer sqc, cqc, eqc, eqe; + u32 eq_head; + rwlock_t qps_lock; + unsigned long *qp_bitmap; + struct hisi_qp **qp_array; + struct mutex mailbox_lock; + struct hisi_acc_qm_hw_ops *ops; + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV + struct vfio_spimdev spimdev; + const struct attribute_group **mdev_dev_groups; +#endif +}; +#define QM_ADDR(qm, off) ((qm)->io_base + off) + +struct hisi_acc_qp_status { + u16 sq_tail; + u16 sq_head; + u16 cq_head; + bool cqc_phase; + int is_sq_full; +}; + +struct hisi_qp; + +struct hisi_qp_ops { + int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm); +}; + +struct qp_phy_pages { + u64 size; + int order; + int node_id; + u64 phy_addr; +}; + +struct hisi_qp { + /* sq number in this function */ + u32 queue_id; + u8 alg_type; + u8 req_type; + struct qm_dma_buffer sqc, cqc; + struct qm_dma_buffer scqe; + struct hisi_acc_qp_status qp_status; + struct qm_info *qm; + struct qp_phy_pages udma_buf;/* For user space */ + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV + struct vfio_spimdev_queue *spimdev_q; +#endif + + /* for crypto sync API */ + struct completion completion; + + struct hisi_qp_ops *hw_ops; + void *qp_ctx; + void (*event_cb)(struct hisi_qp *qp); + void (*req_cb)(struct hisi_qp *qp, void *data); +}; + +extern int hisi_qm_init(struct qm_info *qm, enum qm_type type); +extern void hisi_qm_uninit(struct qm_info *qm); +extern int hisi_qm_start(struct qm_info *qm); +extern void hisi_qm_stop(struct qm_info *qm); +extern int hisi_qm_mem_start(struct qm_info *qm); +extern struct hisi_qp *hisi_qm_create_qp(struct qm_info *qm, u8 alg_type); +extern int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg); +extern void hisi_qm_release_qp(struct hisi_qp *qp); +extern int hisi_qp_send(struct hisi_qp *qp, void *msg); +#endif diff --git a/drivers/crypto/hisilicon/zip/Makefile b/drivers/crypto/hisilicon/zip/Makefile new file mode 100644 index 000000000000..a936f099ee22 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/Makefile @@ -0,0 +1,2 @@ +obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += hisi_zip.o +hisi_zip-objs = zip_main.o zip_crypto.o diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h new file mode 100644 index 000000000000..87515e158b17 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -0,0 +1,57 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef HISI_ZIP_H +#define HISI_ZIP_H + +#include +#include "../qm.h" + +#define HZIP_SQE_SIZE 128 +#define HZIP_SQ_SIZE (HZIP_SQE_SIZE * QM_Q_DEPTH) +#define QM_CQ_SIZE (QM_CQE_SIZE * QM_Q_DEPTH) +#define HZIP_PF_DEF_Q_NUM 64 +#define HZIP_PF_DEF_Q_BASE 0 + +struct hisi_zip { + struct qm_info qm; + struct list_head list; + +#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV + struct vfio_spimdev *spimdev; +#endif +}; + +struct hisi_zip_sqe { + __u32 consumed; + __u32 produced; + __u32 comp_data_length; + __u32 dw3; + __u32 input_data_length; + __u32 lba_l; + __u32 lba_h; + __u32 dw7; + __u32 dw8; + __u32 dw9; + __u32 dw10; + __u32 priv_info; + __u32 dw12; + __u32 tag; + __u32 dest_avail_out; + __u32 rsvd0; + __u32 comp_head_addr_l; + __u32 comp_head_addr_h; + __u32 source_addr_l; + __u32 source_addr_h; + __u32 dest_addr_l; + __u32 dest_addr_h; + __u32 stream_ctx_addr_l; + __u32 stream_ctx_addr_h; + __u32 cipher_key1_addr_l; + __u32 cipher_key1_addr_h; + __u32 cipher_key2_addr_l; + __u32 cipher_key2_addr_h; + __u32 rsvd1[4]; +}; + +extern struct list_head hisi_zip_list; + +#endif diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.c b/drivers/crypto/hisilicon/zip/zip_crypto.c new file mode 100644 index 000000000000..6a4e9bbe0f0b --- /dev/null +++ b/drivers/crypto/hisilicon/zip/zip_crypto.c @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include "../qm.h" +#include "zip.h" + +#define INPUT_BUFFER_SIZE (64 * 1024) +#define OUTPUT_BUFFER_SIZE (64 * 1024) + +#define COMP_NAME_TO_TYPE(alg_name) \ + (!strcmp((alg_name), "zlib-deflate") ? 0x02 : \ + !strcmp((alg_name), "gzip") ? 0x03 : 0) \ + +struct hisi_zip_buffer { + u8 *input; + dma_addr_t input_dma; + u8 *output; + dma_addr_t output_dma; +}; + +struct hisi_zip_qp_ctx { + struct hisi_zip_buffer buffer; + struct hisi_qp *qp; + struct hisi_zip_sqe zip_sqe; +}; + +struct hisi_zip_ctx { +#define QPC_COMP 0 +#define QPC_DECOMP 1 + struct hisi_zip_qp_ctx qp_ctx[2]; +}; + +static struct hisi_zip *find_zip_device(int node) +{ + struct hisi_zip *hisi_zip, *ret = NULL; + struct device *dev; + int min_distance = 100; + int dev_node = 0; + + list_for_each_entry(hisi_zip, &hisi_zip_list, list) { + dev = &hisi_zip->qm.pdev->dev; +#ifdef CONFIG_NUMA + dev_node = dev->numa_node; +#endif + if (node_distance(dev_node, node) < min_distance) { + ret = hisi_zip; + min_distance = node_distance(dev_node, node); + } + } + + return ret; +} + +static void hisi_zip_qp_event_notifier(struct hisi_qp *qp) +{ + complete(&qp->completion); +} + +static int hisi_zip_fill_sqe_v1(void *sqe, void *q_parm, u32 len) +{ + struct hisi_zip_sqe *zip_sqe = (struct hisi_zip_sqe *)sqe; + struct hisi_zip_qp_ctx *qp_ctx = (struct hisi_zip_qp_ctx *)q_parm; + struct hisi_zip_buffer *buffer = &qp_ctx->buffer; + + memset(zip_sqe, 0, sizeof(struct hisi_zip_sqe)); + + zip_sqe->input_data_length = len; + zip_sqe->dw9 = qp_ctx->qp->req_type; + zip_sqe->dest_avail_out = OUTPUT_BUFFER_SIZE; + zip_sqe->source_addr_l = lower_32_bits(buffer->input_dma); + zip_sqe->source_addr_h = upper_32_bits(buffer->input_dma); + zip_sqe->dest_addr_l = lower_32_bits(buffer->output_dma); + zip_sqe->dest_addr_h = upper_32_bits(buffer->output_dma); + + return 0; +} + +/* let's allocate one buffer now, may have problem in async case */ +static int hisi_zip_alloc_qp_buffer(struct hisi_zip_qp_ctx *hisi_zip_qp_ctx) +{ + struct hisi_zip_buffer *buffer = &hisi_zip_qp_ctx->buffer; + struct hisi_qp *qp = hisi_zip_qp_ctx->qp; + struct device *dev = &qp->qm->pdev->dev; + int ret; + + buffer->input = dma_alloc_coherent(dev, INPUT_BUFFER_SIZE, + &buffer->input_dma, GFP_KERNEL); + if (!buffer->input) + return -ENOMEM; + + buffer->output = dma_alloc_coherent(dev, OUTPUT_BUFFER_SIZE, + &buffer->output_dma, GFP_KERNEL); + if (!buffer->output) { + ret = -ENOMEM; + goto err_alloc_output_buffer; + } + + return 0; + +err_alloc_output_buffer: + dma_free_coherent(dev, INPUT_BUFFER_SIZE, buffer->input, + buffer->input_dma); + return ret; +} + +static void hisi_zip_free_qp_buffer(struct hisi_zip_qp_ctx *hisi_zip_qp_ctx) +{ + struct hisi_zip_buffer *buffer = &hisi_zip_qp_ctx->buffer; + struct hisi_qp *qp = hisi_zip_qp_ctx->qp; + struct device *dev = &qp->qm->pdev->dev; + + dma_free_coherent(dev, INPUT_BUFFER_SIZE, buffer->input, + buffer->input_dma); + dma_free_coherent(dev, OUTPUT_BUFFER_SIZE, buffer->output, + buffer->output_dma); +} + +static int hisi_zip_create_qp(struct qm_info *qm, struct hisi_zip_qp_ctx *ctx, + int alg_type, int req_type) +{ + struct hisi_qp *qp; + int ret; + + qp = hisi_qm_create_qp(qm, alg_type); + + if (IS_ERR(qp)) + return PTR_ERR(qp); + + qp->event_cb = hisi_zip_qp_event_notifier; + qp->req_type = req_type; + + qp->qp_ctx = ctx; + ctx->qp = qp; + + ret = hisi_zip_alloc_qp_buffer(ctx); + if (ret) + goto err_with_qp; + + ret = hisi_qm_start_qp(qp, 0); + if (ret < 0) + goto err_with_qp_buffer; + + return 0; +err_with_qp_buffer: + hisi_zip_free_qp_buffer(ctx); +err_with_qp: + hisi_qm_release_qp(qp); + return ret; +} + +static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx) +{ + hisi_qm_release_qp(ctx->qp); + hisi_zip_free_qp_buffer(ctx); +} + +static int hisi_zip_alloc_comp_ctx(struct crypto_tfm *tfm) +{ + struct hisi_zip_ctx *hisi_zip_ctx = crypto_tfm_ctx(tfm); + const char *alg_name = crypto_tfm_alg_name(tfm); + struct hisi_zip *hisi_zip; + struct qm_info *qm; + int ret, i, j; + + u8 req_type = COMP_NAME_TO_TYPE(alg_name); + + /* find the proper zip device */ + hisi_zip = find_zip_device(cpu_to_node(smp_processor_id())); + if (!hisi_zip) { + pr_err("Can not find proper ZIP device!\n"); + return -ENODEV; + } + qm = &hisi_zip->qm; + + for (i = 0; i < 2; i++) { + /* it is just happen that 0 is compress, 1 is decompress on alg_type */ + ret = hisi_zip_create_qp(qm, &hisi_zip_ctx->qp_ctx[i], i, + req_type); + if (ret) + goto err; + } + + return 0; +err: + for (j = i-1; j >= 0; j--) + hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[j]); + + return ret; +} + +static void hisi_zip_free_comp_ctx(struct crypto_tfm *tfm) +{ + struct hisi_zip_ctx *hisi_zip_ctx = crypto_tfm_ctx(tfm); + int i; + + /* release the qp */ + for (i = 1; i >= 0; i--) + hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]); +} + +static int hisi_zip_copy_data_to_buffer(struct hisi_zip_qp_ctx *qp_ctx, + const u8 *src, unsigned int slen) +{ + struct hisi_zip_buffer *buffer = &qp_ctx->buffer; + + if (slen > INPUT_BUFFER_SIZE) + return -EINVAL; + + memcpy(buffer->input, src, slen); + + return 0; +} + +static struct hisi_zip_sqe *hisi_zip_get_writeback_sqe(struct hisi_qp *qp) +{ + struct hisi_acc_qp_status *qp_status = &qp->qp_status; + struct hisi_zip_sqe *sq_base = QP_SQE_ADDR(qp); + u16 sq_head = qp_status->sq_head; + + return sq_base + sq_head; +} + +static int hisi_zip_copy_data_from_buffer(struct hisi_zip_qp_ctx *qp_ctx, + u8 *dst, unsigned int *dlen) +{ + struct hisi_zip_buffer *buffer = &qp_ctx->buffer; + struct hisi_qp *qp = qp_ctx->qp; + struct hisi_zip_sqe *zip_sqe = hisi_zip_get_writeback_sqe(qp); + u32 status = zip_sqe->dw3 & 0xff; + u16 sq_head; + + if (status != 0) { + pr_err("hisi zip: %s fail!\n", (qp->alg_type == 0) ? + "compression" : "decompression"); + return status; + } + + if (zip_sqe->produced > OUTPUT_BUFFER_SIZE) + return -ENOMEM; + + memcpy(dst, buffer->output, zip_sqe->produced); + *dlen = zip_sqe->produced; + + sq_head = qp->qp_status.sq_head; + if (sq_head == QM_Q_DEPTH - 1) + qp->qp_status.sq_head = 0; + else + qp->qp_status.sq_head++; + + return 0; +} + +static int hisi_zip_compress(struct crypto_tfm *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int *dlen) +{ + struct hisi_zip_ctx *hisi_zip_ctx = crypto_tfm_ctx(tfm); + struct hisi_zip_qp_ctx *qp_ctx = &hisi_zip_ctx->qp_ctx[QPC_COMP]; + struct hisi_qp *qp = qp_ctx->qp; + struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe; + int ret; + + ret = hisi_zip_copy_data_to_buffer(qp_ctx, src, slen); + if (ret < 0) + return ret; + + hisi_zip_fill_sqe_v1(zip_sqe, qp_ctx, slen); + + /* send command to start the compress job */ + hisi_qp_send(qp, zip_sqe); + + return hisi_zip_copy_data_from_buffer(qp_ctx, dst, dlen); +} + +static int hisi_zip_decompress(struct crypto_tfm *tfm, const u8 *src, + unsigned int slen, u8 *dst, unsigned int *dlen) +{ + struct hisi_zip_ctx *hisi_zip_ctx = crypto_tfm_ctx(tfm); + struct hisi_zip_qp_ctx *qp_ctx = &hisi_zip_ctx->qp_ctx[QPC_DECOMP]; + struct hisi_qp *qp = qp_ctx->qp; + struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe; + int ret; + + ret = hisi_zip_copy_data_to_buffer(qp_ctx, src, slen); + if (ret < 0) + return ret; + + hisi_zip_fill_sqe_v1(zip_sqe, qp_ctx, slen); + + /* send command to start the decompress job */ + hisi_qp_send(qp, zip_sqe); + + return hisi_zip_copy_data_from_buffer(qp_ctx, dst, dlen); +} + +static struct crypto_alg hisi_zip_zlib = { + .cra_name = "zlib-deflate", + .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, + .cra_ctxsize = sizeof(struct hisi_zip_ctx), + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_init = hisi_zip_alloc_comp_ctx, + .cra_exit = hisi_zip_free_comp_ctx, + .cra_u = { + .compress = { + .coa_compress = hisi_zip_compress, + .coa_decompress = hisi_zip_decompress + } + } +}; + +static struct crypto_alg hisi_zip_gzip = { + .cra_name = "gzip", + .cra_flags = CRYPTO_ALG_TYPE_COMPRESS, + .cra_ctxsize = sizeof(struct hisi_zip_ctx), + .cra_priority = 300, + .cra_module = THIS_MODULE, + .cra_init = hisi_zip_alloc_comp_ctx, + .cra_exit = hisi_zip_free_comp_ctx, + .cra_u = { + .compress = { + .coa_compress = hisi_zip_compress, + .coa_decompress = hisi_zip_decompress + } + } +}; + +int hisi_zip_register_to_crypto(void) +{ + int ret; + + ret = crypto_register_alg(&hisi_zip_zlib); + if (ret < 0) { + pr_err("Zlib algorithm registration failed\n"); + return ret; + } + + ret = crypto_register_alg(&hisi_zip_gzip); + if (ret < 0) { + pr_err("Gzip algorithm registration failed\n"); + goto err_unregister_zlib; + } + + return 0; + +err_unregister_zlib: + crypto_unregister_alg(&hisi_zip_zlib); + + return ret; +} + +void hisi_zip_unregister_from_crypto(void) +{ + crypto_unregister_alg(&hisi_zip_zlib); + crypto_unregister_alg(&hisi_zip_gzip); +} diff --git a/drivers/crypto/hisilicon/zip/zip_crypto.h b/drivers/crypto/hisilicon/zip/zip_crypto.h new file mode 100644 index 000000000000..84eefd74c9c4 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/zip_crypto.h @@ -0,0 +1,8 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef HISI_ZIP_CRYPTO_H +#define HISI_ZIP_CRYPTO_H + +int hisi_zip_register_to_crypto(void); +void hisi_zip_unregister_from_crypto(void); + +#endif diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c new file mode 100644 index 000000000000..20a89dc9ced3 --- /dev/null +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -0,0 +1,202 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include +#include + +#include "zip.h" +#include "zip_crypto.h" + +#define HZIP_VF_NUM 63 +#define HZIP_QUEUE_NUM_V1 4096 +#define HZIP_QUEUE_NUM_V2 1024 + +#define HZIP_FSM_MAX_CNT 0x301008 + +#define HZIP_PORT_ARCA_CHE_0 0x301040 +#define HZIP_PORT_ARCA_CHE_1 0x301044 +#define HZIP_PORT_AWCA_CHE_0 0x301060 +#define HZIP_PORT_AWCA_CHE_1 0x301064 + +#define HZIP_BD_RUSER_32_63 0x301110 +#define HZIP_SGL_RUSER_32_63 0x30111c +#define HZIP_DATA_RUSER_32_63 0x301128 +#define HZIP_DATA_WUSER_32_63 0x301134 +#define HZIP_BD_WUSER_32_63 0x301140 + +LIST_HEAD(hisi_zip_list); +DEFINE_MUTEX(hisi_zip_list_lock); + +static const struct pci_device_id hisi_zip_dev_ids[] = { + { PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa250) }, + { 0, } +}; + +static inline void hisi_zip_add_to_list(struct hisi_zip *hisi_zip) +{ + mutex_lock(&hisi_zip_list_lock); + list_add_tail(&hisi_zip->list, &hisi_zip_list); + mutex_unlock(&hisi_zip_list_lock); +} + +static inline void hisi_zip_remove_from_list(struct hisi_zip *hisi_zip) +{ + mutex_lock(&hisi_zip_list_lock); + list_del(&hisi_zip->list); + mutex_unlock(&hisi_zip_list_lock); +} + +static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip) +{ + u32 val; + + /* qm user domain */ + writel(0x40001070, hisi_zip->qm.io_base + QM_ARUSER_M_CFG_1); + writel(0xfffffffe, hisi_zip->qm.io_base + QM_ARUSER_M_CFG_ENABLE); + writel(0x40001070, hisi_zip->qm.io_base + QM_AWUSER_M_CFG_1); + writel(0xfffffffe, hisi_zip->qm.io_base + QM_AWUSER_M_CFG_ENABLE); + writel(0xffffffff, hisi_zip->qm.io_base + QM_WUSER_M_CFG_ENABLE); + + val = readl(hisi_zip->qm.io_base + QM_PEH_AXUSER_CFG); + val |= (1 << 11); + writel(val, hisi_zip->qm.io_base + QM_PEH_AXUSER_CFG); + + /* qm cache */ + writel(0xffff, hisi_zip->qm.io_base + QM_AXI_M_CFG); + writel(0xffffffff, hisi_zip->qm.io_base + QM_AXI_M_CFG_ENABLE); + writel(0xffffffff, hisi_zip->qm.io_base + QM_PEH_AXUSER_CFG_ENABLE); + + /* cache */ + writel(0xffffffff, hisi_zip->qm.io_base + HZIP_PORT_ARCA_CHE_0); + writel(0xffffffff, hisi_zip->qm.io_base + HZIP_PORT_ARCA_CHE_1); + writel(0xffffffff, hisi_zip->qm.io_base + HZIP_PORT_AWCA_CHE_0); + writel(0xffffffff, hisi_zip->qm.io_base + HZIP_PORT_AWCA_CHE_1); + + /* user domain configurations */ + writel(0x40001070, hisi_zip->qm.io_base + HZIP_BD_RUSER_32_63); + writel(0x40001070, hisi_zip->qm.io_base + HZIP_SGL_RUSER_32_63); +#ifdef CONFIG_IOMMU_SVA + writel(0x40001071, hisi_zip->qm.io_base + HZIP_DATA_RUSER_32_63); + writel(0x40001071, hisi_zip->qm.io_base + HZIP_DATA_WUSER_32_63); +#else + writel(0x40001070, hisi_zip->qm.io_base + HZIP_DATA_RUSER_32_63); + writel(0x40001070, hisi_zip->qm.io_base + HZIP_DATA_WUSER_32_63); +#endif + writel(0x40001070, hisi_zip->qm.io_base + HZIP_BD_WUSER_32_63); + + /* fsm count */ + writel(0xfffffff, hisi_zip->qm.io_base + HZIP_FSM_MAX_CNT); + + /* clock gating, core, decompress verify enable */ + writel(0x10005, hisi_zip->qm.io_base + 0x301004); +} + +static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id) +{ + struct hisi_zip *hisi_zip; + struct qm_info *qm; + int ret; + u8 rev_id; + + hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL); + if (!hisi_zip) + return -ENOMEM; + hisi_zip_add_to_list(hisi_zip); + pci_set_drvdata(pdev, hisi_zip); + + qm = &hisi_zip->qm; + qm->pdev = pdev; + + pci_read_config_byte(pdev, PCI_REVISION_ID, &rev_id); + if (rev_id == 0x20) + qm->ver = QM_HW_V1; + else if (rev_id == 0x21) + qm->ver = QM_HW_V2; + qm->sqe_size = HZIP_SQE_SIZE; + + ret = hisi_qm_init(qm, ZIP); + if (ret) + goto err_with_hisi_zip; + + if (pdev->is_physfn) { + ret = hisi_qm_mem_start(qm); + if (ret) { + dev_err(&pdev->dev, "Can't start QM mem of Hisilicon!\n"); + goto err_with_qm_init; + } + + hisi_zip_set_user_domain_and_cache(hisi_zip); + + qm->qp_base = HZIP_PF_DEF_Q_BASE; + qm->qp_num = HZIP_PF_DEF_Q_NUM; + qm->free_qp = qm->qp_num; + } + ret = hisi_qm_start(qm); + if (ret) { + dev_err(&pdev->dev, "Can't start QM of Hisilicon!\n"); + goto err_with_qm_init; + } + + return 0; + +err_with_qm_init: + hisi_qm_uninit(qm); +err_with_hisi_zip: + hisi_zip_remove_from_list(hisi_zip); + kfree(hisi_zip); + return ret; +} + +static void hisi_zip_remove(struct pci_dev *pdev) +{ + struct hisi_zip *hisi_zip = pci_get_drvdata(pdev); + struct qm_info *qm = &hisi_zip->qm; + + hisi_qm_stop(qm); + hisi_qm_uninit(qm); + hisi_zip_remove_from_list(hisi_zip); + kfree(hisi_zip); +} + +static struct pci_driver hisi_zip_pci_driver = { + .name = "hisi_zip", + .id_table = hisi_zip_dev_ids, + .probe = hisi_zip_probe, + .remove = hisi_zip_remove, +}; + +static int __init hisi_zip_init(void) +{ + int ret; + + ret = pci_register_driver(&hisi_zip_pci_driver); + if (ret < 0) { + pr_err("zip: can't register hisi zip driver.\n"); + return ret; + } + + ret = hisi_zip_register_to_crypto(); + if (ret < 0) { + pr_err("zip: can't register hisi zip to crypto.\n"); + pci_unregister_driver(&hisi_zip_pci_driver); + return ret; + } + + return 0; +} + +static void __exit hisi_zip_exit(void) +{ + hisi_zip_unregister_from_crypto(); + pci_unregister_driver(&hisi_zip_pci_driver); +} + +module_init(hisi_zip_init); +module_exit(hisi_zip_exit); + +MODULE_LICENSE("GPL v2"); +MODULE_AUTHOR("Zhou Wang "); +MODULE_DESCRIPTION("Driver for HiSilicon ZIP accelerator"); +MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids); -- GitLab