Add Hi1620 ZIP&HPRE drivers with common QM driver, which are adpated to Crypto and SPIMDEV.

Feature or Bugfix:Feature
Signed-off-by: Nxuzaibo <xuzaibo@huawei.com>

drivers/crypto/hisilicon/Kconfig

@@ -12,3 +12,36 @@ config CRYPTO_DEV_HISI_SEC
 
 	  To compile this as a module, choose M here: the module
 	  will be called hisi_sec.
+
+config CRYPTO_DEV_HISILICON
+	tristate "Support for HISILICON CRYPTO ACCELERATOR"
+	help
+	  Enable this to use Hisilicon Hardware Accelerators
+
+config CRYPTO_DEV_HISI_SPIMDEV
+	bool "Enable SPIMDEV interface"
+	depends on CRYPTO_DEV_HISILICON
+	select VFIO_SPIMDEV
+	help
+	  Enable this enable the SPIMDEV, "shared parent IOMMU Mediated Device"
+	  interface for all Hisilicon accelerators if they can. The SPIMDEV
+	  enable the WarpDrive user space accelerator driver to access the
+	  hardware function directly.
+
+config CRYPTO_DEV_HISI_QM
+	tristate
+	depends on ARM64 && PCI
+
+config CRYPTO_DEV_HISI_ZIP
+	tristate "Support for HISI ZIP Driver"
+	depends on ARM64 && CRYPTO_DEV_HISILICON
+	select CRYPTO_DEV_HISI_QM
+	help
+	  Support for HiSilicon HIP08 ZIP Driver
+
+config CRYPTO_DEV_HISI_HPRE
+	tristate "Support for HISI HPRE Driver"
+	depends on ARM64 && CRYPTO_DEV_HISILICON
+	select CRYPTO_DEV_HISI_QM
+	help
+	  Support for HiSilicon HIP08 HPRE Driver
\ No newline at end of file

drivers/crypto/hisilicon/Makefile

 # SPDX-License-Identifier: GPL-2.0
+obj-$(CONFIG_CRYPTO_DEV_HISI_QM) += qm.o
+obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += zip/
+obj-$(CONFIG_CRYPTO_DEV_HISI_HPRE) += hpre/
 obj-$(CONFIG_CRYPTO_DEV_HISI_SEC) += sec/

drivers/crypto/hisilicon/hpre/Makefile

+obj-$(CONFIG_CRYPTO_DEV_HISI_HPRE) += hisi_hpre.o
+hisi_hpre-objs = hpre_main.o hpre_crypto.o

drivers/crypto/hisilicon/hpre/hpre.h

+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef __HISI_HPRE_H
+#define __HISI_HPRE_H
+
+#include <linux/list.h>
+#include "../qm.h"
+
+#define HPRE_SQE_SIZE			64
+#define HPRE_SQ_SIZE			(HPRE_SQE_SIZE * QM_Q_DEPTH)
+#define QM_CQ_SIZE			(QM_CQE_SIZE * QM_Q_DEPTH)
+#define HPRE_PF_DEF_Q_NUM		64
+#define HPRE_PF_DEF_Q_BASE		0
+
+struct hisi_hpre {
+	struct qm_info qm;
+	struct list_head list;
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+	struct vfio_spimdev *spimdev;
+#endif
+};
+
+enum hisi_hpre_alg_type {
+	HPRE_ALG_NC_NCRT = 0x0,
+	HPRE_ALG_NC_CRT = 0x1,
+	HPRE_ALG_KG_STD = 0x2,
+	HPRE_ALG_KG_CRT = 0x3,
+	HPRE_ALG_DH_G2 = 0x4,
+	HPRE_ALG_DH = 0x5,
+	HPRE_ALG_PRIME = 0x6,
+	HPRE_ALG_MOD = 0x7,
+	HPRE_ALG_MOD_INV = 0x8,
+	HPRE_ALG_MUL = 0x9,
+	HPRE_ALG_COPRIME = 0xA
+};
+
+struct hisi_hpre_sqe {
+	__u32 alg	: 5;
+
+	/* error type */
+	__u32 etype	:11;
+	__u32 resv0	: 14;
+	__u32 done	: 2;
+	__u32 task_len1	: 8;
+	__u32 task_len2	: 8;
+	__u32 mrttest_num	: 8;
+	__u32 resv1	: 8;
+	__u32 low_key;
+	__u32 hi_key;
+	__u32 low_in;
+	__u32 hi_in;
+	__u32 low_out;
+	__u32 hi_out;
+	__u32 tag	:16;
+	__u32 resv2	:16;
+	__u32 rsvd1[7];
+};
+
+extern struct list_head hisi_hpre_list;
+
+extern int hpre_algs_register(void);
+extern void hpre_algs_unregister(void);
+
+#endif

drivers/crypto/hisilicon/hpre/hpre_crypto.c

+// SPDX-License-Identifier: GPL-2.0+
+#include <linux/module.h>
+#include <crypto/internal/rsa.h>
+#include <crypto/internal/akcipher.h>
+#include <crypto/akcipher.h>
+#include <crypto/kpp.h>
+#include <crypto/internal/kpp.h>
+#include <crypto/dh.h>
+#include <linux/dma-mapping.h>
+#include <linux/fips.h>
+#include <crypto/scatterwalk.h>
+
+#include "hpre.h"
+
+static DEFINE_MUTEX(algs_lock);
+static unsigned int active_devs;
+
+struct hpre_ctx;
+
+#define GET_DEV(ctx)		((ctx)->qp->qm->pdev->dev)
+
+#define BITS64_MERGE(low_32, high_32)	(((u64)(low_32)) | \
+					(((u64)(high_32)) << 32))
+typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe);
+
+struct _rsa_ctx {
+	/* low address: e--->n */
+	char *pubkey;
+
+	/* low address: d--->n */
+	char *prikey;
+
+	/* low address: dq->dp->q->p->qinv */
+	char *crt_prikey;
+	dma_addr_t dma_pubkey;
+	dma_addr_t dma_prikey;
+	dma_addr_t dma_crt_prikey;
+	struct crypto_akcipher *soft_tfm;
+};
+
+struct _dh_ctx {
+	/*
+	 * If base is g we compute the public key
+	 *	ya = g^xa mod p; [RFC2631 sec 2.1.1]
+	 * else if base if the counterpart public key we
+	 * compute the shared secret
+	 *	ZZ = yb^xa mod p; [RFC2631 sec 2.1.1]
+	 */
+	char *xa_p; /* low address: d--->n */
+	char *g; /* m */
+	dma_addr_t dma_xa_p;
+	dma_addr_t dma_g;
+};
+
+struct hpre_ctx {
+	struct hisi_qp *qp;
+	struct hpre_asym_request **req_list;
+	unsigned long *req_bitmap;
+	spinlock_t req_lock;
+	unsigned int key_sz;
+	bool crt_g2_mode;
+	union {
+		struct _rsa_ctx rsa;
+		struct _dh_ctx dh;
+		u64 resv[7];
+	};
+};
+
+struct hpre_asym_request {
+	char *src_align;
+	char *dst_align;
+	struct hisi_hpre_sqe req;
+	struct hpre_ctx *ctx;
+	union {
+		struct akcipher_request *rsa;
+		struct kpp_request *dh;
+	} areq;
+	int err;
+	int req_id;
+	hpre_cb cb;
+};
+
+void hpre_bn_format(unsigned char *buf, int len)
+{
+	int i = len - 1, j;
+
+
+	while (!buf[i] && i >= 0)
+		i--;
+	if (i == len - 1)
+		return;
+	for (j = len - 1; j >= 0; j--, i--) {
+		if (i >= 0)
+			buf[j] = buf[i];
+		else
+			buf[j] = 0;
+	}
+}
+
+static struct hisi_hpre *find_hpre_device(int node)
+{
+	struct hisi_hpre *hisi_hpre, *ret = NULL;
+	struct device *dev;
+	int min_distance = 100;
+	int dev_node = 0;
+
+	list_for_each_entry(hisi_hpre, &hisi_hpre_list, list) {
+		dev = &hisi_hpre->qm.pdev->dev;
+#ifdef CONFIG_NUMA
+		dev_node = dev->numa_node;
+#endif
+		if (node_distance(dev_node, node) < min_distance) {
+			ret = hisi_hpre;
+			min_distance = node_distance(dev_node, node);
+		}
+	}
+
+	return ret;
+}
+
+static int hpre_alloc_req_id(struct hpre_ctx *ctx)
+{
+	int id;
+	unsigned long flags;
+
+	spin_lock_irqsave(&ctx->req_lock, flags);
+	id = find_first_zero_bit(ctx->req_bitmap, QM_Q_DEPTH);
+	if (id >= QM_Q_DEPTH) {
+		spin_unlock_irqrestore(&ctx->req_lock, flags);
+		pr_err("\nno free req id!");
+		return -EBUSY;
+	}
+	set_bit(id, ctx->req_bitmap);
+	spin_unlock_irqrestore(&ctx->req_lock, flags);
+
+	return id;
+}
+
+static void hpre_free_req_id(struct hpre_ctx *ctx, int req_id)
+{
+	unsigned long flags;
+
+	spin_lock_irqsave(&ctx->req_lock, flags);
+	bitmap_clear(ctx->req_bitmap, req_id, 1);
+	spin_unlock_irqrestore(&ctx->req_lock, flags);
+}
+
+static int hpre_add_req_to_ctx(struct hpre_asym_request *hpre_req)
+{
+	struct hpre_ctx *ctx;
+	int id;
+
+	ctx = hpre_req->ctx;
+	id = hpre_alloc_req_id(ctx);
+	if (id < 0)
+		return -EINVAL;
+
+	ctx->req_list[id] = hpre_req;
+	hpre_req->req_id = id;
+
+	return id;
+}
+
+static void hpre_rm_req_from_ctx(struct hpre_asym_request *hpre_req)
+{
+	int id = hpre_req->req_id;
+	struct hpre_ctx *ctx = hpre_req->ctx;
+
+	ctx->req_list[id] = NULL;
+	hpre_free_req_id(ctx, id);
+}
+
+static struct hisi_qp *hpre_get_qp(void)
+{
+	struct hisi_qp *qp = NULL;
+	struct hisi_hpre *hpre;
+	int ret;
+
+	/* find the proper hpre device */
+	hpre = find_hpre_device(cpu_to_node(smp_processor_id()));
+	if (!hpre) {
+		pr_err("Can not find proper hpre device!\n");
+		return ERR_PTR(-ENODEV);
+	}
+	qp = hisi_qm_create_qp(&hpre->qm, 0);
+	if (!qp) {
+		dev_err(&hpre->qm.pdev->dev, "Can not create qp!\n");
+		return ERR_PTR(-ENODEV);
+	}
+	ret = hisi_qm_start_qp(qp, 0);
+	if (ret) {
+		hisi_qm_release_qp(qp);
+		dev_err(&hpre->qm.pdev->dev, "Can not start qp!\n");
+		return ERR_PTR(-EINVAL);
+	}
+
+	return qp;
+}
+
+static int _hw_data_init(struct hpre_asym_request *hpre_req,
+			struct scatterlist *data, unsigned int len,
+			int is_src)
+{
+	struct hisi_hpre_sqe *msg = &hpre_req->req;
+	struct hpre_ctx *ctx = hpre_req->ctx;
+	struct device *dev = &GET_DEV(ctx);
+	enum dma_data_direction dma_dir;
+	dma_addr_t tmp;
+	char *ptr;
+
+	if (sg_is_last(data) && len == ctx->key_sz) {
+		if (is_src) {
+			hpre_req->src_align = NULL;
+			dma_dir = DMA_TO_DEVICE;
+		} else {
+			hpre_req->dst_align = NULL;
+			dma_dir = DMA_FROM_DEVICE;
+		}
+		tmp = dma_map_single(dev, sg_virt(data),
+				len, dma_dir);
+		if (unlikely(dma_mapping_error(dev, tmp))) {
+			dev_err(dev, "\ndma map data err!");
+			return -ENOMEM;
+		}
+	} else {
+		int shift = ctx->key_sz - len;
+
+		ptr = dma_zalloc_coherent(dev, ctx->key_sz, &tmp, GFP_KERNEL);
+		if (unlikely(!ptr)) {
+			dev_err(dev, "\ndma alloc data err!");
+			return -ENOMEM;
+		}
+		if (is_src) {
+			scatterwalk_map_and_copy(ptr + shift, data, 0, len, 0);
+			hpre_req->src_align = ptr;
+		} else {
+			hpre_req->dst_align = ptr;
+		}
+	}
+	if (is_src) {
+		msg->low_in = lower_32_bits(tmp);
+		msg->hi_in = upper_32_bits(tmp);
+	} else {
+		msg->low_out = lower_32_bits(tmp);
+		msg->hi_out = upper_32_bits(tmp);
+	}
+
+	return 0;
+}
+
+static void _hw_data_clr_all(struct hpre_ctx *ctx,
+			struct hpre_asym_request *req,
+			struct scatterlist *dst, struct scatterlist *src)
+{
+	dma_addr_t tmp;
+	struct device *dev = &GET_DEV(ctx);
+	struct hisi_hpre_sqe *sqe = &req->req;
+
+	tmp = BITS64_MERGE(sqe->low_in, sqe->hi_in);
+	if (src && tmp) {
+		if (req->src_align)
+			dma_free_coherent(dev, ctx->key_sz,
+					  req->src_align, tmp);
+		else
+			dma_unmap_single(dev, tmp,
+					 ctx->key_sz, DMA_TO_DEVICE);
+	}
+	tmp = BITS64_MERGE(sqe->low_out, sqe->hi_out);
+	if (req->dst_align && tmp) {
+		if (dst)
+			scatterwalk_map_and_copy(req->dst_align, dst, 0,
+					 ctx->key_sz, 1);
+		dma_free_coherent(dev, ctx->key_sz, req->dst_align, tmp);
+	} else {
+		dma_unmap_single(dev, tmp, ctx->key_sz, DMA_FROM_DEVICE);
+	}
+}
+
+static int _alg_res_post_hf(struct hpre_ctx *ctx, struct hisi_hpre_sqe *sqe,
+			    void **kreq)
+{
+	struct hpre_asym_request *req;
+	int err, id;
+
+	id = (int)sqe->tag;
+	req = ctx->req_list[id];
+	hpre_rm_req_from_ctx(req);
+	*kreq = req;
+	err = sqe->etype;
+	err = (err == 0 && sqe->done == 3) ? 0 : -EINVAL;
+
+	return err;
+}
+
+static int _ctx_init(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen)
+{
+	int ret = -ENOMEM;
+
+	if (!ctx || !qp || qlen < 0)
+		return -EINVAL;
+
+	spin_lock_init(&ctx->req_lock);
+	ctx->req_bitmap = kcalloc(BITS_TO_LONGS(qlen), sizeof(long),
+			GFP_KERNEL);
+	if (!ctx->req_bitmap)
+		return ret;
+	ctx->qp = qp;
+	ctx->req_list = kcalloc(qlen, sizeof(void *), GFP_KERNEL);
+	if (!ctx->req_list) {
+		kfree(ctx->req_bitmap);
+		return ret;
+	}
+	ctx->key_sz = 0;
+	ctx->crt_g2_mode = false;
+
+	return 0;
+}
+
+static void _ctx_clear(struct hpre_ctx *ctx, int is_exit)
+{
+	if (is_exit) {
+		kfree(ctx->req_bitmap);
+		kfree(ctx->req_list);
+		hisi_qm_release_qp(ctx->qp);
+	}
+
+	ctx->crt_g2_mode = false;
+	ctx->key_sz = 0;
+}
+
+static void _dh_cb(struct hpre_ctx *ctx, void *resp)
+{
+	struct kpp_request *areq;
+	struct hpre_asym_request *req;
+	int ret;
+
+	ret = _alg_res_post_hf(ctx, resp, (void **)&req);
+	areq = req->areq.dh;
+	areq->dst_len = ctx->key_sz;
+	_hw_data_clr_all(ctx, req, areq->dst, areq->src);
+	kpp_request_complete(areq, ret);
+}
+
+void hpre_alg_cb(struct hisi_qp *qp, void *_resp)
+{
+	struct hisi_hpre_sqe *sqe = _resp;
+	struct hpre_asym_request *areq;
+	struct hpre_ctx *ctx = qp->qp_ctx;
+	int id = (int)sqe->tag;
+
+	areq = ctx->req_list[id];
+	areq->cb(ctx, _resp);
+}
+
+static int hpre_ctx_init(struct hpre_ctx *ctx)
+{
+	struct hisi_qp *qp;
+
+	qp = hpre_get_qp();
+	if (IS_ERR(qp))
+		return PTR_ERR(qp);
+	qp->qp_ctx = ctx;
+	qp->req_cb = hpre_alg_cb;
+
+	return _ctx_init(ctx, qp, QM_Q_DEPTH);
+}
+
+static int hpre_dh_compute_value(struct kpp_request *req)
+{
+	struct crypto_kpp *tfm = crypto_kpp_reqtfm(req);
+	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+	struct hpre_asym_request *hpre_req =
+			PTR_ALIGN(kpp_request_ctx(req), 64);
+	struct hisi_hpre_sqe *msg = &hpre_req->req;
+	int ret = -ENOMEM, ctr = 0, req_id;
+
+	if (unlikely(!ctx->dh.xa_p))
+		return -EINVAL;
+
+	if (req->dst_len < ctx->key_sz) {
+		req->dst_len = ctx->key_sz;
+		return -EOVERFLOW;
+	}
+	memset(msg, '\0', sizeof(*msg));
+
+	msg->done = 1;
+	hpre_req->cb = _dh_cb;
+	hpre_req->ctx = ctx;
+	hpre_req->areq.dh = req;
+	msg->task_len1 = (ctx->key_sz >> 3) - 1;
+
+	msg->low_key = lower_32_bits(ctx->dh.dma_xa_p);
+	msg->hi_key = upper_32_bits(ctx->dh.dma_xa_p);
+	if (!req->src && !ctx->crt_g2_mode) {
+		msg->low_in = lower_32_bits(ctx->dh.dma_g);
+		msg->hi_in = upper_32_bits(ctx->dh.dma_g);
+	}
+
+	if (req->src) {
+		(void)hpre_bn_format(sg_virt(req->src), ctx->key_sz);
+		ret = _hw_data_init(hpre_req, req->src, req->src_len, 1);
+		if (ret)
+			return ret;
+	}
+	ret = _hw_data_init(hpre_req, req->dst, req->dst_len, 0);
+	if (ret) {
+		_hw_data_clr_all(ctx, hpre_req, NULL, req->src);
+		return ret;
+	}
+	if (ctx->crt_g2_mode && !req->src)
+		msg->alg = HPRE_ALG_DH_G2;
+	else
+		msg->alg = HPRE_ALG_DH;
+	req_id = hpre_add_req_to_ctx(hpre_req);
+	if (req_id < 0) {
+		ret = -EBUSY;
+		goto clear_all;
+	}
+	msg->tag = (u16)req_id;
+
+	do {
+		ret = hisi_qp_send(ctx->qp, (void *)msg);
+	} while (ret == -EBUSY && ctr++ < 100);
+
+	if (!ret)
+		return -EINPROGRESS;
+
+	hpre_rm_req_from_ctx(hpre_req);
+clear_all:
+	_hw_data_clr_all(ctx, hpre_req, NULL, req->src);
+	return ret;
+}
+
+static int hpre_dh_check_params_length(unsigned int key_sz)
+{
+	switch (key_sz) {
+	case 768:
+	case 1024:
+	case 1536:
+	case 2048:
+	case 3072:
+	case 4096:
+		return 0;
+	}
+	return -EINVAL;
+}
+
+static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params)
+{
+	struct device *dev = &GET_DEV(ctx);
+	unsigned int sz;
+
+	if (hpre_dh_check_params_length(params->p_size << 3))
+		return -EINVAL;
+
+	sz = ctx->key_sz = params->p_size;
+	ctx->dh.xa_p = dma_zalloc_coherent(dev, sz << 1,
+				&ctx->dh.dma_xa_p, GFP_KERNEL);
+	if (!ctx->dh.xa_p)
+		return -ENOMEM;
+	memcpy(ctx->dh.xa_p + sz, params->p, sz);
+	hpre_bn_format((unsigned char *)ctx->dh.xa_p + sz, sz);
+
+	/* If g equals 2 don't copy it */
+	if (params->g_size == 1 && *(char *)params->g == 0x02) {
+		ctx->crt_g2_mode = true;
+		return 0;
+	}
+
+	ctx->dh.g = dma_zalloc_coherent(dev, sz, &ctx->dh.dma_g, GFP_KERNEL);
+	if (!ctx->dh.g)
+		return -ENOMEM;
+	memcpy(ctx->dh.g + (sz - params->g_size), params->g,
+	       params->g_size);
+	hpre_bn_format(ctx->dh.g, ctx->key_sz);
+
+	return 0;
+}
+
+static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, int is_exit)
+{
+	unsigned int sz = ctx->key_sz;
+	struct device *dev = &GET_DEV(ctx);
+
+	if (ctx->dh.g) {
+		dma_free_coherent(dev, sz, ctx->dh.g, ctx->dh.dma_g);
+		ctx->dh.g = NULL;
+	}
+	if (ctx->dh.xa_p) {
+		dma_free_coherent(dev, sz << 1, ctx->dh.xa_p,
+				ctx->dh.dma_xa_p);
+		ctx->dh.xa_p = NULL;
+	}
+	_ctx_clear(ctx, is_exit);
+}
+
+static int hpre_dh_set_secret(struct crypto_kpp *tfm, const void *buf,
+			     unsigned int len)
+{
+	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+	struct dh params;
+	int ret;
+
+	if (crypto_dh_decode_key(buf, len, &params) < 0)
+		return -EINVAL;
+
+	/* Free old secret if any */
+	hpre_dh_clear_ctx(ctx, 0);
+
+	ret = hpre_dh_set_params(ctx, &params);
+	if (ret < 0)
+		goto err_clear_ctx;
+
+	memcpy(ctx->dh.xa_p + (ctx->key_sz - params.key_size), params.key,
+		params.key_size);
+	hpre_bn_format((unsigned char *)ctx->dh.xa_p, ctx->key_sz);
+
+	return 0;
+
+err_clear_ctx:
+	hpre_dh_clear_ctx(ctx, 0);
+	return ret;
+}
+
+static unsigned int hpre_dh_max_size(struct crypto_kpp *tfm)
+{
+	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+
+	return ctx->key_sz;
+}
+
+static int hpre_dh_init_tfm(struct crypto_kpp *tfm)
+{
+	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+
+	return hpre_ctx_init(ctx);
+}
+
+static void hpre_dh_exit_tfm(struct crypto_kpp *tfm)
+{
+	struct hpre_ctx *ctx = kpp_tfm_ctx(tfm);
+
+	hpre_dh_clear_ctx(ctx, 1);
+}
+
+static void _rsa_cb(struct hpre_ctx *ctx, void *resp)
+{
+	struct akcipher_request *areq;
+	struct hpre_asym_request *req;
+	int ret;
+
+	ret = _alg_res_post_hf(ctx, resp, (void **)&req);
+	areq = req->areq.rsa;
+	areq->dst_len = ctx->key_sz;
+	_hw_data_clr_all(ctx, req, areq->dst, areq->src);
+	akcipher_request_complete(areq, ret);
+}
+
+static unsigned long hpre_rsa_key_size_check(unsigned int len)
+{
+	unsigned int bitslen = len << 3;
+
+	switch (bitslen) {
+	/* 512bits is not supported by HPRE now! */
+	case 512:
+	case 1024:
+	case 2048:
+	case 3072:
+	case 4096:
+		return 0;
+	default:
+		return -1;
+	};
+}
+
+static int hpre_rsa_enc(struct akcipher_request *req)
+{
+	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
+	struct hpre_asym_request *hpre_req =
+			PTR_ALIGN(akcipher_request_ctx(req), 64);
+	struct hisi_hpre_sqe *msg = &hpre_req->req;
+	int ret = -ENOMEM, ctr = 0, req_id;
+
+	if (ctx->key_sz == 64 && ctx->rsa.soft_tfm) {
+		akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
+		ret = crypto_akcipher_encrypt(req);
+		akcipher_request_set_tfm(req, tfm);
+		return ret;
+	}
+	if (unlikely(!ctx->rsa.pubkey))
+		return -EINVAL;
+
+	if (req->dst_len < ctx->key_sz) {
+		req->dst_len = ctx->key_sz;
+		return -EOVERFLOW;
+	}
+	memset(msg, '\0', sizeof(*msg));
+	msg->done = 1;
+	msg->task_len1 = (ctx->key_sz >> 3) - 1;
+	msg->low_key = lower_32_bits(ctx->rsa.dma_pubkey);
+	msg->hi_key = upper_32_bits(ctx->rsa.dma_pubkey);
+	hpre_req->cb = _rsa_cb;
+	hpre_req->ctx = ctx;
+	hpre_req->areq.rsa = req;
+	ret = _hw_data_init(hpre_req, req->src, req->src_len, 1);
+	if (ret)
+		return ret;
+	ret = _hw_data_init(hpre_req, req->dst, req->dst_len, 0);
+	if (ret) {
+		_hw_data_clr_all(ctx, hpre_req, NULL, req->src);
+		return ret;
+	}
+	msg->alg = HPRE_ALG_NC_NCRT;
+	req_id = hpre_add_req_to_ctx(hpre_req);
+	if (req_id < 0) {
+		ret = -EBUSY;
+		goto clear_all;
+	}
+	msg->tag = (u16)req_id;
+	do {
+		ret = hisi_qp_send(ctx->qp, (void *)msg);
+	} while (ret == -EBUSY && ctr++ < 100);
+
+	if (!ret)
+		return -EINPROGRESS;
+
+	hpre_rm_req_from_ctx(hpre_req);
+clear_all:
+	_hw_data_clr_all(ctx, hpre_req, NULL, req->src);
+	return ret;
+}
+
+static int hpre_rsa_dec(struct akcipher_request *req)
+{
+	struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req);
+	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
+	struct hpre_asym_request *hpre_req =
+			PTR_ALIGN(akcipher_request_ctx(req), 64);
+	struct hisi_hpre_sqe *msg = &hpre_req->req;
+	int ret = -ENOMEM, ctr = 0, req_id;
+
+	if (ctx->key_sz == 64 && ctx->rsa.soft_tfm) {
+		akcipher_request_set_tfm(req, ctx->rsa.soft_tfm);
+		ret = crypto_akcipher_decrypt(req);
+		akcipher_request_set_tfm(req, tfm);
+		return ret;
+	}
+	if (unlikely(!ctx->rsa.prikey))
+		return -EINVAL;
+
+	if (req->dst_len < ctx->key_sz) {
+		req->dst_len = ctx->key_sz;
+		return -EOVERFLOW;
+	}
+	memset(msg, '\0', sizeof(*msg));
+	msg->task_len1 = (ctx->key_sz >> 3) - 1;
+	msg->done = 1;
+	if (ctx->crt_g2_mode) {
+		msg->low_key = lower_32_bits(ctx->rsa.dma_crt_prikey);
+		msg->hi_key = upper_32_bits(ctx->rsa.dma_crt_prikey);
+	} else {
+		msg->low_key = lower_32_bits(ctx->rsa.dma_prikey);
+		msg->hi_key = upper_32_bits(ctx->rsa.dma_prikey);
+	}
+
+	hpre_req->cb = _rsa_cb;
+	hpre_req->ctx = ctx;
+	hpre_req->areq.rsa = req;
+	ret = _hw_data_init(hpre_req, req->src, req->src_len, 1);
+	if (ret)
+		return ret;
+	ret = _hw_data_init(hpre_req, req->dst, req->dst_len, 0);
+	if (ret) {
+		_hw_data_clr_all(ctx, hpre_req, NULL, req->src);
+		return ret;
+	}
+	if (ctx->crt_g2_mode)
+		msg->alg = HPRE_ALG_NC_CRT;
+	else
+		msg->alg = HPRE_ALG_NC_NCRT;
+	req_id = hpre_add_req_to_ctx(hpre_req);
+	if (req_id < 0) {
+		ret = -EBUSY;
+		goto clear_all;
+	}
+	msg->tag = (u16)req_id;
+	do {
+		ret = hisi_qp_send(ctx->qp, (void *)msg);
+	} while (ret == -EBUSY && ctr++ < 100);
+
+	if (!ret)
+		return -EINPROGRESS;
+
+	hpre_rm_req_from_ctx(hpre_req);
+clear_all:
+	_hw_data_clr_all(ctx, hpre_req, NULL, req->src);
+	return ret;
+}
+
+static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value,
+			 size_t vlen, bool private)
+{
+	const char *ptr = value;
+	int ret = -EINVAL;
+
+	while (!*ptr && vlen) {
+		ptr++;
+		vlen--;
+	}
+	ctx->key_sz = vlen;
+
+	/* invalid key size provided */
+	if (hpre_rsa_key_size_check(ctx->key_sz))
+		goto err;
+	if (private) {
+		ctx->rsa.prikey = dma_zalloc_coherent(&GET_DEV(ctx),
+						  vlen << 1,
+						  &ctx->rsa.dma_prikey,
+						  GFP_KERNEL);
+		if (!ctx->rsa.prikey)
+			return -ENOMEM;
+	}
+	ctx->rsa.pubkey = dma_zalloc_coherent(&GET_DEV(ctx),
+					  vlen << 1,
+					  &ctx->rsa.dma_pubkey,
+					  GFP_KERNEL);
+	if (!ctx->rsa.pubkey)
+		return -ENOMEM;
+	memcpy(ctx->rsa.pubkey + vlen, ptr, vlen);
+	hpre_bn_format((unsigned char *)ctx->rsa.pubkey + vlen, vlen);
+	if (ctx->rsa.prikey) {
+		memcpy(ctx->rsa.prikey + vlen, ptr, vlen);
+		hpre_bn_format((unsigned char *)ctx->rsa.prikey + vlen, vlen);
+	}
+
+	return 0;
+err:
+	ctx->key_sz = 0;
+
+	return ret;
+}
+
+static int hpre_rsa_set_e(struct hpre_ctx *ctx, const char *value,
+			  size_t vlen)
+{
+	const char *ptr = value;
+
+	while (!*ptr && vlen) {
+		ptr++;
+		vlen--;
+	}
+	if (!ctx->key_sz || !vlen || vlen > ctx->key_sz) {
+		ctx->rsa.pubkey = NULL;
+		return -EINVAL;
+	}
+
+	memcpy(ctx->rsa.pubkey, ptr, vlen);
+	hpre_bn_format((unsigned char *)ctx->rsa.pubkey, ctx->key_sz);
+
+	return 0;
+}
+
+static int hpre_rsa_set_d(struct hpre_ctx *ctx, const char *value,
+			  size_t vlen)
+{
+	const char *ptr = value;
+	int ret = -EINVAL;
+
+	while (!*ptr && vlen) {
+		ptr++;
+		vlen--;
+	}
+	if (!ctx->key_sz || !vlen || vlen > ctx->key_sz)
+		goto err;
+
+	memcpy(ctx->rsa.prikey, ptr, vlen);
+	hpre_bn_format((unsigned char *)ctx->rsa.prikey, ctx->key_sz);
+	return 0;
+err:
+	ctx->rsa.prikey = NULL;
+	return ret;
+}
+
+static void hpre_rsa_drop_leading_zeros(const char **ptr, unsigned int *len)
+{
+	while (!**ptr && *len) {
+		(*ptr)++;
+		(*len)--;
+	}
+}
+
+static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key)
+{
+	struct device *dev = &GET_DEV(ctx);
+	unsigned int half_key_sz = ctx->key_sz / 2;
+	const char *ptr;
+	unsigned int len;
+	int ret = -EINVAL;
+
+	ctx->rsa.crt_prikey = dma_zalloc_coherent(dev, half_key_sz * 5,
+					      &ctx->rsa.dma_crt_prikey,
+					      GFP_KERNEL);
+	if (!ctx->rsa.crt_prikey)
+		return -ENOMEM;
+
+	/* dq */
+	ptr = rsa_key->dq;
+	len = rsa_key->dq_sz;
+	hpre_rsa_drop_leading_zeros(&ptr, &len);
+	if (!len)
+		goto free_key;
+
+	memcpy(ctx->rsa.crt_prikey, ptr, len);
+	hpre_bn_format((unsigned char *)ctx->rsa.crt_prikey, half_key_sz);
+
+	/* dp */
+	ptr = rsa_key->dp;
+	len = rsa_key->dp_sz;
+	hpre_rsa_drop_leading_zeros(&ptr, &len);
+	if (!len)
+		goto free_key;
+	memcpy(ctx->rsa.crt_prikey + half_key_sz, ptr, len);
+	hpre_bn_format((unsigned char *)ctx->rsa.crt_prikey + half_key_sz,
+		       half_key_sz);
+
+	/* q */
+	ptr = rsa_key->q;
+	len = rsa_key->q_sz;
+	hpre_rsa_drop_leading_zeros(&ptr, &len);
+	if (!len)
+		goto free_key;
+	memcpy(ctx->rsa.crt_prikey + (half_key_sz * 2), ptr, len);
+	hpre_bn_format((unsigned char *)ctx->rsa.crt_prikey + half_key_sz * 2,
+		       half_key_sz);
+
+	/* p */
+	ptr = rsa_key->p;
+	len = rsa_key->p_sz;
+	hpre_rsa_drop_leading_zeros(&ptr, &len);
+	if (!len)
+		goto free_key;
+	memcpy(ctx->rsa.crt_prikey + half_key_sz * 3, ptr, len);
+	hpre_bn_format((unsigned char *)ctx->rsa.crt_prikey + half_key_sz * 3,
+		       half_key_sz);
+
+	/* qinv */
+	ptr = rsa_key->qinv;
+	len = rsa_key->qinv_sz;
+	hpre_rsa_drop_leading_zeros(&ptr, &len);
+	if (!len)
+		goto free_key;
+	memcpy(ctx->rsa.crt_prikey + (half_key_sz * 4), ptr, len);
+	hpre_bn_format((unsigned char *)ctx->rsa.crt_prikey + half_key_sz * 4,
+		       half_key_sz);
+	ctx->crt_g2_mode = true;
+
+	return 0;
+free_key:
+	memset(ctx->rsa.crt_prikey + half_key_sz * 5, '\0', half_key_sz);
+	dma_free_coherent(dev, half_key_sz * 5, ctx->rsa.crt_prikey,
+			  ctx->rsa.dma_crt_prikey);
+	ctx->rsa.crt_prikey = NULL;
+
+	ctx->crt_g2_mode = false;
+
+	return ret;
+}
+
+static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, int is_exit)
+{
+	unsigned int half_key_sz = ctx->key_sz >> 1;
+	struct device *dev = &GET_DEV(ctx);
+
+	if (ctx->rsa.pubkey) {
+		dma_free_coherent(dev, ctx->key_sz << 1,
+				  ctx->rsa.pubkey, ctx->rsa.dma_pubkey);
+		ctx->rsa.pubkey = NULL;
+	}
+	if (ctx->rsa.crt_prikey) {
+		memset(ctx->rsa.crt_prikey, '\0', half_key_sz * 5);
+		dma_free_coherent(dev, half_key_sz * 5,
+				ctx->rsa.crt_prikey, ctx->rsa.dma_crt_prikey);
+		ctx->rsa.crt_prikey = NULL;
+	}
+	if (ctx->rsa.prikey) {
+		memset(ctx->rsa.prikey, '\0', ctx->key_sz);
+		dma_free_coherent(dev, ctx->key_sz << 1, ctx->rsa.prikey,
+				  ctx->rsa.dma_prikey);
+		ctx->rsa.prikey = NULL;
+	}
+
+	_ctx_clear(ctx, is_exit);
+}
+
+static int hpre_rsa_setkey(struct hpre_ctx *ctx, const void *key,
+			   unsigned int keylen, bool private)
+{
+	struct rsa_key rsa_key;
+	int ret;
+
+	hpre_rsa_clear_ctx(ctx, 0);
+
+	if (private)
+		ret = rsa_parse_priv_key(&rsa_key, key, keylen);
+	else
+		ret = rsa_parse_pub_key(&rsa_key, key, keylen);
+	if (ret < 0)
+		goto free;
+
+	ret = hpre_rsa_set_n(ctx, rsa_key.n, rsa_key.n_sz, private);
+	if (ret < 0)
+		goto free;
+
+	if (private) {
+		ret = hpre_rsa_set_d(ctx, rsa_key.d, rsa_key.d_sz);
+		if (ret < 0)
+			goto free;
+		hpre_rsa_setkey_crt(ctx, &rsa_key);
+	}
+	ret = hpre_rsa_set_e(ctx, rsa_key.e, rsa_key.e_sz);
+	if (ret < 0)
+		goto free;
+
+	if (!ctx->rsa.pubkey) {
+		/* invalid key provided */
+		ret = -EINVAL;
+		goto free;
+	}
+	if (private && !ctx->rsa.prikey) {
+		/* invalid private key provided */
+		ret = -EINVAL;
+		goto free;
+	}
+
+	return 0;
+free:
+	hpre_rsa_clear_ctx(ctx, 0);
+	return ret;
+}
+
+static int hpre_rsa_setpubkey(struct crypto_akcipher *tfm, const void *key,
+			      unsigned int keylen)
+{
+	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
+	int ret;
+
+	ret = crypto_akcipher_set_pub_key(ctx->rsa.soft_tfm, key, keylen);
+	if (ret)
+		return ret;
+	return hpre_rsa_setkey(ctx, key, keylen, false);
+}
+
+static int hpre_rsa_setprivkey(struct crypto_akcipher *tfm, const void *key,
+			       unsigned int keylen)
+{
+	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
+	int ret;
+
+	ret = crypto_akcipher_set_priv_key(ctx->rsa.soft_tfm, key, keylen);
+	if (ret)
+		return ret;
+	return hpre_rsa_setkey(ctx, key, keylen, true);
+}
+
+static unsigned int hpre_rsa_max_size(struct crypto_akcipher *tfm)
+{
+	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
+
+	if (ctx->rsa.soft_tfm && ctx->key_sz == 64)
+		return crypto_akcipher_maxsize(ctx->rsa.soft_tfm);
+	return ctx->key_sz;
+}
+
+static int hpre_rsa_init_tfm(struct crypto_akcipher *tfm)
+{
+	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
+
+	ctx->rsa.soft_tfm = crypto_alloc_akcipher("rsa-generic", 0, 0);
+	if (IS_ERR(ctx->rsa.soft_tfm)) {
+		pr_err("Can not alloc_akcipher!\n");
+		return PTR_ERR(tfm);
+	}
+
+	return hpre_ctx_init(ctx);
+}
+
+static void hpre_rsa_exit_tfm(struct crypto_akcipher *tfm)
+{
+	struct hpre_ctx *ctx = akcipher_tfm_ctx(tfm);
+
+	hpre_rsa_clear_ctx(ctx, 1);
+	if (ctx->rsa.soft_tfm)
+		crypto_free_akcipher(ctx->rsa.soft_tfm);
+}
+
+static struct akcipher_alg rsa = {
+	.encrypt = hpre_rsa_enc,
+	.decrypt = hpre_rsa_dec,
+	.sign = hpre_rsa_dec,
+	.verify = hpre_rsa_enc,
+	.set_pub_key = hpre_rsa_setpubkey,
+	.set_priv_key = hpre_rsa_setprivkey,
+	.max_size = hpre_rsa_max_size,
+	.init = hpre_rsa_init_tfm,
+	.exit = hpre_rsa_exit_tfm,
+	.reqsize = sizeof(struct hpre_asym_request) + 64,
+	.base = {
+		.cra_name = "rsa",
+		.cra_driver_name = "hpre-rsa",
+		.cra_priority = 1000,
+		.cra_module = THIS_MODULE,
+		.cra_ctxsize = sizeof(struct hpre_ctx),
+	},
+};
+
+static struct kpp_alg dh = {
+	.set_secret = hpre_dh_set_secret,
+	.generate_public_key = hpre_dh_compute_value,
+	.compute_shared_secret = hpre_dh_compute_value,
+	.max_size = hpre_dh_max_size,
+	.init = hpre_dh_init_tfm,
+	.exit = hpre_dh_exit_tfm,
+	.reqsize = sizeof(struct hpre_asym_request) + 64,
+	.base = {
+		.cra_name = "dh",
+		.cra_driver_name = "hpre-dh",
+		.cra_priority = 1000,
+		.cra_module = THIS_MODULE,
+		.cra_ctxsize = sizeof(struct hpre_ctx),
+	},
+};
+
+int hpre_algs_register(void)
+{
+	int ret = 0;
+
+	mutex_lock(&algs_lock);
+	if (++active_devs == 1) {
+		rsa.base.cra_flags = 0;
+		ret = crypto_register_akcipher(&rsa);
+		if (ret)
+			goto unlock;
+		ret = crypto_register_kpp(&dh);
+	}
+unlock:
+	mutex_unlock(&algs_lock);
+	return ret;
+}
+
+void hpre_algs_unregister(void)
+{
+	mutex_lock(&algs_lock);
+	if (--active_devs == 0) {
+		crypto_unregister_akcipher(&rsa);
+		crypto_unregister_kpp(&dh);
+	}
+	mutex_unlock(&algs_lock);
+}

drivers/crypto/hisilicon/hpre/hpre_main.c

+// SPDX-License-Identifier: GPL-2.0+
+#include <linux/io.h>
+#include <linux/bitops.h>
+#include <linux/init.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/vfio_spimdev.h>
+#include "hpre.h"
+
+#define HPRE_VF_NUM			63
+#define HPRE_QUEUE_NUM_V1		4096
+#define HPRE_QUEUE_NUM_V2		1024
+
+#define HPRE_COMM_CNT_CLR_CE		0x0
+#define HPRE_FSM_MAX_CNT		0x301008
+#define HPRE_VFG_AXQOS			0x30100c
+#define HPRE_VFG_AXCACHE		0x301010
+#define HPRE_RDCHN_INI_CFG		0x301014
+#define HPRE_BD_ENDIAN			0x301020
+#define HPRE_ECC_BYPASS			0x301024
+#define HPRE_POISON_BYPASS		0x30102c
+#define HPRE_ARUSR_CFG			0x301030
+#define HPRE_AWUSR_CFG			0x301034
+#define HPRE_INT_MASK			0x301400
+#define HPRE_RAS_ECC_1BIT_TH		0x30140c
+
+#define HPRE_TYPES_ENB			0x301038
+#define HPRE_PORT_ARCA_CHE_0		0x301040
+#define HPRE_PORT_ARCA_CHE_1		0x301044
+#define HPRE_PORT_AWCA_CHE_0		0x301060
+#define HPRE_PORT_AWCA_CHE_1		0x301064
+
+#define HPRE_BD_RUSER_32_63		0x301110
+#define HPRE_SGL_RUSER_32_63		0x30111c
+#define HPRE_DATA_RUSER_32_63		0x301128
+#define HPRE_DATA_WUSER_32_63		0x301134
+#define HPRE_BD_WUSER_32_63		0x301140
+#define HPRE_RDCHN_INI_ST		0x301a00
+#define HPRE_CORE_ENB			0x302004
+#define HPRE_CORE_INI_CFG		0x302020
+#define HPRE_CORE_INI_STATUS		0x302080
+
+
+LIST_HEAD(hisi_hpre_list);
+DEFINE_MUTEX(hisi_hpre_list_lock);
+
+static const struct pci_device_id hisi_hpre_dev_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa258) },
+	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa259) },
+	{ 0, }
+};
+
+
+static inline void hisi_hpre_add_to_list(struct hisi_hpre *hisi_hpre)
+{
+	mutex_lock(&hisi_hpre_list_lock);
+	list_add_tail(&hisi_hpre->list, &hisi_hpre_list);
+	mutex_unlock(&hisi_hpre_list_lock);
+}
+
+static inline void hisi_hpre_remove_from_list(struct hisi_hpre *hisi_hpre)
+{
+	mutex_lock(&hisi_hpre_list_lock);
+	list_del(&hisi_hpre->list);
+	mutex_unlock(&hisi_hpre_list_lock);
+}
+
+static int hisi_hpre_set_user_domain_and_cache(struct hisi_hpre *hisi_hpre)
+{
+	int ret;
+	u32 val;
+
+	writel(0x1, hisi_hpre->qm.io_base + HPRE_TYPES_ENB);
+	writel(0x0, hisi_hpre->qm.io_base + HPRE_VFG_AXQOS);
+	writel(0xff, hisi_hpre->qm.io_base + HPRE_VFG_AXCACHE);
+	writel(0x0, hisi_hpre->qm.io_base + HPRE_BD_ENDIAN);
+	writel(0x0, hisi_hpre->qm.io_base + HPRE_INT_MASK);
+	writel(0x0, hisi_hpre->qm.io_base + HPRE_RAS_ECC_1BIT_TH);
+	writel(0x0, hisi_hpre->qm.io_base + HPRE_POISON_BYPASS);
+	writel(0x0, hisi_hpre->qm.io_base + HPRE_COMM_CNT_CLR_CE);
+	writel(0x0, hisi_hpre->qm.io_base + HPRE_ECC_BYPASS);
+#ifndef CONFIG_ARM_SMMU_V3
+	writel(0x1, hisi_hpre->qm.io_base + HPRE_ARUSR_CFG);
+	writel(0x1, hisi_hpre->qm.io_base + HPRE_AWUSR_CFG);
+#else
+	writel(0x203, hisi_hpre->qm.io_base + HPRE_ARUSR_CFG);
+	writel(0x203, hisi_hpre->qm.io_base + HPRE_AWUSR_CFG);
+#endif
+	writel(0x1, hisi_hpre->qm.io_base + HPRE_RDCHN_INI_CFG);
+	ret = readl_relaxed_poll_timeout(hisi_hpre->qm.io_base +
+			HPRE_RDCHN_INI_ST, val, val & BIT(0), 10, 1000);
+	if (ret) {
+		pr_err("\nHPRE:INI ST TIMEOUT");
+		return -ETIMEDOUT;
+	}
+	/* First cluster initiating */
+	writel(0xf, hisi_hpre->qm.io_base + HPRE_CORE_ENB);
+	writel(0x1, hisi_hpre->qm.io_base + HPRE_CORE_INI_CFG);
+	ret = readl_relaxed_poll_timeout(hisi_hpre->qm.io_base +
+					 HPRE_CORE_INI_STATUS,
+					 val, ((val & 0xf) == 0xf), 10, 1000);
+	if (ret) {
+		pr_err("\nHPRE:CLUSTER 1 INI ST STATUS timeout");
+		return -ETIMEDOUT;
+	}
+	/* Second cluster initiating, reg's address is 0x1000 more*/
+	/* writel(0xf, hpre->io_base + 0x1000 + HPRE_CORE_ENB);*/
+	writel(0x0, hisi_hpre->qm.io_base + 0x1000 + HPRE_CORE_ENB);
+	writel(0x1, hisi_hpre->qm.io_base + 0x1000 + HPRE_CORE_INI_CFG);
+	ret = readl_relaxed_poll_timeout(hisi_hpre->qm.io_base + 0x1000 +
+				    HPRE_CORE_INI_STATUS,
+				    val, ((val & 0xf) == 0xf), 10, 1000);
+	if (ret) {
+		pr_err("\nHPRE:CLUSTER 2 INI ST STATUS timeout");
+		return -ETIMEDOUT;
+	}
+
+	return ret;
+}
+
+static int hisi_hpre_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct hisi_hpre *hisi_hpre;
+	struct qm_info *qm;
+	int ret;
+	u8 rev_id = 0;
+
+#ifdef CONFIG_ARM_SMMU_V3
+	u32 val;
+#endif
+
+	hisi_hpre = devm_kzalloc(&pdev->dev, sizeof(*hisi_hpre), GFP_KERNEL);
+	if (!hisi_hpre)
+		return -ENOMEM;
+	hisi_hpre_add_to_list(hisi_hpre);
+
+	qm = &hisi_hpre->qm;
+	qm->pdev = pdev;
+	pci_read_config_byte(pdev, PCI_REVISION_ID, &rev_id);
+	if (rev_id == 0x20)
+		qm->ver = QM_HW_V1;
+	else if (rev_id == 0x21)
+		qm->ver = QM_HW_V2;
+	qm->sqe_size = HPRE_SQE_SIZE;
+	ret = hisi_qm_init(qm, HPRE);
+	if (ret)
+		goto err_with_hisi_hpre;
+
+#define HPRE_ADDR(offset) QM_ADDR(qm, offset)
+
+	if (pdev->is_physfn) {
+		/* user domain */
+		writel(0x40000070, HPRE_ADDR(QM_ARUSER_M_CFG_1));
+		writel(0x007ffffc, HPRE_ADDR(QM_ARUSER_M_CFG_ENABLE));
+		writel(0x40000070, HPRE_ADDR(QM_AWUSER_M_CFG_1));
+		writel(0x007ffffc, HPRE_ADDR(QM_AWUSER_M_CFG_ENABLE));
+		writel(0x00000001, HPRE_ADDR(QM_WUSER_M_CFG_ENABLE));
+		writel(0x1833, HPRE_ADDR(QM_CACHE_CTL));
+		writel(0x00400001, HPRE_ADDR(QM_PEH_AXUSER_CFG));
+#ifdef CONFIG_ARM_SMMU_V3
+		writel(0x40000070, HPRE_ADDR(QM_ARUSER_M_CFG_1));
+		writel(0xfffffffe, HPRE_ADDR(QM_ARUSER_M_CFG_ENABLE));
+		writel(0x40000070, HPRE_ADDR(QM_AWUSER_M_CFG_1));
+		writel(0xfffffffe, HPRE_ADDR(QM_AWUSER_M_CFG_ENABLE));
+
+		val = readl_relaxed(HPRE_ADDR(QM_ARUSER_M_CFG_1));
+		val &= ~GENMASK(14, 12);
+		val |= (1 << 12);
+		writel(val, HPRE_ADDR(QM_ARUSER_M_CFG_1));
+
+		val = readl_relaxed(HPRE_ADDR(QM_AWUSER_M_CFG_1));
+		val &= ~GENMASK(14, 12);
+		val |= (1 << 12);
+		writel(val, HPRE_ADDR(QM_AWUSER_M_CFG_1));
+
+		val = readl_relaxed(HPRE_ADDR(QM_ARUSER_M_CFG_ENABLE));
+		val &= ~0x1;
+		writel(val, HPRE_ADDR(QM_ARUSER_M_CFG_ENABLE));
+
+		val = readl_relaxed(HPRE_ADDR(QM_AWUSER_M_CFG_ENABLE));
+		val &= ~0x1;
+		writel(val, HPRE_ADDR(QM_AWUSER_M_CFG_ENABLE));
+#endif
+		/* cache */
+		writel_relaxed(0x0303, /* 0xffff IT */
+			HPRE_ADDR(QM_AXI_M_CFG));
+		writel_relaxed(0xf,
+			HPRE_ADDR(QM_AXI_M_CFG_ENABLE));
+		writel_relaxed(0x7f,
+			HPRE_ADDR(QM_PEH_AXUSER_CFG_ENABLE));
+#ifdef CONFIG_ARM_SMMU_V3
+		writel_relaxed(0xffff,
+			HPRE_ADDR(QM_AXI_M_CFG));
+		writel_relaxed(0xffffffff,
+			HPRE_ADDR(QM_AXI_M_CFG_ENABLE));
+		writel_relaxed(0xffffffff,
+			HPRE_ADDR(QM_PEH_AXUSER_CFG_ENABLE));
+#endif
+		ret = hisi_qm_mem_start(qm);
+		if (ret)
+			goto err_with_qm_init;
+		ret = hisi_hpre_set_user_domain_and_cache(hisi_hpre);
+		if (ret)
+			return ret;
+		qm->qp_base = HPRE_PF_DEF_Q_BASE;
+		qm->qp_num = HPRE_PF_DEF_Q_NUM;
+		qm->free_qp = qm->qp_num;
+	}
+
+	ret = hisi_qm_start(qm);
+	if (ret)
+		goto err_with_qm_init;
+
+	/* todo: exception irq handler register, ES did not support */
+
+	return 0;
+
+err_with_qm_init:
+	hisi_qm_uninit(qm);
+
+err_with_hisi_hpre:
+	hisi_hpre_remove_from_list(hisi_hpre);
+	kfree(hisi_hpre);
+
+	return ret;
+}
+
+static void hisi_hpre_remove(struct pci_dev *pdev)
+{
+	struct hisi_hpre *hisi_hpre = pci_get_drvdata(pdev);
+	struct qm_info *qm = &hisi_hpre->qm;
+
+	hisi_qm_stop(qm);
+	hisi_qm_uninit(qm);
+	hisi_hpre_remove_from_list(hisi_hpre);
+	kfree(hisi_hpre);
+}
+
+static int hisi_hpre_pci_sriov_configure(struct pci_dev *pdev, int num_vfs)
+{
+	/* todo: set queue number for VFs */
+
+	return 0;
+}
+
+static struct pci_driver hisi_hpre_pci_driver = {
+	.name			= "hisi_hpre",
+	.id_table		= hisi_hpre_dev_ids,
+	.probe			= hisi_hpre_probe,
+	.remove			= hisi_hpre_remove,
+	.sriov_configure	= hisi_hpre_pci_sriov_configure
+};
+
+static int __init hisi_hpre_init(void)
+{
+	int ret;
+
+	ret = pci_register_driver(&hisi_hpre_pci_driver);
+	if (ret < 0) {
+		pr_err("hpre: can't register hisi hpre driver.\n");
+		return ret;
+	}
+
+	ret = hpre_algs_register();
+	if (ret < 0) {
+		pr_err("hpre: can't register hisi hpre to crypto.\n");
+		pci_unregister_driver(&hisi_hpre_pci_driver);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void __exit hisi_hpre_exit(void)
+{
+	hpre_algs_unregister();
+	pci_unregister_driver(&hisi_hpre_pci_driver);
+}
+
+module_init(hisi_hpre_init);
+module_exit(hisi_hpre_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Zaibo Xu <xuzaibo@huawei.com>");
+MODULE_DESCRIPTION("Driver for HiSilicon HPRE accelerator");
+MODULE_DEVICE_TABLE(pci, hisi_hpre_dev_ids);

drivers/crypto/hisilicon/qm.c

+// SPDX-License-Identifier: GPL-2.0+
+#include <asm/page.h>
+#include <linux/bitmap.h>
+#include <linux/dma-mapping.h>
+#include <linux/io.h>
+#include <linux/irqreturn.h>
+#include <linux/log2.h>
+#include "qm.h"
+
+#define QM_DEF_Q_NUM			128
+
+/* eq/aeq irq enable */
+#define QM_VF_AEQ_INT_SOURCE		0x0
+#define QM_VF_AEQ_INT_MASK		0x4
+#define QM_VF_EQ_INT_SOURCE		0x8
+#define QM_VF_EQ_INT_MASK		0xc
+
+/* mailbox */
+#define MAILBOX_CMD_SQC			0x0
+#define MAILBOX_CMD_CQC			0x1
+#define MAILBOX_CMD_EQC			0x2
+#define MAILBOX_CMD_SQC_BT		0x4
+#define MAILBOX_CMD_CQC_BT		0x5
+
+#define MAILBOX_CMD_SEND_BASE		0x300
+#define MAILBOX_EVENT_SHIFT		8
+#define MAILBOX_STATUS_SHIFT		9
+#define MAILBOX_BUSY_SHIFT		13
+#define MAILBOX_OP_SHIFT		14
+#define MAILBOX_QUEUE_SHIFT		16
+
+/* sqc shift */
+#define SQ_HEAD_SHIFT			0
+#define SQ_TAIL_SHIFI			16
+#define SQ_HOP_NUM_SHIFT		0
+#define SQ_PAGE_SIZE_SHIFT		4
+#define SQ_BUF_SIZE_SHIFT		8
+#define SQ_SQE_SIZE_SHIFT		12
+#define SQ_HEAD_IDX_SIG_SHIFT		0
+#define SQ_TAIL_IDX_SIG_SHIFT		0
+#define SQ_CQN_SHIFT			0
+#define SQ_PRIORITY_SHIFT		0
+#define SQ_ORDERS_SHIFT			4
+#define SQ_TYPE_SHIFT			8
+
+#define SQ_TYPE_MASK			0xf
+
+/* cqc shift */
+#define CQ_HEAD_SHIFT			0
+#define CQ_TAIL_SHIFI			16
+#define CQ_HOP_NUM_SHIFT		0
+#define CQ_PAGE_SIZE_SHIFT		4
+#define CQ_BUF_SIZE_SHIFT		8
+#define CQ_SQE_SIZE_SHIFT		12
+#define CQ_PASID			0
+#define CQ_HEAD_IDX_SIG_SHIFT		0
+#define CQ_TAIL_IDX_SIG_SHIFT		0
+#define CQ_CQN_SHIFT			0
+#define CQ_PRIORITY_SHIFT		16
+#define CQ_ORDERS_SHIFT			0
+#define CQ_TYPE_SHIFT			0
+#define CQ_PHASE_SHIFT			0
+#define CQ_FLAG_SHIFT			1
+
+#define CQC_HEAD_INDEX(cqc)		((cqc)->cq_head)
+#define CQC_PHASE(cqc)			(((cqc)->dw6) & 0x1)
+#define CQC_CQ_ADDRESS(cqc)		(((u64)((cqc)->cq_base_h) << 32) | \
+					 ((cqc)->cq_base_l))
+#define CQC_PHASE_BIT			0x1
+
+/* eqc shift */
+#define MB_EQC_EQE_SHIFT		12
+#define MB_EQC_PHASE_SHIFT		16
+
+#define EQC_HEAD_INDEX(eqc)		((eqc)->eq_head)
+#define EQC_TAIL_INDEX(eqc)		((eqc)->eq_tail)
+#define EQC_PHASE(eqc)			((((eqc)->dw6) >> 16) & 0x1)
+
+#define EQC_PHASE_BIT			0x00010000
+
+/* cqe shift */
+#define CQE_PHASE(cqe)			((cqe)->w7 & 0x1)
+#define CQE_SQ_NUM(cqe)			((cqe)->sq_num)
+#define CQE_SQ_HEAD_INDEX(cqe)		((cqe)->sq_head)
+
+/* eqe shift */
+#define EQE_PHASE(eqe)			(((eqe)->dw0 >> 16) & 0x1)
+#define EQE_CQN(eqe)			(((eqe)->dw0) & 0xffff)
+
+#define QM_EQE_CQN_MASK			0xffff
+
+/* doorbell */
+#define DOORBELL_CMD_SQ			0
+#define DOORBELL_CMD_CQ			1
+#define DOORBELL_CMD_EQ			2
+#define DOORBELL_CMD_AEQ		3
+
+#define DOORBELL_CMD_SEND_BASE_V1	0x340
+#define DOORBELL_CMD_SEND_BASE_V2	0x1000
+#define QM_MEM_START_INIT		0x100040
+#define QM_MEM_INIT_DONE		0x100044
+#define QM_VFT_CFG_RDY			0x10006c
+#define QM_VFT_CFG_OP_WR		0x100058
+#define QM_VFT_CFG_TYPE			0x10005c
+#define QM_SQC_VFT			0x0
+#define QM_CQC_VFT			0x1
+#define QM_VFT_CFG_ADDRESS		0x100060
+#define QM_VFT_CFG_OP_ENABLE		0x100054
+
+#define QM_VFT_CFG_DATA_L		0x100064
+#define QM_VFT_CFG_DATA_H		0x100068
+#define QM_SQC_VFT_BUF_SIZE		(7ULL << 8)
+#define QM_SQC_VFT_SQC_SIZE		(5ULL << 12)
+#define QM_SQC_VFT_INDEX_NUMBER		(1ULL << 16)
+#define QM_SQC_VFT_BT_INDEX_SHIFT	22
+#define QM_SQC_VFT_START_SQN_SHIFT	28
+#define QM_SQC_VFT_VALID		(1ULL << 44)
+#define QM_SQC_VFT_SQN_SHIFT		45
+#define QM_CQC_VFT_BUF_SIZE		(7ULL << 8)
+#define QM_CQC_VFT_SQC_SIZE		(5ULL << 12)
+#define QM_CQC_VFT_INDEX_NUMBER		(1ULL << 16)
+#define QM_CQC_VFT_BT_INDEX_SHIFT	22
+#define QM_CQC_VFT_VALID		(1ULL << 28)
+
+struct cqe {
+	__le32 rsvd0;
+	__le16 cmd_id;
+	__le16 rsvd1;
+	__le16 sq_head;
+	__le16 sq_num;
+	__le16 rsvd2;
+	__le16 w7;
+};
+
+struct eqe {
+	__le32 dw0;
+};
+
+struct sqc {
+	__le16 head;
+	__le16 tail;
+	__le32 base_l;
+	__le32 base_h;
+	__le32 dw3;
+	__le16 w8;
+	__le16 rsvd0;
+	__le16 pasid;
+	__le16 w11;
+	__le16 cq_num;
+	__le16 w13;
+	__le32 rsvd1;
+};
+
+struct cqc {
+	__le16 head;
+	__le16 tail;
+	__le32 base_l;
+	__le32 base_h;
+	__le32 dw3;
+	__le16 w8;
+	__le16 rsvd0;
+	__le16 pasid;
+	__le16 w11;
+	__le32 dw6;
+	__le32 rsvd1;
+};
+
+#define INIT_QC(qc, base) do { \
+	(qc)->head = 0; \
+	(qc)->tail = 0; \
+	(qc)->base_l = lower_32_bits(base); \
+	(qc)->base_h = upper_32_bits(base); \
+	(qc)->pasid = 0; \
+	(qc)->w11 = 0; \
+	(qc)->rsvd1 = 0; \
+} while (0)
+
+struct eqc {
+	__le16 head;
+	__le16 tail;
+	__le32 base_l;
+	__le32 base_h;
+	__le32 dw3;
+	__le32 rsvd[2];
+	__le32 dw6;
+};
+
+struct mailbox {
+	__le16 w0;
+	__le16 queue_num;
+	__le32 base_l;
+	__le32 base_h;
+	__le32 rsvd;
+};
+
+struct doorbell {
+	__le16 queue_num;
+	__le16 cmd;
+	__le16 index;
+	__le16 priority;
+};
+
+#define QM_DMA_BUF(p, buf) ((struct buf *)(p)->buf.addr)
+#define QM_SQC(p) QM_DMA_BUF(p, sqc)
+#define QM_CQC(p) QM_DMA_BUF(p, cqc)
+#define QM_EQC(p) QM_DMA_BUF(p, eqc)
+#define QM_EQE(p) QM_DMA_BUF(p, eqe)
+
+#define QP_SQE_DMA(qp) ((qp)->scqe.dma)
+#define QP_CQE(qp) ((struct cqe *)((qp)->scqe.addr + \
+				   qp->qm->sqe_size * QM_Q_DEPTH))
+#define QP_CQE_DMA(qp) ((qp)->scqe.dma + qp->qm->sqe_size * QM_Q_DEPTH)
+
+static inline void qm_writel(struct qm_info *qm, u32 val, u32 offset)
+{
+	writel(val, qm->io_base + offset);
+}
+
+struct qm_info;
+
+struct hisi_acc_qm_hw_ops {
+	int (*vft_config)(struct qm_info *qm, u16 base, u32 number);
+	int (*qm_start_qp)(struct hisi_qp *qp, unsigned long arg);
+	void (*qm_db)(struct qm_info *qm, u16 qn,
+			u8 cmd, u16 index, u8 priority);
+};
+
+static inline int hacc_qm_mb_is_busy(struct qm_info *qm)
+{
+	u32 val;
+
+	return readl_relaxed_poll_timeout(QM_ADDR(qm, MAILBOX_CMD_SEND_BASE),
+		val, !((val >> MAILBOX_BUSY_SHIFT) & 0x1), 10, 1000);
+}
+
+static inline void qm_mb_write(struct qm_info *qm, void *src)
+{
+	void __iomem *fun_base = QM_ADDR(qm, MAILBOX_CMD_SEND_BASE);
+	unsigned long tmp0 = 0, tmp1 = 0;
+
+	asm volatile("ldp %0, %1, %3\n"
+		     "stp %0, %1, %2\n"
+		     "dsb sy\n"
+		     : "=&r" (tmp0),
+		       "=&r" (tmp1),
+		       "+Q" (*((char *)fun_base))
+		     : "Q" (*((char *)src))
+		     : "memory");
+}
+
+static int qm_mb(struct qm_info *qm, u8 cmd, dma_addr_t dma_addr, u16 queue,
+		 bool op, bool event)
+{
+	struct mailbox mailbox;
+	int i = 0;
+	int ret = 0;
+
+	memset(&mailbox, 0, sizeof(struct mailbox));
+
+	mailbox.w0 = cmd |
+		     (event ? 0x1 << MAILBOX_EVENT_SHIFT : 0) |
+		     (op ? 0x1 << MAILBOX_OP_SHIFT : 0) |
+		     (0x1 << MAILBOX_BUSY_SHIFT);
+	mailbox.queue_num = queue;
+	mailbox.base_l = lower_32_bits(dma_addr);
+	mailbox.base_h = upper_32_bits(dma_addr);
+	mailbox.rsvd = 0;
+
+	mutex_lock(&qm->mailbox_lock);
+
+	while (hacc_qm_mb_is_busy(qm) && i < 10)
+		i++;
+	if (i >= 10) {
+		ret = -EBUSY;
+		dev_err(&qm->pdev->dev, "QM mail box is busy!");
+		goto busy_unlock;
+	}
+	qm_mb_write(qm, &mailbox);
+	i = 0;
+	while (hacc_qm_mb_is_busy(qm) && i < 10)
+		i++;
+	if (i >= 10) {
+		ret = -EBUSY;
+		dev_err(&qm->pdev->dev, "QM mail box is still busy!");
+		goto busy_unlock;
+	}
+
+busy_unlock:
+	mutex_unlock(&qm->mailbox_lock);
+
+	return ret;
+}
+
+static void qm_db_v1(struct qm_info *qm, u16 qn, u8 cmd, u16 index, u8 priority)
+{
+	u64 doorbell = 0;
+
+	doorbell = (u64)qn | ((u64)cmd << 16);
+	doorbell |= ((u64)index | ((u64)priority << 16)) << 32;
+
+	writeq(doorbell, QM_ADDR(qm, DOORBELL_CMD_SEND_BASE_V1));
+}
+
+static void qm_db_v2(struct qm_info *qm, u16 qn, u8 cmd, u16 index, u8 priority)
+{
+	u64 doorbell = 0;
+	u16 randate = 0;
+
+	doorbell = (u64)qn | ((u64)cmd << 12) | ((u64)randate << 16);
+	doorbell |= ((u64)index | ((u64)priority << 16)) << 32;
+
+	writeq(doorbell, QM_ADDR(qm, DOORBELL_CMD_SEND_BASE_V2));
+}
+
+static void qm_db(struct qm_info *qm, u16 qn, u8 cmd, u16 index, u8 priority)
+{
+	qm->ops->qm_db(qm, qn, cmd, index, priority);
+}
+
+/* @return 0 - cq/eq event, 1 - async event, 2 - abnormal error */
+static u32 qm_get_irq_source(struct qm_info *qm)
+{
+	return readl(QM_ADDR(qm, QM_VF_EQ_INT_SOURCE));
+}
+
+static inline struct hisi_qp *to_hisi_qp(struct qm_info *qm, struct eqe *eqe)
+{
+	u16 cqn = eqe->dw0 & QM_EQE_CQN_MASK;
+	struct hisi_qp *qp;
+
+	read_lock(&qm->qps_lock);
+	qp = qm->qp_array[cqn];
+	read_unlock(&qm->qps_lock);
+
+	return qp;
+}
+
+static inline void qm_cq_head_update(struct hisi_qp *qp)
+{
+	if (qp->qp_status.cq_head == QM_Q_DEPTH - 1) {
+		QM_CQC(qp)->dw6 = QM_CQC(qp)->dw6 ^ CQC_PHASE_BIT;
+		qp->qp_status.cq_head = 0;
+	} else {
+		qp->qp_status.cq_head++;
+	}
+}
+
+static inline void qm_poll_qp(struct hisi_qp *qp, struct qm_info *qm)
+{
+	struct cqe *cqe;
+
+	cqe = QP_CQE(qp) + qp->qp_status.cq_head;
+
+	if (qp->req_cb) {
+		while (CQE_PHASE(cqe) == CQC_PHASE(QM_CQC(qp))) {
+			dma_rmb();
+			qp->req_cb(qp, QP_SQE_ADDR(qp) +
+				   qm->sqe_size *
+				   CQE_SQ_HEAD_INDEX(cqe));
+			qm_cq_head_update(qp);
+			cqe = QP_CQE(qp) + qp->qp_status.cq_head;
+		}
+	} else if (qp->event_cb) {
+		dma_rmb();
+		qp->event_cb(qp);
+		qm_cq_head_update(qp);
+		cqe = QP_CQE(qp) + qp->qp_status.cq_head;
+	}
+
+	qm_db(qm, qp->queue_id, DOORBELL_CMD_CQ, qp->qp_status.cq_head, 0);
+
+	/* set c_flag */
+	qm_db(qm, qp->queue_id, DOORBELL_CMD_CQ, qp->qp_status.cq_head, 1);
+}
+
+static irqreturn_t qm_irq_thread(int irq, void *data)
+{
+	struct qm_info *qm = data;
+	struct eqe *eqe = QM_EQE(qm) + qm->eq_head;
+	struct eqc *eqc = QM_EQC(qm);
+	struct hisi_qp *qp;
+
+	while (EQE_PHASE(eqe) == EQC_PHASE(eqc)) {
+		qp = to_hisi_qp(qm, eqe);
+		if (qp)
+			qm_poll_qp(qp, qm);
+
+		if (qm->eq_head == QM_Q_DEPTH - 1) {
+			eqc->dw6 = eqc->dw6 ^ EQC_PHASE_BIT;
+			eqe = QM_EQE(qm);
+			qm->eq_head = 0;
+		} else {
+			eqe++;
+			qm->eq_head++;
+		}
+
+		qm_db(qm, 0, DOORBELL_CMD_EQ, qm->eq_head, 0);
+	}
+
+	return IRQ_HANDLED;
+}
+
+static void qm_init_qp_status(struct hisi_qp *qp)
+{
+	struct hisi_acc_qp_status *qp_status = &qp->qp_status;
+
+	qp_status->sq_tail = 0;
+	qp_status->sq_head = 0;
+	qp_status->cq_head = 0;
+	qp_status->cqc_phase = 1;
+	qp_status->is_sq_full = 0;
+}
+
+/* check if bit in regs is 1 */
+static inline int qm_acc_check(struct qm_info *qm, u32 offset, u32 bit)
+{
+	int val;
+
+	return readl_relaxed_poll_timeout(QM_ADDR(qm, offset), val,
+					  val & BIT(bit), 10, 1000);
+}
+
+static inline int qm_init_q_buffer(struct device *dev, size_t size,
+				   struct qm_dma_buffer *db)
+{
+	if (db->addr && db->size) {
+		memset(db->addr, 0, db->size);
+		return 0;
+	}
+	db->size = size;
+	db->addr = dma_zalloc_coherent(dev, size, &db->dma, GFP_KERNEL);
+	if (!db->addr)
+		return -ENOMEM;
+
+	return 0;
+}
+
+static inline void qm_uninit_q_buffer(struct device *dev,
+				      struct qm_dma_buffer *db)
+{
+	dma_free_coherent(dev, db->size, db->addr, db->dma);
+}
+
+static inline int qm_init_bt(struct qm_info *qm, struct device *dev,
+			     size_t size, struct qm_dma_buffer *db, int mb_cmd)
+{
+	int ret;
+
+	ret = qm_init_q_buffer(dev, size, db);
+	if (ret)
+		return -ENOMEM;
+
+	ret = qm_mb(qm, mb_cmd, db->dma, 0, 0, 0);
+	if (ret) {
+		qm_uninit_q_buffer(dev, db);
+		return ret;
+	}
+
+	return 0;
+}
+
+/* the config should be conducted after hisi_acc_init_qm_mem() */
+static int qm_vft_common_config_v1(struct qm_info *qm, u16 base, u32 number)
+{
+	u64 tmp;
+	int ret;
+
+	ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0);
+	if (ret)
+		return ret;
+	qm_writel(qm, 0x0, QM_VFT_CFG_OP_WR);
+	qm_writel(qm, QM_SQC_VFT, QM_VFT_CFG_TYPE);
+	qm_writel(qm, qm->pdev->devfn, QM_VFT_CFG_ADDRESS);
+
+	tmp = QM_SQC_VFT_BUF_SIZE			|
+	      QM_SQC_VFT_SQC_SIZE			|
+	      QM_SQC_VFT_INDEX_NUMBER			|
+	      QM_SQC_VFT_VALID				|
+	      (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
+
+	qm_writel(qm, tmp & 0xffffffff, QM_VFT_CFG_DATA_L);
+	qm_writel(qm, tmp >> 32, QM_VFT_CFG_DATA_H);
+
+	qm_writel(qm, 0x0, QM_VFT_CFG_RDY);
+	qm_writel(qm, 0x1, QM_VFT_CFG_OP_ENABLE);
+	ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0);
+	if (ret)
+		return ret;
+	tmp = 0;
+
+	qm_writel(qm, 0x0, QM_VFT_CFG_OP_WR);
+	qm_writel(qm, QM_CQC_VFT, QM_VFT_CFG_TYPE);
+	qm_writel(qm, qm->pdev->devfn, QM_VFT_CFG_ADDRESS);
+
+	tmp = QM_CQC_VFT_BUF_SIZE			|
+	      QM_CQC_VFT_SQC_SIZE			|
+	      QM_CQC_VFT_INDEX_NUMBER			|
+	      QM_CQC_VFT_VALID;
+
+	qm_writel(qm, tmp & 0xffffffff, QM_VFT_CFG_DATA_L);
+	qm_writel(qm, tmp >> 32, QM_VFT_CFG_DATA_H);
+
+	qm_writel(qm, 0x0, QM_VFT_CFG_RDY);
+	qm_writel(qm, 0x1, QM_VFT_CFG_OP_ENABLE);
+	ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+static int qm_vft_common_config_v2(struct qm_info *qm, u16 base, u32 number)
+{
+	u64 tmp;
+	int ret;
+
+	ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0);
+	if (ret)
+		return ret;
+	qm_writel(qm, 0x0, QM_VFT_CFG_OP_WR);
+	qm_writel(qm, QM_SQC_VFT, QM_VFT_CFG_TYPE);
+	qm_writel(qm, qm->pdev->devfn, QM_VFT_CFG_ADDRESS);
+
+	tmp = (u64)number << QM_SQC_VFT_SQN_SHIFT	|
+	      QM_SQC_VFT_VALID			|
+	      (u64)base << QM_SQC_VFT_START_SQN_SHIFT;
+
+	qm_writel(qm, tmp & 0xffffffff, QM_VFT_CFG_DATA_L);
+	qm_writel(qm, tmp >> 32, QM_VFT_CFG_DATA_H);
+
+	qm_writel(qm, 0x0, QM_VFT_CFG_RDY);
+	qm_writel(qm, 0x1, QM_VFT_CFG_OP_ENABLE);
+	ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0);
+	if (ret)
+		return ret;
+	tmp = 0;
+
+	qm_writel(qm, 0x0, QM_VFT_CFG_OP_WR);
+	qm_writel(qm, QM_CQC_VFT, QM_VFT_CFG_TYPE);
+	qm_writel(qm, qm->pdev->devfn, QM_VFT_CFG_ADDRESS);
+
+	tmp = QM_CQC_VFT_VALID;
+
+	qm_writel(qm, tmp & 0xffffffff, QM_VFT_CFG_DATA_L);
+	qm_writel(qm, tmp >> 32, QM_VFT_CFG_DATA_H);
+
+	qm_writel(qm, 0x0, QM_VFT_CFG_RDY);
+	qm_writel(qm, 0x1, QM_VFT_CFG_OP_ENABLE);
+	ret = qm_acc_check(qm, QM_VFT_CFG_RDY, 0);
+	if (ret)
+		return ret;
+	return 0;
+}
+
+struct hisi_qp *hisi_qm_create_qp(struct qm_info *qm, u8 alg_type)
+{
+	struct hisi_qp *qp;
+	int qp_index;
+	int ret;
+
+	write_lock(&qm->qps_lock);
+	qp_index = find_first_zero_bit(qm->qp_bitmap, qm->qp_num);
+	if (qp_index >= qm->qp_num) {
+		write_unlock(&qm->qps_lock);
+		return ERR_PTR(-EBUSY);
+	}
+	set_bit(qp_index, qm->qp_bitmap);
+
+	qp = kzalloc(sizeof(*qp), GFP_KERNEL);
+	if (!qp) {
+		ret = -ENOMEM;
+		write_unlock(&qm->qps_lock);
+		goto err_with_bitset;
+	}
+
+	qp->queue_id = qp_index;
+	qp->qm = qm;
+	qp->alg_type = alg_type;
+	qm_init_qp_status(qp);
+	write_unlock(&qm->qps_lock);
+	qm->free_qp--;
+	return qp;
+
+err_with_bitset:
+	clear_bit(qp_index, qm->qp_bitmap);
+
+	return ERR_PTR(ret);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_create_qp);
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+static void hisi_qm_reg_user_info(struct hisi_qp *qp, u32 version)
+{
+	/* Fill qp_index and qm version information here, for user space */
+	*(u32 *)qp->scqe.addr = qp->queue_id;
+	*((u32 *)qp->scqe.addr + 1) = version;
+
+	/* To be fixed */
+
+}
+#endif
+
+int hisi_qm_start_qp_v1(struct hisi_qp *qp, unsigned long arg)
+{
+	struct qm_info *qm = qp->qm;
+	struct device *dev = &qm->pdev->dev;
+	struct sqc *sqc;
+	struct cqc *cqc;
+	int qp_index = qp->queue_id;
+	int pasid = arg;
+	int ret;
+
+	/* set sq and cq context */
+	if (!qp->sqc.addr) {
+		qp->sqc.addr = QM_SQC(qm) + qp_index;
+		qp->sqc.dma = qm->sqc.dma + qp_index * sizeof(struct sqc);
+	}
+	sqc = QM_SQC(qp);
+
+	if (!qp->cqc.addr) {
+		qp->cqc.addr = QM_CQC(qm) + qp_index;
+		qp->cqc.dma = qm->cqc.dma + qp_index * sizeof(struct cqc);
+	}
+	cqc = QM_CQC(qp);
+
+	/* allocate sq and cq */
+	ret = qm_init_q_buffer(dev,
+		qm->sqe_size * QM_Q_DEPTH + sizeof(struct cqe) * QM_Q_DEPTH,
+		&qp->scqe);
+	if (ret)
+		return ret;
+
+	INIT_QC(sqc, qp->scqe.dma);
+	sqc->pasid = pasid;
+	sqc->dw3 = (0 << SQ_HOP_NUM_SHIFT)      |
+		   (0 << SQ_PAGE_SIZE_SHIFT)    |
+		   (0 << SQ_BUF_SIZE_SHIFT)     |
+		   (ilog2(qm->sqe_size) << SQ_SQE_SIZE_SHIFT);
+	sqc->w8 = QM_Q_DEPTH - 1;
+	sqc->cq_num = qp_index;
+	sqc->w13 = 0 << SQ_PRIORITY_SHIFT	|
+		   1 << SQ_ORDERS_SHIFT		|
+		   (qp->alg_type & SQ_TYPE_MASK) << SQ_TYPE_SHIFT;
+
+	ret = qm_mb(qm, MAILBOX_CMD_SQC, qp->sqc.dma, qp_index, 0, 0);
+	if (ret)
+		return ret;
+
+	INIT_QC(cqc, qp->scqe.dma + qm->sqe_size * QM_Q_DEPTH);
+	cqc->dw3 = (0 << CQ_HOP_NUM_SHIFT)	|
+		   (0 << CQ_PAGE_SIZE_SHIFT)	|
+		   (0 << CQ_BUF_SIZE_SHIFT)	|
+		   (4 << CQ_SQE_SIZE_SHIFT);
+	cqc->dw6 = 1 << CQ_PHASE_SHIFT | 1 << CQ_FLAG_SHIFT;
+	cqc->w8 = QM_Q_DEPTH - 1;
+	ret = qm_mb(qm, MAILBOX_CMD_CQC, qp->cqc.dma, qp_index, 0, 0);
+	if (ret)
+		return ret;
+
+	write_lock(&qm->qps_lock);
+	qm->qp_array[qp_index] = qp;
+	init_completion(&qp->completion);
+	write_unlock(&qm->qps_lock);
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+	hisi_qm_reg_user_info(qp, qm->ver);
+#endif
+
+	return qp_index;
+}
+
+
+int hisi_qm_start_qp_v2(struct hisi_qp *qp, unsigned long arg)
+{
+	struct qm_info *qm = qp->qm;
+	struct device *dev = &qm->pdev->dev;
+	int ret;
+	struct sqc *sqc;
+	struct cqc *cqc;
+	int qp_index = qp->queue_id;
+	int pasid = arg;
+
+	/* set sq and cq context */
+	if (!qp->sqc.addr) {
+		qp->sqc.addr = QM_SQC(qm) + qp_index;
+		qp->sqc.dma = qm->sqc.dma + qp_index * sizeof(struct sqc);
+	}
+	sqc = QM_SQC(qp);
+
+	if (!qp->cqc.addr) {
+		qp->cqc.addr = QM_CQC(qm) + qp_index;
+		qp->cqc.dma = qm->cqc.dma + qp_index * sizeof(struct cqc);
+	}
+	cqc = QM_CQC(qp);
+
+	/* allocate sq and cq */
+	ret = qm_init_q_buffer(dev,
+		qm->sqe_size * QM_Q_DEPTH + sizeof(struct cqe) * QM_Q_DEPTH,
+		&qp->scqe);
+	if (ret)
+		return ret;
+
+	INIT_QC(sqc, qp->scqe.dma);
+	sqc->pasid = pasid;
+	sqc->dw3 = (QM_Q_DEPTH - 1) |
+		   (ilog2(qm->sqe_size) << SQ_SQE_SIZE_SHIFT);
+	sqc->w8 = 0;/*rand_qc*/
+	sqc->cq_num = qp_index;
+	sqc->w13 = 0 << SQ_PRIORITY_SHIFT	|
+		   1 << SQ_ORDERS_SHIFT		|
+		   (qp->alg_type & SQ_TYPE_MASK) << SQ_TYPE_SHIFT;
+
+	ret = qm_mb(qm, MAILBOX_CMD_SQC, qp->sqc.dma, qp_index, 0, 0);
+	if (ret)
+		return ret;
+
+	INIT_QC(cqc, qp->scqe.dma + qm->sqe_size * QM_Q_DEPTH);
+	cqc->pasid = pasid;
+	cqc->dw3 = (QM_Q_DEPTH - 1) |
+		   (4 << CQ_SQE_SIZE_SHIFT);
+	cqc->w8 = 0;/*rand_qc*/
+	cqc->dw6 = 1 << CQ_PHASE_SHIFT | 1 << CQ_FLAG_SHIFT;
+
+	ret = qm_mb(qm, MAILBOX_CMD_CQC, qp->cqc.dma, qp_index, 0, 0);
+	if (ret)
+		return ret;
+
+	write_lock(&qm->qps_lock);
+	qm->qp_array[qp_index] = qp;
+	init_completion(&qp->completion);
+	write_unlock(&qm->qps_lock);
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+	hisi_qm_reg_user_info(qp, qm->ver);
+#endif
+	return qp_index;
+}
+
+
+int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg)
+{
+	struct qm_info *qm = qp->qm;
+
+	return qm->ops->qm_start_qp(qp, arg);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_start_qp);
+
+static struct hisi_acc_qm_hw_ops qm_hw_ops_v1 = {
+	.vft_config = qm_vft_common_config_v1,
+	.qm_start_qp = hisi_qm_start_qp_v1,
+	.qm_db = qm_db_v1,
+};
+
+static struct hisi_acc_qm_hw_ops qm_hw_ops_v2 = {
+	.vft_config = qm_vft_common_config_v2,
+	.qm_start_qp = hisi_qm_start_qp_v2,
+	.qm_db = qm_db_v2,
+};
+
+void hisi_qm_release_qp(struct hisi_qp *qp)
+{
+	struct qm_info *qm = qp->qm;
+	struct device *dev = &qm->pdev->dev;
+	u64 phy = qp->udma_buf.phy_addr;
+	int order = qp->udma_buf.order;
+
+	if (order > MAX_ORDER || phy & 0xfff)
+		return;
+	if (phy)
+		__free_pages(phys_to_page(phy), order);
+
+	write_lock(&qm->qps_lock);
+	qm->free_qp++;
+	qm->qp_array[qp->queue_id] = NULL;
+	bitmap_clear(qm->qp_bitmap, qp->queue_id, 1);
+	write_unlock(&qm->qps_lock);
+
+	qm_uninit_q_buffer(dev, &qp->scqe);
+	kfree(qp);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_release_qp);
+
+static void *qm_get_avail_sqe(struct hisi_qp *qp)
+{
+	struct hisi_acc_qp_status *qp_status = &qp->qp_status;
+	void *sq_base = QP_SQE_ADDR(qp);
+	u16 sq_tail = qp_status->sq_tail;
+
+	if (qp_status->is_sq_full == 1)
+		return NULL;
+
+	return sq_base + sq_tail * qp->qm->sqe_size;
+}
+
+int hisi_qp_send(struct hisi_qp *qp, void *msg)
+{
+	struct hisi_acc_qp_status *qp_status = &qp->qp_status;
+	u16 sq_tail = qp_status->sq_tail;
+	u16 sq_tail_next = (sq_tail + 1) % QM_Q_DEPTH;
+	unsigned long timeout = 100;
+	void *sqe = qm_get_avail_sqe(qp);
+
+	if (!sqe)
+		return -ENOSPC;
+
+	memcpy(sqe, msg, qp->qm->sqe_size);
+
+	qm_db(qp->qm, qp->queue_id, DOORBELL_CMD_SQ, sq_tail_next, 0);
+
+	qp_status->sq_tail = sq_tail_next;
+
+	if (qp_status->sq_tail == qp_status->sq_head)
+		qp_status->is_sq_full = 1;
+
+	/* wait until job finished */
+	wait_for_completion_timeout(&qp->completion, timeout);
+
+	return 0;
+}
+EXPORT_SYMBOL_GPL(hisi_qp_send);
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+/* mdev->supported_type_groups */
+static struct attribute *hisi_qm_type_attrs[] = {
+	&mdev_type_attr_device_api.attr,
+	&mdev_type_attr_available_instances.attr,
+	&mdev_type_attr_type.attr,
+	NULL,
+};
+
+/* Other supported types algorithms can be added here */
+static struct attribute_group hisi_zlib_type_group = {
+	.name = "zlib",
+	.attrs = hisi_qm_type_attrs,
+};
+
+static struct attribute_group hisi_gzip_type_group = {
+	.name = "gzip",
+	.attrs = hisi_qm_type_attrs,
+};
+
+static struct attribute_group hisi_rsa_type_group = {
+	.name = "rsa",
+	.attrs = hisi_qm_type_attrs,
+};
+
+static struct attribute_group hisi_dh_type_group = {
+	.name = "dh",
+	.attrs = hisi_qm_type_attrs,
+};
+
+static struct attribute_group *mdev_zip_type_groups[] = {
+	&hisi_zlib_type_group,
+	&hisi_gzip_type_group,
+	NULL,
+};
+
+static struct attribute_group *mdev_hpre_type_groups[] = {
+	&hisi_rsa_type_group,
+	&hisi_dh_type_group,
+	NULL,
+};
+
+static struct attribute *mdev_dev_attrs[] = {
+	&dev_attr_pid.attr,
+	NULL,
+};
+
+static const struct attribute_group mdev_dev_group = {
+	.name  = VFIO_SPIMDEV_MDEV_ATTRS_GRP,
+	.attrs = mdev_dev_attrs,
+};
+
+/* this will be showed under virtual device's directory */
+static const  struct attribute_group *mdev_dev_groups[] = {
+	&mdev_dev_group,
+	NULL,
+};
+
+static int hisi_qm_get_queue(struct vfio_spimdev *spimdev, const char *alg,
+			     struct vfio_spimdev_queue **q)
+{
+	struct qm_info *qm = spimdev->priv;
+	struct hisi_qp *qp = NULL;
+	struct vfio_spimdev_queue *wd_q;
+	u8 alg_type = 0;
+	int ret = 0;
+
+	/* alg_type can be gotten from alg name */
+	qp = hisi_qm_create_qp(qm, alg_type);
+	if (IS_ERR(qp))
+		return PTR_ERR(qp);
+
+	wd_q = kzalloc(sizeof(struct vfio_spimdev_queue), GFP_KERNEL);
+	if (!wd_q) {
+		ret = -ENOMEM;
+		goto err_with_qp;
+	}
+
+	wd_q->priv = qp;
+	wd_q->spimdev = spimdev;
+	wd_q->alg = alg;
+	*q = wd_q;
+	qp->spimdev_q = wd_q;
+
+	/* PASID is 0 now, to be fixed */
+	ret = qm->ops->qm_start_qp(qp, 0);
+	if (ret < 0)
+		goto err_with_wd_q;
+	wd_q->qid = ret;
+	return ret;
+
+err_with_wd_q:
+	kfree(wd_q);
+err_with_qp:
+	hisi_qm_release_qp(qp);
+	return ret;
+}
+
+static int hisi_qm_put_queue(struct vfio_spimdev_queue *q)
+{
+	struct hisi_qp *qp = q->priv;
+
+	/* need to stop hardware, but can not support in v1 */
+	hisi_qm_release_qp(qp);
+	kfree(q);
+	return 0;
+}
+
+static int hisi_qm_reset_queue(struct vfio_spimdev_queue *q)
+{
+	struct hisi_qp *qp = q->priv;
+
+	qm_init_qp_status(qp);
+	return hisi_qm_start_qp(qp, 0);
+}
+
+/* map sq/cq/doorbell to user space */
+static int hisi_qm_mmap(struct vfio_spimdev_queue *q,
+			struct vm_area_struct *vma)
+{
+	struct hisi_qp *qp = (struct hisi_qp *)q->priv;
+	struct qm_info *qm = qp->qm;
+	struct device *dev = &qm->pdev->dev;
+	size_t sz = vma->vm_end - vma->vm_start;
+	u8 region;
+
+	vma->vm_flags |= (VM_IO | VM_LOCKED | VM_DONTEXPAND | VM_DONTDUMP);
+	region = _VFIO_SPIMDEV_REGION(vma->vm_pgoff);
+
+	switch (region) {
+	case 0:
+		if (sz > PAGE_SIZE)
+			return -EINVAL;
+		/*
+		 * Warning: This is not safe as multiple queues use the same
+		 * doorbell, v1 hardware interface problem. v2 will fix it
+		 */
+		if (qm->ver == QM_HW_V2)
+			return remap_pfn_range(vma, vma->vm_start,
+					       (qm->phys_base + PAGE_SIZE) >>
+					       PAGE_SHIFT, sz,
+					       pgprot_noncached(
+					       vma->vm_page_prot));
+		else if (qm->ver == QM_HW_V1)
+			return remap_pfn_range(vma, vma->vm_start,
+				       qm->phys_base >> PAGE_SHIFT,
+				       sz, pgprot_noncached(vma->vm_page_prot));
+		else
+			return -ENODEV;
+	case 1:
+		vma->vm_pgoff = 0;
+		if (sz > qp->scqe.size)
+			return -EINVAL;
+
+		return dma_mmap_coherent(dev, vma, qp->scqe.addr, qp->scqe.dma,
+					 sz);
+
+	default:
+		return -EINVAL;
+	}
+}
+
+static int hisi_qm_get_available_qnum(struct vfio_spimdev *spimdev)
+{
+	int num;
+	struct qm_info *qm = spimdev->priv;
+
+	read_lock(&qm->qps_lock);
+	num = qm->free_qp;
+	read_unlock(&qm->qps_lock);
+
+	return num;
+}
+
+static long qm_ioctl(struct vfio_spimdev_queue *q, unsigned int cmd,
+		     unsigned long arg)
+{
+	struct vfio_spimdev *spimdev = q->spimdev;
+	struct hisi_qp *qp = q->priv;
+	unsigned long long phy;
+	void *page;
+	int order;
+	struct page *page_list;
+	u64 size;
+
+	switch (cmd) {
+	/* While supporting NO-IOMMU, we need this */
+	case _GET_DMA_PAGES:
+#ifdef CONFIG_STRICT_DEVMEM
+		dev_err(spimdev->dev, "\nSTRICT_DEVMEM is on!");
+		return -EPERM;
+#endif
+		if (copy_from_user(&size, (unsigned long long *)arg,
+				   sizeof(u64)))
+			return -EFAULT;
+		if (size == 0)
+			return -EINVAL;
+		order = get_order(size);
+		if (order > MAX_ORDER)
+			return -ENOMEM;
+		page_list = alloc_pages_node(spimdev->node_id, GFP_DMA, order);
+		if (!page_list) {
+			dev_err(spimdev->dev,
+				"alloc 2^%d pages fail!\n", order);
+			return -ENOMEM;
+		}
+		page = page_address(page_list);
+		phy = (unsigned long long)virt_to_phys(page);
+		qp->udma_buf.phy_addr = phy;
+		qp->udma_buf.order = order;
+
+		if (copy_to_user((unsigned long long *)arg, &phy,
+		    sizeof(unsigned long long)))
+			return -EFAULT;
+		break;
+	case _PUT_DMA_PAGES:
+		if (copy_from_user(&phy,
+				(unsigned long long *)arg,
+				sizeof(unsigned long long)))
+			return -EFAULT;
+		phy = phy & PAGE_MASK;
+		order = qp->udma_buf.order;
+		if (order > MAX_ORDER || phy != qp->udma_buf.phy_addr)
+			return -EINVAL;
+		__free_pages(phys_to_page(phy), order);
+		qp->udma_buf.phy_addr = 0;
+
+		break;
+	default:
+		dev_err(spimdev->dev,
+	 "%s, ioctl cmd (0x%x) is not supported!\n", __func__, cmd);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+static const struct vfio_spimdev_ops qm_ops = {
+	.get_queue = hisi_qm_get_queue,
+	.put_queue = hisi_qm_put_queue,
+	.mmap = hisi_qm_mmap,
+	.get_available_instances = hisi_qm_get_available_qnum,
+	.ioctl = qm_ioctl,
+	.reset_queue = hisi_qm_reset_queue,
+};
+
+static int qm_register_spimdev(struct qm_info *qm)
+{
+	struct pci_dev *pdev = qm->pdev;
+	struct vfio_spimdev *spimdev = &qm->spimdev;
+
+#ifdef CONFIG_IOMMU_SVA
+	spimdev->dma_flag = VFIO_SPIMDEV_DMA_MULTI_PROC_MAP;
+#endif
+
+#ifdef CONFIG_ARM_SMMU_V3
+	spimdev->iommu_type = VFIO_TYPE1_IOMMU;
+	spimdev->dma_flag = VFIO_SPIMDEV_DMA_SINGLE_PROC_MAP;
+#else
+	spimdev->iommu_type = VFIO_SPIMDEV_IOMMU;
+	spimdev->dma_flag = VFIO_SPIMDEV_DMA_PHY;
+#endif
+	spimdev->owner = THIS_MODULE;
+	spimdev->name = qm->dev_name;
+	spimdev->dev = &pdev->dev;
+	spimdev->is_vf = pdev->is_virtfn;
+	spimdev->priv = qm;
+	if (qm->ver == QM_HW_V1)
+		spimdev->api_ver = "hisi_qm_v1";
+	else
+		spimdev->api_ver = "hisi_qm_v2";
+	spimdev->flags = VFIO_SPIMDEV_SAME_ALG_QFLG;
+	qm->mdev_dev_groups = mdev_dev_groups;
+	spimdev->mdev_fops.mdev_attr_groups = qm->mdev_dev_groups;
+	if (qm->type == ZIP)
+		spimdev->mdev_fops.supported_type_groups =
+						mdev_zip_type_groups;
+	else if (qm->type == HPRE)
+		spimdev->mdev_fops.supported_type_groups =
+						mdev_hpre_type_groups;
+	spimdev->ops = &qm_ops;
+
+	return vfio_spimdev_register(spimdev);
+}
+#endif
+
+int hisi_qm_init(struct qm_info *qm, enum qm_type type)
+{
+	struct pci_dev *pdev = qm->pdev;
+	int ret;
+
+	ret = pci_enable_device_mem(pdev);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Can't enable device mem!\n");
+		return ret;
+	}
+	qm->type = type;
+	qm->dev_name = pdev->driver->name;
+	ret = pci_request_mem_regions(pdev, qm->dev_name);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Can't request mem regions!\n");
+		goto err_with_pcidev;
+	}
+
+	qm->phys_base = pci_resource_start(pdev, 2);
+	qm->size = pci_resource_len(qm->pdev, 2);
+	qm->io_base = devm_ioremap(&pdev->dev, qm->phys_base, qm->size);
+	if (!qm->io_base) {
+		ret = -EIO;
+		dev_err(&pdev->dev, "ioremap qm physical base fail!\n");
+		goto err_with_mem_regions;
+	}
+
+	dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
+	pci_set_master(pdev);
+
+	ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
+	if (ret < 0) {
+		dev_err(&pdev->dev, "Enable MSI vectors fail!\n");
+		goto err_with_mem_regions;
+	}
+
+	qm->eq_head = 0;
+	mutex_init(&qm->mailbox_lock);
+	rwlock_init(&qm->qps_lock);
+
+	if (qm->ver == QM_HW_V1)
+		qm->ops = &qm_hw_ops_v1;
+	else if (qm->ver == QM_HW_V2)
+		qm->ops = &qm_hw_ops_v2;
+
+	return 0;
+
+err_with_mem_regions:
+	pci_release_mem_regions(pdev);
+err_with_pcidev:
+	pci_disable_device(pdev);
+
+	return ret;
+}
+EXPORT_SYMBOL_GPL(hisi_qm_init);
+
+void hisi_qm_uninit(struct qm_info *qm)
+{
+	struct pci_dev *pdev = qm->pdev;
+
+	pci_free_irq_vectors(pdev);
+	pci_release_mem_regions(pdev);
+	pci_disable_device(pdev);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_uninit);
+
+static irqreturn_t qm_irq(int irq, void *data)
+{
+	struct qm_info *qm = data;
+	u32 int_source;
+
+	int_source = qm_get_irq_source(qm);
+	if (int_source)
+		return IRQ_WAKE_THREAD;
+
+	dev_err(&qm->pdev->dev, "invalid int source %d\n", int_source);
+
+	return IRQ_HANDLED;
+}
+
+int hisi_qm_start(struct qm_info *qm)
+{
+	struct pci_dev *pdev = qm->pdev;
+	struct device *dev = &pdev->dev;
+	int ret;
+
+	if (qm->pdev->is_physfn)
+		qm->ops->vft_config(qm, qm->qp_base, qm->qp_num);
+
+	ret = qm_init_q_buffer(dev, sizeof(struct eqc), &qm->eqc);
+	if (ret)
+		goto err_out;
+
+	ret = qm_init_q_buffer(dev, sizeof(struct eqe) * QM_Q_DEPTH, &qm->eqe);
+	if (ret)
+		goto err_with_eqc;
+
+	QM_EQC(qm)->base_l = lower_32_bits(qm->eqe.dma);
+	QM_EQC(qm)->base_h = upper_32_bits(qm->eqe.dma);
+	QM_EQC(qm)->dw3 = 2 << MB_EQC_EQE_SHIFT;
+	QM_EQC(qm)->dw6 = (QM_Q_DEPTH - 1) | (1 << MB_EQC_PHASE_SHIFT);
+	ret = qm_mb(qm, MAILBOX_CMD_EQC, qm->eqc.dma, 0, 0, 0);
+	if (ret)
+		goto err_with_eqe;
+
+	qm->qp_bitmap = kcalloc(BITS_TO_LONGS(qm->qp_num), sizeof(long),
+				GFP_KERNEL);
+	if (!qm->qp_bitmap)
+		goto err_with_eqe;
+
+	qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp *),
+			       GFP_KERNEL);
+	if (!qm->qp_array)
+		goto err_with_bitmap;
+
+	/* Init sqc_bt */
+	ret = qm_init_bt(qm, dev, sizeof(struct sqc) * qm->qp_num, &qm->sqc,
+			 MAILBOX_CMD_SQC_BT);
+	if (ret)
+		goto err_with_qp_array;
+
+	/* Init cqc_bt */
+	ret = qm_init_bt(qm, dev, sizeof(struct cqc) * qm->qp_num, &qm->cqc,
+			 MAILBOX_CMD_CQC_BT);
+	if (ret)
+		goto err_with_sqc;
+
+	ret = request_threaded_irq(pci_irq_vector(pdev, 0), qm_irq,
+				   qm_irq_thread, IRQF_SHARED, qm->dev_name,
+				   qm);
+	if (ret)
+		goto err_with_cqc;
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+	ret = qm_register_spimdev(qm);
+	if (ret) {
+		dev_err(dev, "Hisilicon QM register to SPIMDEV fail!\n");
+		goto err_with_cqc;
+	}
+#endif
+
+	writel(0x0, QM_ADDR(qm, QM_VF_EQ_INT_MASK));
+
+	return 0;
+
+err_with_cqc:
+	qm_uninit_q_buffer(dev, &qm->cqc);
+err_with_sqc:
+	qm_uninit_q_buffer(dev, &qm->sqc);
+err_with_qp_array:
+	kfree(qm->qp_array);
+err_with_bitmap:
+	kfree(qm->qp_bitmap);
+err_with_eqe:
+	qm_uninit_q_buffer(dev, &qm->eqe);
+err_with_eqc:
+	qm_uninit_q_buffer(dev, &qm->eqc);
+err_out:
+	return ret;
+}
+EXPORT_SYMBOL_GPL(hisi_qm_start);
+
+void hisi_qm_stop(struct qm_info *qm)
+{
+	struct pci_dev *pdev = qm->pdev;
+	struct device *dev = &pdev->dev;
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+	vfio_spimdev_unregister(&qm->spimdev);
+#endif
+
+	free_irq(pci_irq_vector(pdev, 0), qm);
+	qm_uninit_q_buffer(dev, &qm->cqc);
+	kfree(qm->qp_array);
+	kfree(qm->qp_bitmap);
+	qm_uninit_q_buffer(dev, &qm->eqe);
+	qm_uninit_q_buffer(dev, &qm->eqc);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_stop);
+
+/* put qm into init state, so the acce config become available */
+int hisi_qm_mem_start(struct qm_info *qm)
+{
+	u32 val;
+
+	qm_writel(qm, 0x1, QM_MEM_START_INIT);
+	return readl_relaxed_poll_timeout(QM_ADDR(qm, QM_MEM_INIT_DONE), val,
+					  val & BIT(0), 10, 1000);
+}
+EXPORT_SYMBOL_GPL(hisi_qm_mem_start);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
+MODULE_DESCRIPTION("HiSilicon Accelerator queue manager driver");

drivers/crypto/hisilicon/qm.h

+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef HISI_ACC_QM_H
+#define HISI_ACC_QM_H
+
+#include <linux/dmapool.h>
+#include <linux/iopoll.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+#include <linux/slab.h>
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+#include <linux/vfio_spimdev.h>
+#endif
+
+#define QM_HW_V1			1
+#define QM_HW_V2			2
+
+#define QM_CQE_SIZE			16
+/* default queue depth for sq/cq/eq */
+#define QM_Q_DEPTH			1024
+
+/* qm user domain */
+#define QM_ARUSER_M_CFG_1		0x100088
+#define QM_ARUSER_M_CFG_ENABLE		0x100090
+#define QM_AWUSER_M_CFG_1		0x100098
+#define QM_AWUSER_M_CFG_ENABLE		0x1000a0
+#define QM_WUSER_M_CFG_ENABLE		0x1000a8
+
+/* qm cache */
+#define QM_CACHE_CTL			0x100050
+#define QM_AXI_M_CFG			0x1000ac
+#define QM_AXI_M_CFG_ENABLE		0x1000b0
+#define QM_PEH_AXUSER_CFG		0x1000cc
+#define QM_PEH_AXUSER_CFG_ENABLE	0x1000d0
+
+#define QP_SQE_ADDR(qp) ((qp)->scqe.addr)
+#define _GET_DMA_PAGES		_IOW('d', 3, unsigned long long)
+#define _PUT_DMA_PAGES		_IOW('d', 4, unsigned long long)
+enum qm_type {
+	ZIP = 1,
+	HPRE,
+	SEC,
+};
+
+struct qm_dma_buffer {
+	int size;
+	void *addr;
+	dma_addr_t dma;
+};
+
+struct qm_info {
+	int ver;
+	enum qm_type type;
+	const char *dev_name;
+	struct pci_dev *pdev;
+	resource_size_t phys_base;
+	resource_size_t size;
+	void __iomem *io_base;
+	u32 sqe_size;
+	u32 qp_base;
+	u32 qp_num;
+	u32 free_qp;
+	struct qm_dma_buffer sqc, cqc, eqc, eqe;
+	u32 eq_head;
+	rwlock_t qps_lock;
+	unsigned long *qp_bitmap;
+	struct hisi_qp **qp_array;
+	struct mutex mailbox_lock;
+	struct hisi_acc_qm_hw_ops *ops;
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+	struct vfio_spimdev spimdev;
+	const struct attribute_group **mdev_dev_groups;
+#endif
+};
+#define QM_ADDR(qm, off) ((qm)->io_base + off)
+
+struct hisi_acc_qp_status {
+	u16 sq_tail;
+	u16 sq_head;
+	u16 cq_head;
+	bool cqc_phase;
+	int is_sq_full;
+};
+
+struct hisi_qp;
+
+struct hisi_qp_ops {
+	int (*fill_sqe)(void *sqe, void *q_parm, void *d_parm);
+};
+
+struct qp_phy_pages {
+	u64 size;
+	int order;
+	int node_id;
+	u64 phy_addr;
+};
+
+struct hisi_qp {
+	/* sq number in this function */
+	u32 queue_id;
+	u8 alg_type;
+	u8 req_type;
+	struct qm_dma_buffer sqc, cqc;
+	struct qm_dma_buffer scqe;
+	struct hisi_acc_qp_status qp_status;
+	struct qm_info *qm;
+	struct qp_phy_pages udma_buf;/* For user space */
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+	struct vfio_spimdev_queue *spimdev_q;
+#endif
+
+	/* for crypto sync API */
+	struct completion completion;
+
+	struct hisi_qp_ops *hw_ops;
+	void *qp_ctx;
+	void (*event_cb)(struct hisi_qp *qp);
+	void (*req_cb)(struct hisi_qp *qp, void *data);
+};
+
+extern int hisi_qm_init(struct qm_info *qm, enum qm_type type);
+extern void hisi_qm_uninit(struct qm_info *qm);
+extern int hisi_qm_start(struct qm_info *qm);
+extern void hisi_qm_stop(struct qm_info *qm);
+extern int hisi_qm_mem_start(struct qm_info *qm);
+extern struct hisi_qp *hisi_qm_create_qp(struct qm_info *qm, u8 alg_type);
+extern int hisi_qm_start_qp(struct hisi_qp *qp, unsigned long arg);
+extern void hisi_qm_release_qp(struct hisi_qp *qp);
+extern int hisi_qp_send(struct hisi_qp *qp, void *msg);
+#endif

drivers/crypto/hisilicon/zip/Makefile

+obj-$(CONFIG_CRYPTO_DEV_HISI_ZIP) += hisi_zip.o
+hisi_zip-objs = zip_main.o zip_crypto.o

drivers/crypto/hisilicon/zip/zip.h

+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef HISI_ZIP_H
+#define HISI_ZIP_H
+
+#include <linux/list.h>
+#include "../qm.h"
+
+#define HZIP_SQE_SIZE			128
+#define HZIP_SQ_SIZE			(HZIP_SQE_SIZE * QM_Q_DEPTH)
+#define QM_CQ_SIZE			(QM_CQE_SIZE * QM_Q_DEPTH)
+#define HZIP_PF_DEF_Q_NUM		64
+#define HZIP_PF_DEF_Q_BASE		0
+
+struct hisi_zip {
+	struct qm_info qm;
+	struct list_head list;
+
+#ifdef CONFIG_CRYPTO_DEV_HISI_SPIMDEV
+	struct vfio_spimdev *spimdev;
+#endif
+};
+
+struct hisi_zip_sqe {
+	__u32 consumed;
+	__u32 produced;
+	__u32 comp_data_length;
+	__u32 dw3;
+	__u32 input_data_length;
+	__u32 lba_l;
+	__u32 lba_h;
+	__u32 dw7;
+	__u32 dw8;
+	__u32 dw9;
+	__u32 dw10;
+	__u32 priv_info;
+	__u32 dw12;
+	__u32 tag;
+	__u32 dest_avail_out;
+	__u32 rsvd0;
+	__u32 comp_head_addr_l;
+	__u32 comp_head_addr_h;
+	__u32 source_addr_l;
+	__u32 source_addr_h;
+	__u32 dest_addr_l;
+	__u32 dest_addr_h;
+	__u32 stream_ctx_addr_l;
+	__u32 stream_ctx_addr_h;
+	__u32 cipher_key1_addr_l;
+	__u32 cipher_key1_addr_h;
+	__u32 cipher_key2_addr_l;
+	__u32 cipher_key2_addr_h;
+	__u32 rsvd1[4];
+};
+
+extern struct list_head hisi_zip_list;
+
+#endif

drivers/crypto/hisilicon/zip/zip_crypto.c

+// SPDX-License-Identifier: GPL-2.0+
+#include <linux/crypto.h>
+#include <linux/dma-mapping.h>
+#include <linux/pci.h>
+#include <linux/topology.h>
+#include "../qm.h"
+#include "zip.h"
+
+#define INPUT_BUFFER_SIZE	(64 * 1024)
+#define OUTPUT_BUFFER_SIZE	(64 * 1024)
+
+#define COMP_NAME_TO_TYPE(alg_name)			\
+	(!strcmp((alg_name), "zlib-deflate") ? 0x02 :	\
+	 !strcmp((alg_name), "gzip") ? 0x03 : 0)	\
+
+struct hisi_zip_buffer {
+	u8 *input;
+	dma_addr_t input_dma;
+	u8 *output;
+	dma_addr_t output_dma;
+};
+
+struct hisi_zip_qp_ctx {
+	struct hisi_zip_buffer buffer;
+	struct hisi_qp *qp;
+	struct hisi_zip_sqe zip_sqe;
+};
+
+struct hisi_zip_ctx {
+#define QPC_COMP	0
+#define QPC_DECOMP	1
+	struct hisi_zip_qp_ctx qp_ctx[2];
+};
+
+static struct hisi_zip *find_zip_device(int node)
+{
+	struct hisi_zip *hisi_zip, *ret = NULL;
+	struct device *dev;
+	int min_distance = 100;
+	int dev_node = 0;
+
+	list_for_each_entry(hisi_zip, &hisi_zip_list, list) {
+		dev = &hisi_zip->qm.pdev->dev;
+#ifdef CONFIG_NUMA
+		dev_node = dev->numa_node;
+#endif
+		if (node_distance(dev_node, node) < min_distance) {
+			ret = hisi_zip;
+			min_distance = node_distance(dev_node, node);
+		}
+	}
+
+	return ret;
+}
+
+static void hisi_zip_qp_event_notifier(struct hisi_qp *qp)
+{
+	complete(&qp->completion);
+}
+
+static int hisi_zip_fill_sqe_v1(void *sqe, void *q_parm, u32 len)
+{
+	struct hisi_zip_sqe *zip_sqe = (struct hisi_zip_sqe *)sqe;
+	struct hisi_zip_qp_ctx *qp_ctx = (struct hisi_zip_qp_ctx *)q_parm;
+	struct hisi_zip_buffer *buffer = &qp_ctx->buffer;
+
+	memset(zip_sqe, 0, sizeof(struct hisi_zip_sqe));
+
+	zip_sqe->input_data_length = len;
+	zip_sqe->dw9 = qp_ctx->qp->req_type;
+	zip_sqe->dest_avail_out = OUTPUT_BUFFER_SIZE;
+	zip_sqe->source_addr_l = lower_32_bits(buffer->input_dma);
+	zip_sqe->source_addr_h = upper_32_bits(buffer->input_dma);
+	zip_sqe->dest_addr_l = lower_32_bits(buffer->output_dma);
+	zip_sqe->dest_addr_h = upper_32_bits(buffer->output_dma);
+
+	return 0;
+}
+
+/* let's allocate one buffer now, may have problem in async case */
+static int hisi_zip_alloc_qp_buffer(struct hisi_zip_qp_ctx *hisi_zip_qp_ctx)
+{
+	struct hisi_zip_buffer *buffer = &hisi_zip_qp_ctx->buffer;
+	struct hisi_qp *qp = hisi_zip_qp_ctx->qp;
+	struct device *dev = &qp->qm->pdev->dev;
+	int ret;
+
+	buffer->input = dma_alloc_coherent(dev, INPUT_BUFFER_SIZE,
+					   &buffer->input_dma, GFP_KERNEL);
+	if (!buffer->input)
+		return -ENOMEM;
+
+	buffer->output = dma_alloc_coherent(dev, OUTPUT_BUFFER_SIZE,
+					    &buffer->output_dma, GFP_KERNEL);
+	if (!buffer->output) {
+		ret = -ENOMEM;
+		goto err_alloc_output_buffer;
+	}
+
+	return 0;
+
+err_alloc_output_buffer:
+	dma_free_coherent(dev, INPUT_BUFFER_SIZE, buffer->input,
+			  buffer->input_dma);
+	return ret;
+}
+
+static void hisi_zip_free_qp_buffer(struct hisi_zip_qp_ctx *hisi_zip_qp_ctx)
+{
+	struct hisi_zip_buffer *buffer = &hisi_zip_qp_ctx->buffer;
+	struct hisi_qp *qp = hisi_zip_qp_ctx->qp;
+	struct device *dev = &qp->qm->pdev->dev;
+
+	dma_free_coherent(dev, INPUT_BUFFER_SIZE, buffer->input,
+			  buffer->input_dma);
+	dma_free_coherent(dev, OUTPUT_BUFFER_SIZE, buffer->output,
+			  buffer->output_dma);
+}
+
+static int hisi_zip_create_qp(struct qm_info *qm, struct hisi_zip_qp_ctx *ctx,
+			      int alg_type, int req_type)
+{
+	struct hisi_qp *qp;
+	int ret;
+
+	qp = hisi_qm_create_qp(qm, alg_type);
+
+	if (IS_ERR(qp))
+		return PTR_ERR(qp);
+
+	qp->event_cb = hisi_zip_qp_event_notifier;
+	qp->req_type = req_type;
+
+	qp->qp_ctx = ctx;
+	ctx->qp = qp;
+
+	ret = hisi_zip_alloc_qp_buffer(ctx);
+	if (ret)
+		goto err_with_qp;
+
+	ret = hisi_qm_start_qp(qp, 0);
+	if (ret < 0)
+		goto err_with_qp_buffer;
+
+	return 0;
+err_with_qp_buffer:
+	hisi_zip_free_qp_buffer(ctx);
+err_with_qp:
+	hisi_qm_release_qp(qp);
+	return ret;
+}
+
+static void hisi_zip_release_qp(struct hisi_zip_qp_ctx *ctx)
+{
+	hisi_qm_release_qp(ctx->qp);
+	hisi_zip_free_qp_buffer(ctx);
+}
+
+static int hisi_zip_alloc_comp_ctx(struct crypto_tfm *tfm)
+{
+	struct hisi_zip_ctx *hisi_zip_ctx = crypto_tfm_ctx(tfm);
+	const char *alg_name = crypto_tfm_alg_name(tfm);
+	struct hisi_zip *hisi_zip;
+	struct qm_info *qm;
+	int ret, i, j;
+
+	u8 req_type = COMP_NAME_TO_TYPE(alg_name);
+
+	/* find the proper zip device */
+	hisi_zip = find_zip_device(cpu_to_node(smp_processor_id()));
+	if (!hisi_zip) {
+		pr_err("Can not find proper ZIP device!\n");
+		return -ENODEV;
+	}
+	qm = &hisi_zip->qm;
+
+	for (i = 0; i < 2; i++) {
+	/* it is just happen that 0 is compress, 1 is decompress on alg_type */
+		ret = hisi_zip_create_qp(qm, &hisi_zip_ctx->qp_ctx[i], i,
+					 req_type);
+		if (ret)
+			goto err;
+	}
+
+	return 0;
+err:
+	for (j = i-1; j >= 0; j--)
+		hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[j]);
+
+	return ret;
+}
+
+static void hisi_zip_free_comp_ctx(struct crypto_tfm *tfm)
+{
+	struct hisi_zip_ctx *hisi_zip_ctx = crypto_tfm_ctx(tfm);
+	int i;
+
+	/* release the qp */
+	for (i = 1; i >= 0; i--)
+		hisi_zip_release_qp(&hisi_zip_ctx->qp_ctx[i]);
+}
+
+static int hisi_zip_copy_data_to_buffer(struct hisi_zip_qp_ctx *qp_ctx,
+					const u8 *src, unsigned int slen)
+{
+	struct hisi_zip_buffer *buffer = &qp_ctx->buffer;
+
+	if (slen > INPUT_BUFFER_SIZE)
+		return -EINVAL;
+
+	memcpy(buffer->input, src, slen);
+
+	return 0;
+}
+
+static struct hisi_zip_sqe *hisi_zip_get_writeback_sqe(struct hisi_qp *qp)
+{
+	struct hisi_acc_qp_status *qp_status = &qp->qp_status;
+	struct hisi_zip_sqe *sq_base = QP_SQE_ADDR(qp);
+	u16 sq_head = qp_status->sq_head;
+
+	return sq_base + sq_head;
+}
+
+static int hisi_zip_copy_data_from_buffer(struct hisi_zip_qp_ctx *qp_ctx,
+					  u8 *dst, unsigned int *dlen)
+{
+	struct hisi_zip_buffer *buffer = &qp_ctx->buffer;
+	struct hisi_qp *qp = qp_ctx->qp;
+	struct hisi_zip_sqe *zip_sqe = hisi_zip_get_writeback_sqe(qp);
+	u32 status = zip_sqe->dw3 & 0xff;
+	u16 sq_head;
+
+	if (status != 0) {
+		pr_err("hisi zip: %s fail!\n", (qp->alg_type == 0) ?
+		       "compression" : "decompression");
+		return status;
+	}
+
+	if (zip_sqe->produced > OUTPUT_BUFFER_SIZE)
+		return -ENOMEM;
+
+	memcpy(dst, buffer->output, zip_sqe->produced);
+	*dlen = zip_sqe->produced;
+
+	sq_head = qp->qp_status.sq_head;
+	if (sq_head == QM_Q_DEPTH - 1)
+		qp->qp_status.sq_head = 0;
+	else
+		qp->qp_status.sq_head++;
+
+	return 0;
+}
+
+static int hisi_zip_compress(struct crypto_tfm *tfm, const u8 *src,
+			     unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	struct hisi_zip_ctx *hisi_zip_ctx = crypto_tfm_ctx(tfm);
+	struct hisi_zip_qp_ctx *qp_ctx = &hisi_zip_ctx->qp_ctx[QPC_COMP];
+	struct hisi_qp *qp = qp_ctx->qp;
+	struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe;
+	int ret;
+
+	ret = hisi_zip_copy_data_to_buffer(qp_ctx, src, slen);
+	if (ret < 0)
+		return ret;
+
+	hisi_zip_fill_sqe_v1(zip_sqe, qp_ctx, slen);
+
+	/* send command to start the compress job */
+	hisi_qp_send(qp, zip_sqe);
+
+	return hisi_zip_copy_data_from_buffer(qp_ctx, dst, dlen);
+}
+
+static int hisi_zip_decompress(struct crypto_tfm *tfm, const u8 *src,
+			       unsigned int slen, u8 *dst, unsigned int *dlen)
+{
+	struct hisi_zip_ctx *hisi_zip_ctx = crypto_tfm_ctx(tfm);
+	struct hisi_zip_qp_ctx *qp_ctx = &hisi_zip_ctx->qp_ctx[QPC_DECOMP];
+	struct hisi_qp *qp = qp_ctx->qp;
+	struct hisi_zip_sqe *zip_sqe = &qp_ctx->zip_sqe;
+	int ret;
+
+	ret = hisi_zip_copy_data_to_buffer(qp_ctx, src, slen);
+	if (ret < 0)
+		return ret;
+
+	hisi_zip_fill_sqe_v1(zip_sqe, qp_ctx, slen);
+
+	/* send command to start the decompress job */
+	hisi_qp_send(qp, zip_sqe);
+
+	return hisi_zip_copy_data_from_buffer(qp_ctx, dst, dlen);
+}
+
+static struct crypto_alg hisi_zip_zlib = {
+	.cra_name		= "zlib-deflate",
+	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
+	.cra_ctxsize		= sizeof(struct hisi_zip_ctx),
+	.cra_priority           = 300,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= hisi_zip_alloc_comp_ctx,
+	.cra_exit		= hisi_zip_free_comp_ctx,
+	.cra_u			= {
+		.compress = {
+			.coa_compress	= hisi_zip_compress,
+			.coa_decompress	= hisi_zip_decompress
+		}
+	}
+};
+
+static struct crypto_alg hisi_zip_gzip = {
+	.cra_name		= "gzip",
+	.cra_flags		= CRYPTO_ALG_TYPE_COMPRESS,
+	.cra_ctxsize		= sizeof(struct hisi_zip_ctx),
+	.cra_priority           = 300,
+	.cra_module		= THIS_MODULE,
+	.cra_init		= hisi_zip_alloc_comp_ctx,
+	.cra_exit		= hisi_zip_free_comp_ctx,
+	.cra_u			= {
+		.compress = {
+			.coa_compress	= hisi_zip_compress,
+			.coa_decompress	= hisi_zip_decompress
+		}
+	}
+};
+
+int hisi_zip_register_to_crypto(void)
+{
+	int ret;
+
+	ret = crypto_register_alg(&hisi_zip_zlib);
+	if (ret < 0) {
+		pr_err("Zlib algorithm registration failed\n");
+		return ret;
+	}
+
+	ret = crypto_register_alg(&hisi_zip_gzip);
+	if (ret < 0) {
+		pr_err("Gzip algorithm registration failed\n");
+		goto err_unregister_zlib;
+	}
+
+	return 0;
+
+err_unregister_zlib:
+	crypto_unregister_alg(&hisi_zip_zlib);
+
+	return ret;
+}
+
+void hisi_zip_unregister_from_crypto(void)
+{
+	crypto_unregister_alg(&hisi_zip_zlib);
+	crypto_unregister_alg(&hisi_zip_gzip);
+}

drivers/crypto/hisilicon/zip/zip_crypto.h

+/* SPDX-License-Identifier: GPL-2.0+ */
+#ifndef HISI_ZIP_CRYPTO_H
+#define HISI_ZIP_CRYPTO_H
+
+int hisi_zip_register_to_crypto(void);
+void hisi_zip_unregister_from_crypto(void);
+
+#endif

drivers/crypto/hisilicon/zip/zip_main.c

+// SPDX-License-Identifier: GPL-2.0+
+#include <linux/bitops.h>
+#include <linux/init.h>
+#include <linux/io.h>
+#include <linux/kernel.h>
+#include <linux/module.h>
+#include <linux/pci.h>
+
+#include "zip.h"
+#include "zip_crypto.h"
+
+#define HZIP_VF_NUM			63
+#define HZIP_QUEUE_NUM_V1		4096
+#define HZIP_QUEUE_NUM_V2		1024
+
+#define HZIP_FSM_MAX_CNT		0x301008
+
+#define HZIP_PORT_ARCA_CHE_0		0x301040
+#define HZIP_PORT_ARCA_CHE_1		0x301044
+#define HZIP_PORT_AWCA_CHE_0		0x301060
+#define HZIP_PORT_AWCA_CHE_1		0x301064
+
+#define HZIP_BD_RUSER_32_63		0x301110
+#define HZIP_SGL_RUSER_32_63		0x30111c
+#define HZIP_DATA_RUSER_32_63		0x301128
+#define HZIP_DATA_WUSER_32_63		0x301134
+#define HZIP_BD_WUSER_32_63		0x301140
+
+LIST_HEAD(hisi_zip_list);
+DEFINE_MUTEX(hisi_zip_list_lock);
+
+static const struct pci_device_id hisi_zip_dev_ids[] = {
+	{ PCI_DEVICE(PCI_VENDOR_ID_HUAWEI, 0xa250) },
+	{ 0, }
+};
+
+static inline void hisi_zip_add_to_list(struct hisi_zip *hisi_zip)
+{
+	mutex_lock(&hisi_zip_list_lock);
+	list_add_tail(&hisi_zip->list, &hisi_zip_list);
+	mutex_unlock(&hisi_zip_list_lock);
+}
+
+static inline void hisi_zip_remove_from_list(struct hisi_zip *hisi_zip)
+{
+	mutex_lock(&hisi_zip_list_lock);
+	list_del(&hisi_zip->list);
+	mutex_unlock(&hisi_zip_list_lock);
+}
+
+static void hisi_zip_set_user_domain_and_cache(struct hisi_zip *hisi_zip)
+{
+	u32 val;
+
+	/* qm user domain */
+	writel(0x40001070, hisi_zip->qm.io_base + QM_ARUSER_M_CFG_1);
+	writel(0xfffffffe, hisi_zip->qm.io_base + QM_ARUSER_M_CFG_ENABLE);
+	writel(0x40001070, hisi_zip->qm.io_base + QM_AWUSER_M_CFG_1);
+	writel(0xfffffffe, hisi_zip->qm.io_base + QM_AWUSER_M_CFG_ENABLE);
+	writel(0xffffffff, hisi_zip->qm.io_base + QM_WUSER_M_CFG_ENABLE);
+
+	val = readl(hisi_zip->qm.io_base + QM_PEH_AXUSER_CFG);
+	val |= (1 << 11);
+	writel(val, hisi_zip->qm.io_base + QM_PEH_AXUSER_CFG);
+
+	/* qm cache */
+	writel(0xffff,     hisi_zip->qm.io_base + QM_AXI_M_CFG);
+	writel(0xffffffff, hisi_zip->qm.io_base + QM_AXI_M_CFG_ENABLE);
+	writel(0xffffffff, hisi_zip->qm.io_base + QM_PEH_AXUSER_CFG_ENABLE);
+
+	/* cache */
+	writel(0xffffffff, hisi_zip->qm.io_base + HZIP_PORT_ARCA_CHE_0);
+	writel(0xffffffff, hisi_zip->qm.io_base + HZIP_PORT_ARCA_CHE_1);
+	writel(0xffffffff, hisi_zip->qm.io_base + HZIP_PORT_AWCA_CHE_0);
+	writel(0xffffffff, hisi_zip->qm.io_base + HZIP_PORT_AWCA_CHE_1);
+
+	/* user domain configurations */
+	writel(0x40001070, hisi_zip->qm.io_base + HZIP_BD_RUSER_32_63);
+	writel(0x40001070, hisi_zip->qm.io_base + HZIP_SGL_RUSER_32_63);
+#ifdef CONFIG_IOMMU_SVA
+	writel(0x40001071, hisi_zip->qm.io_base + HZIP_DATA_RUSER_32_63);
+	writel(0x40001071, hisi_zip->qm.io_base + HZIP_DATA_WUSER_32_63);
+#else
+	writel(0x40001070, hisi_zip->qm.io_base + HZIP_DATA_RUSER_32_63);
+	writel(0x40001070, hisi_zip->qm.io_base + HZIP_DATA_WUSER_32_63);
+#endif
+	writel(0x40001070, hisi_zip->qm.io_base + HZIP_BD_WUSER_32_63);
+
+	/* fsm count */
+	writel(0xfffffff, hisi_zip->qm.io_base + HZIP_FSM_MAX_CNT);
+
+	/* clock gating, core, decompress verify enable */
+	writel(0x10005, hisi_zip->qm.io_base + 0x301004);
+}
+
+static int hisi_zip_probe(struct pci_dev *pdev, const struct pci_device_id *id)
+{
+	struct hisi_zip *hisi_zip;
+	struct qm_info *qm;
+	int ret;
+	u8 rev_id;
+
+	hisi_zip = devm_kzalloc(&pdev->dev, sizeof(*hisi_zip), GFP_KERNEL);
+	if (!hisi_zip)
+		return -ENOMEM;
+	hisi_zip_add_to_list(hisi_zip);
+	pci_set_drvdata(pdev, hisi_zip);
+
+	qm = &hisi_zip->qm;
+	qm->pdev = pdev;
+
+	pci_read_config_byte(pdev, PCI_REVISION_ID, &rev_id);
+	if (rev_id == 0x20)
+		qm->ver = QM_HW_V1;
+	else if (rev_id == 0x21)
+		qm->ver = QM_HW_V2;
+	qm->sqe_size = HZIP_SQE_SIZE;
+
+	ret = hisi_qm_init(qm, ZIP);
+	if (ret)
+		goto err_with_hisi_zip;
+
+	if (pdev->is_physfn) {
+		ret = hisi_qm_mem_start(qm);
+		if (ret) {
+			dev_err(&pdev->dev, "Can't start QM mem of Hisilicon!\n");
+			goto err_with_qm_init;
+		}
+
+		hisi_zip_set_user_domain_and_cache(hisi_zip);
+
+		qm->qp_base = HZIP_PF_DEF_Q_BASE;
+		qm->qp_num = HZIP_PF_DEF_Q_NUM;
+		qm->free_qp = qm->qp_num;
+	}
+	ret = hisi_qm_start(qm);
+	if (ret) {
+		dev_err(&pdev->dev, "Can't start QM of Hisilicon!\n");
+		goto err_with_qm_init;
+	}
+
+	return 0;
+
+err_with_qm_init:
+	hisi_qm_uninit(qm);
+err_with_hisi_zip:
+	hisi_zip_remove_from_list(hisi_zip);
+	kfree(hisi_zip);
+	return ret;
+}
+
+static void hisi_zip_remove(struct pci_dev *pdev)
+{
+	struct hisi_zip *hisi_zip = pci_get_drvdata(pdev);
+	struct qm_info *qm = &hisi_zip->qm;
+
+	hisi_qm_stop(qm);
+	hisi_qm_uninit(qm);
+	hisi_zip_remove_from_list(hisi_zip);
+	kfree(hisi_zip);
+}
+
+static struct pci_driver hisi_zip_pci_driver = {
+	.name		= "hisi_zip",
+	.id_table	= hisi_zip_dev_ids,
+	.probe		= hisi_zip_probe,
+	.remove		= hisi_zip_remove,
+};
+
+static int __init hisi_zip_init(void)
+{
+	int ret;
+
+	ret = pci_register_driver(&hisi_zip_pci_driver);
+	if (ret < 0) {
+		pr_err("zip: can't register hisi zip driver.\n");
+		return ret;
+	}
+
+	ret = hisi_zip_register_to_crypto();
+	if (ret < 0) {
+		pr_err("zip: can't register hisi zip to crypto.\n");
+		pci_unregister_driver(&hisi_zip_pci_driver);
+		return ret;
+	}
+
+	return 0;
+}
+
+static void __exit hisi_zip_exit(void)
+{
+	hisi_zip_unregister_from_crypto();
+	pci_unregister_driver(&hisi_zip_pci_driver);
+}
+
+module_init(hisi_zip_init);
+module_exit(hisi_zip_exit);
+
+MODULE_LICENSE("GPL v2");
+MODULE_AUTHOR("Zhou Wang <wangzhou1@hisilicon.com>");
+MODULE_DESCRIPTION("Driver for HiSilicon ZIP accelerator");
+MODULE_DEVICE_TABLE(pci, hisi_zip_dev_ids);