From 1cd6253c4dda443505d52cdfc2c92a54afaf269b Mon Sep 17 00:00:00 2001 From: xuzaibo Date: Sat, 17 Nov 2018 11:45:22 +0800 Subject: [PATCH] Add user space Warpdrive with Hisilicon ZIP/HPRE driver Add user space Warpdrive with Hisilicon ZIP/HPRE. Signed-off-by: xuzaibo --- samples/warpdrive/AUTHORS | 2 + samples/warpdrive/ChangeLog | 1 + samples/warpdrive/Makefile.am | 14 + samples/warpdrive/NEWS | 1 + samples/warpdrive/README | 32 + samples/warpdrive/autogen.sh | 3 + samples/warpdrive/cleanup.sh | 13 + samples/warpdrive/configure.ac | 62 ++ samples/warpdrive/drv/hisi_hpre_udrv.c | 463 ++++++++++++ samples/warpdrive/drv/hisi_hpre_udrv.h | 108 +++ samples/warpdrive/drv/hisi_zip_udrv.c | 407 +++++++++++ samples/warpdrive/drv/hisi_zip_udrv.h | 56 ++ samples/warpdrive/test/Makefile.am | 9 + samples/warpdrive/test/comp_hw.h | 23 + samples/warpdrive/test/test_hisi_hpre.c | 330 +++++++++ samples/warpdrive/test/test_hisi_zip.c | 357 ++++++++++ samples/warpdrive/wd.c | 911 ++++++++++++++++++++++++ samples/warpdrive/wd.h | 92 +++ samples/warpdrive/wd_adapter.c | 171 +++++ samples/warpdrive/wd_adapter.h | 44 ++ samples/warpdrive/wd_comp.c | 157 ++++ samples/warpdrive/wd_comp.h | 91 +++ samples/warpdrive/wd_dh.c | 166 +++++ samples/warpdrive/wd_dh.h | 83 +++ samples/warpdrive/wd_rsa.c | 263 +++++++ samples/warpdrive/wd_rsa.h | 112 +++ samples/warpdrive/wd_util.c | 18 + samples/warpdrive/wd_util.h | 110 +++ 28 files changed, 4099 insertions(+) create mode 100644 samples/warpdrive/AUTHORS create mode 100644 samples/warpdrive/ChangeLog create mode 100644 samples/warpdrive/Makefile.am create mode 100644 samples/warpdrive/NEWS create mode 100644 samples/warpdrive/README create mode 100755 samples/warpdrive/autogen.sh create mode 100755 samples/warpdrive/cleanup.sh create mode 100644 samples/warpdrive/configure.ac create mode 100644 samples/warpdrive/drv/hisi_hpre_udrv.c create mode 100644 samples/warpdrive/drv/hisi_hpre_udrv.h create mode 100644 samples/warpdrive/drv/hisi_zip_udrv.c create mode 100644 samples/warpdrive/drv/hisi_zip_udrv.h create mode 100644 samples/warpdrive/test/Makefile.am create mode 100644 samples/warpdrive/test/comp_hw.h create mode 100644 samples/warpdrive/test/test_hisi_hpre.c create mode 100644 samples/warpdrive/test/test_hisi_zip.c create mode 100644 samples/warpdrive/wd.c create mode 100644 samples/warpdrive/wd.h create mode 100644 samples/warpdrive/wd_adapter.c create mode 100644 samples/warpdrive/wd_adapter.h create mode 100644 samples/warpdrive/wd_comp.c create mode 100644 samples/warpdrive/wd_comp.h create mode 100644 samples/warpdrive/wd_dh.c create mode 100644 samples/warpdrive/wd_dh.h create mode 100644 samples/warpdrive/wd_rsa.c create mode 100644 samples/warpdrive/wd_rsa.h create mode 100644 samples/warpdrive/wd_util.c create mode 100644 samples/warpdrive/wd_util.h diff --git a/samples/warpdrive/AUTHORS b/samples/warpdrive/AUTHORS new file mode 100644 index 000000000000..fe7dc2413b0d --- /dev/null +++ b/samples/warpdrive/AUTHORS @@ -0,0 +1,2 @@ +Kenneth Lee +Zaibo Xu diff --git a/samples/warpdrive/ChangeLog b/samples/warpdrive/ChangeLog new file mode 100644 index 000000000000..b1b716105590 --- /dev/null +++ b/samples/warpdrive/ChangeLog @@ -0,0 +1 @@ +init diff --git a/samples/warpdrive/Makefile.am b/samples/warpdrive/Makefile.am new file mode 100644 index 000000000000..81c78ac32b86 --- /dev/null +++ b/samples/warpdrive/Makefile.am @@ -0,0 +1,14 @@ +ACLOCAL_AMFLAGS = -I m4 +AUTOMAKE_OPTIONS = foreign subdir-objects +AM_CFLAGS=-Wall -O0 -fno-strict-aliasing + +if DYNDRV +lib_LTLIBRARIES=libwd.la libhisi_zip.la libhisi_hpre.la +libwd_la_SOURCES=wd.c wd_adapter.c wd_util.c wd_comp.c wd_rsa.c wd_dh.c +libhisi_zip_la_SOURCES=drv/hisi_zip_udrv.c +libhisi_hpre_la_SOURCES=drv/hisi_hpre_udrv.c +else +lib_LTLIBRARIES=libwd.la +libwd_la_SOURCES=wd.c wd_adapter.c wd_util.c wd_comp.c wd_rsa.c wd_dh.c drv/hisi_zip_udrv.c drv/hisi_hpre_udrv.c +endif +SUBDIRS=. test diff --git a/samples/warpdrive/NEWS b/samples/warpdrive/NEWS new file mode 100644 index 000000000000..b1b716105590 --- /dev/null +++ b/samples/warpdrive/NEWS @@ -0,0 +1 @@ +init diff --git a/samples/warpdrive/README b/samples/warpdrive/README new file mode 100644 index 000000000000..3adf66b112fc --- /dev/null +++ b/samples/warpdrive/README @@ -0,0 +1,32 @@ +WD User Land Demonstration +========================== + +This directory contains some applications and libraries to demonstrate how a + +WrapDrive application can be constructed. + + +As a demo, we try to make it simple and clear for understanding. It is not + +supposed to be used in business scenario. + + +The directory contains the following elements: + +wd.[ch] + A demonstration WrapDrive fundamental library which wraps the basic + operations to the WrapDrive-ed device. + +wd_adapter.[ch] + User driver adaptor for wd.[ch] + +wd_utils.[ch] + Some utitlities function used by WD and its drivers + +drv/* + User drivers. It helps to fulfill the semantic of wd.[ch] for + particular hardware + +test/* + Test applications to use the wrapdrive library + diff --git a/samples/warpdrive/autogen.sh b/samples/warpdrive/autogen.sh new file mode 100755 index 000000000000..58deaf49de2a --- /dev/null +++ b/samples/warpdrive/autogen.sh @@ -0,0 +1,3 @@ +#!/bin/sh -x + +autoreconf -i -f -v diff --git a/samples/warpdrive/cleanup.sh b/samples/warpdrive/cleanup.sh new file mode 100755 index 000000000000..c5f3d21e5dc1 --- /dev/null +++ b/samples/warpdrive/cleanup.sh @@ -0,0 +1,13 @@ +#!/bin/sh + +if [ -r Makefile ]; then + make distclean +fi + +FILES="aclocal.m4 autom4te.cache compile config.guess config.h.in config.log \ + config.status config.sub configure cscope.out depcomp install-sh \ + libsrc/Makefile libsrc/Makefile.in libtool ltmain.sh Makefile \ + ar-lib m4 \ + Makefile.in missing src/Makefile src/Makefile.in test/Makefile.in" + +rm -vRf $FILES diff --git a/samples/warpdrive/configure.ac b/samples/warpdrive/configure.ac new file mode 100644 index 000000000000..177ccbb63a8f --- /dev/null +++ b/samples/warpdrive/configure.ac @@ -0,0 +1,62 @@ +AC_PREREQ([2.69]) +AC_INIT([warpdrive], [0.1], [liguozhu@hisilicon.com]) +AC_CONFIG_SRCDIR([wd.c]) +AM_INIT_AUTOMAKE([1.10 no-define]) + +AC_CONFIG_MACRO_DIR([m4]) +AC_CONFIG_HEADERS([config.h]) + +# Checks for programs. +AC_PROG_CXX +AC_PROG_AWK +AC_PROG_CC +AC_PROG_CPP +AC_PROG_INSTALL +AC_PROG_LN_S +AC_PROG_MAKE_SET +AC_PROG_RANLIB + +AM_PROG_AR +AC_PROG_LIBTOOL +AM_PROG_LIBTOOL +LT_INIT +AM_PROG_CC_C_O + +AC_DEFINE([HAVE_SVA], [0], [enable SVA support]) +AC_ARG_ENABLE([sva], + [ --enable-sva enable to support sva feature], + AC_DEFINE([HAVE_SVA], [1])) +AC_DEFINE([HAVE_DYNDRV], [0], [enable dynamicaly driver load support]) +AC_ARG_ENABLE([dyndrv], + [ --enable-dyndrv enable to support dynamically load driver library], + AC_DEFINE([HAVE_DYNDRV], [1])) +AM_CONDITIONAL([DYNDRV], [test "x$enable_dyndrv" = "xyes"]) +AC_DEFINE([HAVE_NUMA], [0], [enable NUMA support]) +AC_ARG_ENABLE([numa], + [ --enable-numa enable to support numa system], + AC_DEFINE([HAVE_NUMA], [1])) + +# Checks for libraries. + +# Checks for header files. +AC_CHECK_HEADERS([fcntl.h stdint.h stdlib.h string.h sys/ioctl.h sys/time.h unistd.h]) + +# Checks for typedefs, structures, and compiler characteristics. +AC_CHECK_HEADER_STDBOOL +AC_C_INLINE +AC_TYPE_OFF_T +AC_TYPE_SIZE_T +AC_TYPE_UINT16_T +AC_TYPE_UINT32_T +AC_TYPE_UINT64_T +AC_TYPE_UINT8_T + +# Checks for library functions. +#Remove this at first for rpl_malloc error! +#AC_FUNC_MALLOC +AC_FUNC_MMAP +AC_CHECK_FUNCS([memset munmap]) + +AC_CONFIG_FILES([Makefile + test/Makefile]) +AC_OUTPUT diff --git a/samples/warpdrive/drv/hisi_hpre_udrv.c b/samples/warpdrive/drv/hisi_hpre_udrv.c new file mode 100644 index 000000000000..79a865030002 --- /dev/null +++ b/samples/warpdrive/drv/hisi_hpre_udrv.c @@ -0,0 +1,463 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include + +#include "../wd_util.h" +#include "../wd_adapter.h" +#include "../wd_rsa.h" + +#include "../config.h" +#include "./hisi_hpre_udrv.h" + +/* Memory barrier */ +#define mb() {asm volatile("dsb sy" : /* no out */ : /* no in */ : "memory"); } + +#define HPRE_DOORBELL_OFFSET_V2 (0) +#define HPRE_IOMEM_SIZE 4096 + +static int hpre_bn_format(unsigned char *buf, int len) +{ + int i = len - 1, j; + + if (!buf || len <= 0) { + printf("%s fail!\n", __func__); + return -1; + } + while (!buf[i] && i >= 0) + i--; + if (i == len - 1) + return 0; + + for (j = len - 1; j >= 0; j--, i--) { + if (i >= 0) + buf[j] = buf[i]; + else + buf[j] = 0; + } + + return 0; +} + +static int hpre_db_v2(struct hpre_queue_info *q, __u8 cmd, + __u16 index, __u8 priority) +{ + void *base = q->doorbell_base; + __u16 sqn = q->sqn & 0x3ff; + __u64 doorbell = 0; + + doorbell = (__u64)sqn | ((__u64)(cmd & 0xf) << 12); + doorbell |= ((__u64)index | ((__u64)priority << 16)) << 32; + + *((__u64 *)base) = doorbell; + + return 0; +} + +static int hpre_fill_sqe(void *msg, struct wd_queue *q, __u16 i) +{ + struct hpre_queue_info *info = q->priv; + struct hpre_sqe *cc_sqe = &info->cache_sqe; + struct hpre_sqe *sqe = (struct hpre_sqe *)info->sq_base + i; + char *alg = q->capa.alg; + struct wd_rsa_msg *rsa_msg = msg; + void *dma_buf = info->dma_buf; + + if (!strncmp(alg, "rsa", 3)) { + if (rsa_msg->prikey_type == WD_RSA_PRIKEY2) + cc_sqe->alg = HPRE_ALG_NC_CRT; + else if (rsa_msg->prikey_type == WD_RSA_PRIKEY1) + cc_sqe->alg = HPRE_ALG_NC_NCRT; + else + return -1; + cc_sqe->task_len1 = rsa_msg->nbytes / 8 - 1; + if (rsa_msg->op_type == WD_RSA_SIGN) { + /* Since SVA and key SGLs is not supported now, we + * should copy + */ + if (cc_sqe->alg == HPRE_ALG_NC_CRT) { + struct wd_rsa_prikey2 *prikey2 = + (void *)rsa_msg->prikey; + + memcpy(dma_buf, prikey2->dq, + rsa_msg->nbytes / 2); + (void)hpre_bn_format(dma_buf, rsa_msg->nbytes / + 2); + memcpy(dma_buf + rsa_msg->nbytes / 2, + prikey2->dp, rsa_msg->nbytes / 2); + (void)hpre_bn_format(dma_buf + rsa_msg->nbytes / + 2, rsa_msg->nbytes / 2); + memcpy(dma_buf + rsa_msg->nbytes, prikey2->q, + rsa_msg->nbytes / 2); + (void)hpre_bn_format(dma_buf + rsa_msg->nbytes, + rsa_msg->nbytes / 2); + memcpy(dma_buf + 3 * rsa_msg->nbytes / 2, + prikey2->p, rsa_msg->nbytes / 2); + (void)hpre_bn_format(dma_buf + 3 * + rsa_msg->nbytes / 2, rsa_msg->nbytes / 2); + memcpy(dma_buf + 2 * rsa_msg->nbytes, + prikey2->qinv, rsa_msg->nbytes / 2); + (void)hpre_bn_format(dma_buf + 2 * + rsa_msg->nbytes, rsa_msg->nbytes / 2); + } else { + struct wd_rsa_prikey1 *prikey1 = + (void *)rsa_msg->prikey; + + memcpy(dma_buf, prikey1->d, rsa_msg->nbytes); + (void)hpre_bn_format(dma_buf, rsa_msg->nbytes); + memcpy(dma_buf + rsa_msg->nbytes, prikey1->n, + rsa_msg->nbytes); + (void)hpre_bn_format(dma_buf + rsa_msg->nbytes, + rsa_msg->nbytes); + } + } else if (rsa_msg->op_type == WD_RSA_VERIFY) { + struct wd_rsa_pubkey *pubkey = (void *)rsa_msg->pubkey; + + memcpy(dma_buf, pubkey->e, rsa_msg->nbytes); + (void)hpre_bn_format(dma_buf, rsa_msg->nbytes); + memcpy(dma_buf + rsa_msg->nbytes, pubkey->n, + rsa_msg->nbytes); + (void)hpre_bn_format(dma_buf + rsa_msg->nbytes, + rsa_msg->nbytes); + cc_sqe->alg = HPRE_ALG_NC_NCRT; + + } else { + WD_ERR("\nrsa ALG support only sign and verify now!"); + return -1; + } + } else { + WD_ERR("\nalg=%s,rsa algorithm support only now!", alg); + return -1; + } + dma_buf += 2048; + memcpy(dma_buf, (void *)rsa_msg->in, rsa_msg->nbytes); + + /* This need more processing logic. to do more */ + cc_sqe->tag = (__u32)rsa_msg->udata; + cc_sqe->done = 0x1; + cc_sqe->etype = 0x0; + memcpy((void *)sqe, (void *)cc_sqe, sizeof(*cc_sqe)); + + return 0; +} + +void hpre_sqe_dump(struct wd_queue *q, struct hpre_sqe *sqe) +{ + struct hpre_queue_info *info = q->priv; + struct hpre_sqe *sq_base = info->sq_base; + + printf("sqe=%p, index=%ld\n", sqe, ((unsigned long)sqe - + (unsigned long)sq_base)/sizeof(struct hpre_sqe)); + printf("sqe:alg=0x%x\n", sqe->alg); + printf("sqe:etype=0x%x\n", sqe->etype); + printf("sqe:done=0x%x\n", sqe->done); + printf("sqe:task_len1=0x%x\n", sqe->task_len1); + printf("sqe:task_len2=0x%x\n", sqe->task_len2); + printf("sqe:mrttest_num=0x%x\n", sqe->mrttest_num); + printf("sqe:low_key=0x%x\n", sqe->low_key); + printf("sqe:hi_key=0x%x\n", sqe->hi_key); + printf("sqe:low_in=0x%x\n", sqe->low_in); + printf("sqe:hi_in=0x%x\n", sqe->hi_in); + printf("sqe:low_out=0x%x\n", sqe->low_out); + printf("sqe:hi_out=0x%x\n", sqe->hi_out); + printf("sqe:tag=0x%x\n", sqe->tag); +} + +static int hpre_recv_sqe(struct wd_queue *q, struct hpre_sqe *sqe, + struct wd_rsa_msg *recv_msg) +{ + __u32 status = sqe->done; + struct hpre_queue_info *info = q->priv; + void *out; + + if (q->dma_flag & VFIO_SPIMDEV_DMA_PHY) + out = (void *)((((__u64)(sqe->hi_out) << 32) | + (sqe->low_out)) + ((unsigned long long)info->dma_buf - + info->dma_page)); + else + out = (void *)(((__u64)(sqe->hi_out) << 32) | (sqe->low_out)); + if (status != 0x3 || sqe->etype) { + WD_ERR("HPRE do %s fail!done=0x%x, etype=0x%x\n", "rsa", + status, sqe->etype); + return -1; + } + recv_msg->alg = "rsa"; + recv_msg->aflags = 0; + recv_msg->outbytes = (__u16)((sqe->task_len1 + 1) << 3); + memcpy((void *)recv_msg->out, out, recv_msg->outbytes); + + return 1; +} +#define HPRE_DMA_PAGE 4096 + +static int hpre_init_cache_buf(struct wd_queue *q) +{ + void *dma_buf; + struct hpre_queue_info *info = q->priv; + int ret, fd; + __u64 temp; + + if (q->dma_flag & VFIO_SPIMDEV_DMA_PHY) { + info->dma_page = HPRE_DMA_PAGE; + ret = ioctl(q->fd, HPRE_GET_DMA_PAGES, &info->dma_page); + if (ret == -1) { + printf("HPRE_GET_DMA_PAGE ioctl fail!\n"); + return -1; + } + fd = open("/dev/mem", O_RDWR, 0); + if (fd < 0) { + printf("\n%s():Can't open /dev/mem!", __func__); + return -1; + } + dma_buf = mmap((void *)0x0, HPRE_DMA_PAGE, PROT_READ | + PROT_WRITE, + MAP_SHARED, fd, info->dma_page); + if (!dma_buf || dma_buf == MAP_FAILED) { + close(fd); + printf("\nmmap dma buf fail!"); + return -1; + } + memset(dma_buf, 0, HPRE_DMA_PAGE); + temp = (__u64)info->dma_page; + close(fd); + /* For HPRE, we choose to copy user data. */ + } else { + dma_buf = mmap((void *)0x0, HPRE_DMA_PAGE, PROT_READ | + PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + if (!dma_buf) { + printf("\nmmap dma buf fail!"); + return -1; + } + memset(dma_buf, 0, HPRE_DMA_PAGE); + ret = wd_mem_share(q, (const void *)dma_buf, HPRE_DMA_PAGE, 0); + if (ret) { + printf("\nwd_mem_share dma buf fail!"); + return ret; + } + temp = (__u64)dma_buf; + } + info->dma_buf = dma_buf; + info->cache_sqe.low_key = (__u32)(temp & 0xffffffff); + info->cache_sqe.hi_key = (__u32)((temp >> 32) & 0xffffffff); + temp += 2048; + info->cache_sqe.low_in = (__u32)(temp & 0xffffffff); + info->cache_sqe.hi_in = (__u32)((temp >> 32) & 0xffffffff); + temp += 512; + info->cache_sqe.low_out = (__u32)(temp & 0xffffffff); + info->cache_sqe.hi_out = (__u32)((temp >> 32) & 0xffffffff); + + return 0; +} + +static int hpre_uninit_cache_buf(struct wd_queue *q) +{ + struct hpre_queue_info *info = q->priv; + int ret; + + if (q->dma_flag & VFIO_SPIMDEV_DMA_PHY) { + munmap(info->dma_buf, HPRE_DMA_PAGE); + ret = ioctl(q->fd, HPRE_PUT_DMA_PAGES, &info->dma_page); + if (ret == -1) { + printf("HPRE_PUT_DMA_PAGE ioctl fail!\n"); + return -1; + } + } else { + wd_mem_unshare(q, (const void *)info->dma_buf, HPRE_DMA_PAGE); + munmap(info->dma_buf, HPRE_DMA_PAGE); + } + + return 0; +} + +int hpre_set_queue_dio(struct wd_queue *q) +{ + struct hpre_queue_info *info; + void *vaddr; + int ret; + + info = malloc(sizeof(struct hpre_queue_info)); + if (!info) + return -ENOMEM; + memset((void *)info, 0, sizeof(*info)); + q->priv = info; + vaddr = mmap(NULL, + HPRE_SQE_SIZE * HPRE_EQ_DEPTH + HPRE_CQE_SIZE * HPRE_EQ_DEPTH, + PROT_READ | PROT_WRITE, MAP_SHARED, q->fd, 4096); + if (vaddr <= 0) { + ret = (intptr_t)vaddr; + goto err_with_info; + } + info->sq_base = vaddr; + info->cq_base = vaddr + HPRE_SQE_SIZE * HPRE_EQ_DEPTH; + info->sqn = *(__u32 *)vaddr; + info->ver = *((__u32 *)vaddr + 1); + *(__u64 *)vaddr = 0; + vaddr = mmap(NULL, HPRE_IOMEM_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED, q->fd, 0); + if (vaddr <= 0) { + ret = (intptr_t)vaddr; + goto err_with_scq; + } + + /* Only support version 2 */ + if (info->ver == 2) { + info->db = hpre_db_v2; + info->doorbell_base = vaddr + HPRE_DOORBELL_OFFSET_V2; + } else { + ret = -ENODEV; + munmap(vaddr, HPRE_IOMEM_SIZE); + goto err_with_scq; + } + info->sq_tail_index = 0; + info->sq_head_index = 0; + info->cq_head_index = 0; + info->cqc_phase = 1; + info->is_sq_full = 0; + + ret = hpre_init_cache_buf(q); + if (ret) + goto init_cache_fail; + +init_cache_fail: + (void)hpre_uninit_cache_buf(q); +err_with_scq: + munmap(info->sq_base, + HPRE_SQE_SIZE * HPRE_EQ_DEPTH + HPRE_CQE_SIZE * HPRE_EQ_DEPTH); +err_with_info: + free(info); + return ret; +} + +void hpre_unset_queue_dio(struct wd_queue *q) +{ + struct hpre_queue_info *info = q->priv; + int ret; + + ret = hpre_uninit_cache_buf(q); + if (ret) + return; + + if (info->ver == 2) + munmap(info->doorbell_base - HPRE_DOORBELL_OFFSET_V2, + HPRE_IOMEM_SIZE); + + munmap(info->sq_base, (HPRE_CQE_SIZE + HPRE_SQE_SIZE) * HPRE_EQ_DEPTH); + free(info); + q->priv = NULL; +} + +int hpre_add_to_dio_q(struct wd_queue *q, void *req) +{ + struct wd_rsa_msg *msg = req; + struct hpre_queue_info *info = q->priv; + __u16 i; + struct wd_rsa_msg *recv_msg; + + if (info->is_sq_full) + return -EBUSY; + + i = info->sq_tail_index; + recv_msg = info->recv + i * sizeof(struct wd_rsa_msg); + recv_msg->out = msg->out; + recv_msg->udata = msg->udata; + hpre_fill_sqe(msg, q, i); + + /* memory barrier */ + mb() + + if (i == (HPRE_EQ_DEPTH - 1)) + i = 0; + else + i++; + + info->db(info, DOORBELL_CMD_SQ, i, 0); + + info->sq_tail_index = i; + + if (i == info->sq_head_index) + info->is_sq_full = 1; + + return 0; +} + +int hpre_get_from_dio_q(struct wd_queue *q, void **resp) +{ + struct hpre_queue_info *info = q->priv; + __u16 i = info->cq_head_index; + struct hpre_cqe *cq_base = info->cq_base; + struct hpre_sqe *sq_base = info->sq_base; + struct hpre_cqe *cqe = cq_base + i; + struct hpre_sqe *sqe; + struct wd_rsa_msg *recv_msg = info->recv + + i * sizeof(struct wd_rsa_msg); + int ret; + + if (info->cqc_phase == HPRE_CQE_PHASE(cqe)) { + sqe = sq_base + HPRE_CQE_SQ_HEAD_INDEX(cqe); + ret = hpre_recv_sqe(q, sqe, recv_msg); + if (ret < 0) { + hpre_sqe_dump(q, sqe); + return -EIO; + } + if (info->is_sq_full) + info->is_sq_full = 0; + + } else { + return 0; + } + if (i == (HPRE_EQ_DEPTH - 1)) { + info->cqc_phase = !(info->cqc_phase); + i = 0; + } else { + i++; + } + + info->db(info, DOORBELL_CMD_CQ, i, 0); + info->cq_head_index = i; + info->sq_head_index = i; + *resp = recv_msg; + + return ret; +} + +int hpre_get_capa(struct wd_capa *capa) +{ + if (capa && !strncmp(capa->alg, "rsa", 3)) { + capa->latency = 10; + capa->throughput = 10; + + /* capa->priv is to be extended */ + return 0; + } else if (capa && !strncmp(capa->alg, "dh", 2)) { + capa->latency = 10; + capa->throughput = 10; + + /* capa->priv is to be extended */ + return 0; + } + + return -ENODEV; +} + +#if (defined(HAVE_DYNDRV) & HAVE_DYNDRV) +static struct wd_drv_dio_if hpre_dio_tbl = { + .hw_type = "hisi_hpre", + .open = hpre_set_queue_dio, + .close = hpre_unset_queue_dio, + .send = hpre_add_to_dio_q, + .recv = hpre_get_from_dio_q, + .get_capa = hpre_get_capa, +}; + +void __attribute__ ((constructor)) wd_load_hpre_init(void) +{ + int ret; + + ret = wd_drv_dio_tbl_set(&hpre_dio_tbl); + if (ret) { + printf("Load hisilicon hpre user driver fail!\n"); + return; + } +} +#endif diff --git a/samples/warpdrive/drv/hisi_hpre_udrv.h b/samples/warpdrive/drv/hisi_hpre_udrv.h new file mode 100644 index 000000000000..1da5d93d036f --- /dev/null +++ b/samples/warpdrive/drv/hisi_hpre_udrv.h @@ -0,0 +1,108 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef HISI_HPRE_UDRV_H__ +#define HISI_HPRE_UDRV_H__ + +#include +#include "../wd.h" + +#define HPRE_SQE_SIZE 64 +#define HPRE_CQE_SIZE 16 +#define HPRE_EQ_DEPTH 1024 + +/* cqe shift */ +#define HPRE_CQE_PHASE(cq) (((*((__u32 *)(cq) + 3)) >> 16) & 0x1) +#define HPRE_CQE_SQ_NUM(cq) ((*((__u32 *)(cq) + 2)) >> 16) +#define HPRE_CQE_SQ_HEAD_INDEX(cq) ((*((__u32 *)(cq) + 2)) & 0xffff) + +#define HPRE_BAR2_SIZE (4 * 1024 * 1024) + +#define HPRE_DOORBELL_OFFSET 0x1000 + +#define SQE_DONE_FLAG_SHIFT 30 + +enum hpre_alg_type { + HPRE_ALG_NC_NCRT = 0x0, + HPRE_ALG_NC_CRT = 0x1, + HPRE_ALG_KG_STD = 0x2, + HPRE_ALG_KG_CRT = 0x3, + HPRE_ALG_DH_G2 = 0x4, + HPRE_ALG_DH = 0x5, + HPRE_ALG_PRIME = 0x6, + HPRE_ALG_MOD = 0x7, + HPRE_ALG_MOD_INV = 0x8, + HPRE_ALG_MUL = 0x9, + HPRE_ALG_COPRIME = 0xA +}; + + +struct hpre_cqe { + __le32 rsvd0; + __le16 cmd_id; + __le16 rsvd1; + __le16 sq_head; + __le16 sq_num; + __le16 rsvd2; + __le16 w7; /* phase, status */ +}; + +struct hpre_sqe { + __u32 alg : 5; + + /* error type */ + __u32 etype :11; + __u32 resv0 : 14; + + __u32 done : 2; + + __u32 task_len1 : 8; + __u32 task_len2 : 8; + __u32 mrttest_num : 8; + __u32 resv1 : 8; + + __u32 low_key; + __u32 hi_key; + __u32 low_in; + __u32 hi_in; + __u32 low_out; + __u32 hi_out; + + __u32 tag :16; + __u32 resv2 :16; + __u32 rsvd1[7]; +}; + +struct hpre_queue_info { + int ver; + struct hpre_sqe cache_sqe; + void *dma_buf; + unsigned long long dma_page; + void *sq_base; + void *sq_buf; + void *cq_base; + void *doorbell_base; + __u16 sq_tail_index; + __u16 sq_head_index; + __u16 cq_head_index; + __u16 sqn; + int cqc_phase; + void *recv; + int is_sq_full; + int (*db)(struct hpre_queue_info *q, __u8 cmd, __u16 index, + __u8 priority); +}; + +int hpre_set_queue_dio(struct wd_queue *q); +void hpre_unset_queue_dio(struct wd_queue *q); +int hpre_add_to_dio_q(struct wd_queue *q, void *req); +int hpre_get_from_dio_q(struct wd_queue *q, void **resp); +int hpre_get_capa(struct wd_capa *capa); + +#define HPRE_QM_SET_QP _IOR('d', 1, unsigned long long) +#define HPRE_QM_SET_PASID _IOW('d', 2, unsigned long) +#define HPRE_GET_DMA_PAGES _IOW('d', 3, unsigned long long) +#define HPRE_PUT_DMA_PAGES _IOW('d', 4, unsigned long long) + +#define DOORBELL_CMD_SQ 0 +#define DOORBELL_CMD_CQ 1 + +#endif diff --git a/samples/warpdrive/drv/hisi_zip_udrv.c b/samples/warpdrive/drv/hisi_zip_udrv.c new file mode 100644 index 000000000000..a069d67a41a0 --- /dev/null +++ b/samples/warpdrive/drv/hisi_zip_udrv.c @@ -0,0 +1,407 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "hisi_zip_udrv.h" +#include "wd_adapter.h" +#include "../config.h" + +#if __AARCH64EL__ == 1 +/* Define memory barrier on ARM64 */ +#define mb() {asm volatile("dsb sy" : : : "memory"); } +#else +#warning "this file need to be used on AARCH64EL mode" +/* Doing nothing */ +#define mb() +#endif + +#define PAGE_SHIFT 12 + +#define ZIP_SQE_SIZE 128 +#define ZIP_CQE_SIZE 16 +#define ZIP_EQ_DEPTH 1024 + +/* cqe shift */ +#define CQE_PHASE(cq) (((*((__u32 *)(cq) + 3)) >> 16) & 0x1) +#define CQE_SQ_NUM(cq) ((*((__u32 *)(cq) + 2)) >> 16) +#define CQE_SQ_HEAD_INDEX(cq) ((*((__u32 *)(cq) + 2)) & 0xffff) + +#define ZIP_IOMEM_SIZE 4096 + +/* For D06 board */ +#define ZIP_DOORBELL_OFFSET_V1 (0x340) +#define ZIP_DOORBELL_OFFSET_V2 (0) + +#define ZIP_DMA_BUF_SZ (4096 << 10) + +struct cqe { + __le32 rsvd0; + __le16 cmd_id; + __le16 rsvd1; + __le16 sq_head; + __le16 sq_num; + __le16 rsvd2; + __le16 w7; /* phase, status */ +}; + +struct hisi_zip_queue_info { + void *sq_base; + void *cq_base; + void *doorbell_base; + __u16 sq_tail_index; + __u16 sq_head_index; + __u16 cq_head_index; + __u16 sqn; + __u32 ver; + int cqc_phase; + void *req_cache[ZIP_EQ_DEPTH]; + int is_sq_full; + int (*db)(struct hisi_zip_queue_info *q, __u8 cmd, __u16 index, + __u8 priority); + unsigned long long dma_page; + void *dma_buf; +}; + +int hacc_db_v1(struct hisi_zip_queue_info *q, __u8 cmd, __u16 index, + __u8 priority) +{ + void *base = q->doorbell_base; + __u16 sqn = q->sqn; + __u64 doorbell = 0; + + doorbell = (__u64)sqn | ((__u64)cmd << 16); + doorbell |= ((__u64)index | ((__u64)priority << 16)) << 32; + *((__u64 *)base) = doorbell; + + return 0; +} + +int hacc_db_v2(struct hisi_zip_queue_info *q, __u8 cmd, __u16 index, + __u8 priority) +{ + void *base = q->doorbell_base; + __u16 sqn = q->sqn; + __u64 doorbell = 0; + __u16 randata = 0; + + doorbell = (__u64)sqn | ((__u64)cmd << 12) | ((__u64)randata << 16); + doorbell |= ((__u64)index | ((__u64)priority << 16)) << 32; + *((__u64 *)base) = doorbell; + + return 0; +} + +static int hisi_zip_fill_sqe(void *msg, struct wd_queue *q, __u16 i) +{ + struct hisi_zip_queue_info *info = q->priv; + struct hisi_zip_msg *sqe = (struct hisi_zip_msg *)info->sq_base + i; + unsigned long long src_addr = info->dma_page; + unsigned long long dst_addr = src_addr; + void *in; + + memcpy((void *)sqe, msg, sizeof(struct hisi_zip_msg)); + + if (q->dma_flag & VFIO_SPIMDEV_DMA_PHY) { + /* while decompression, we need more space */ + if (sqe->input_date_length > ZIP_DMA_BUF_SZ) + return -EINVAL; + in = (void *)((__u64)sqe->source_addr_l | + ((__u64)sqe->source_addr_h << 32)); + memcpy(info->dma_buf, in, sqe->input_date_length); + sqe->source_addr_l = src_addr & 0xffffffff; + sqe->source_addr_h = src_addr >> 32; + sqe->dest_addr_l = dst_addr & 0xffffffff; + sqe->dest_addr_h = dst_addr >> 32; + } + assert(!info->req_cache[i]); + info->req_cache[i] = msg; + + return 0; +} + +static int hisi_zip_recv_sqe(struct hisi_zip_msg *sqe, struct wd_queue *q, + __u16 i) +{ + __u32 status = sqe->dw3 & 0xff; + __u32 type = sqe->dw9 & 0xff; + struct hisi_zip_queue_info *info = q->priv; + void *out, *dma_out = info->dma_buf; + struct hisi_zip_msg *umsg = info->req_cache[i]; + + if (status != 0 && status != 0x0d) { + fprintf(stderr, "bad status (s=%d, t=%d)\n", status, type); + return -EIO; + } + + assert(umsg); + if (q->dma_flag & VFIO_SPIMDEV_DMA_PHY) { + out = (void *)((__u64)umsg->dest_addr_l | + ((__u64)umsg->dest_addr_h << 32)); + memcpy(out, dma_out, sqe->produced); + umsg->produced = sqe->produced; + umsg->consumed = sqe->consumed; + umsg->tag = sqe->tag; + + /* other area to be filled */ + return 1; + } + memcpy((void *)info->req_cache[i], sqe, sizeof(*umsg)); + + return 1; +} + +static int zip_get_dma_buf(struct wd_queue *q) +{ + struct hisi_zip_queue_info *info = q->priv; + int ret, fd; + void *dma_buf; + + if (!(q->dma_flag & VFIO_SPIMDEV_DMA_PHY)) + return 0; + + info->dma_page = ZIP_DMA_BUF_SZ; + ret = ioctl(q->fd, ZIP_GET_DMA_PAGES, &info->dma_page); + if (ret < 0) { + printf("GET_DMA_PAGE ioctl fail!\n"); + return ret; + } + fd = open("/dev/mem", O_RDWR, 0); + if (fd < 0) { + printf("\n%s():Can't open /dev/mem!", __func__); + ret = fd; + goto put_pages; + } + dma_buf = mmap((void *)0x0, ZIP_DMA_BUF_SZ, PROT_READ | + PROT_WRITE, + MAP_SHARED, fd, info->dma_page); + if (!dma_buf || dma_buf == MAP_FAILED) { + printf("\nmmap dma buf fail!"); + ret = -1; + goto close_mem; + } + memset(dma_buf, 0, ZIP_DMA_BUF_SZ); + info->dma_buf = dma_buf; + close(fd); + + return 0; +close_mem: + close(fd); + +put_pages: + (void)ioctl(q->fd, ZIP_PUT_DMA_PAGES, &info->dma_page); + + return ret; +} + +static void zip_put_dma_buf(struct wd_queue *q) +{ + struct hisi_zip_queue_info *info = q->priv; + + if (!(q->dma_flag & VFIO_SPIMDEV_DMA_PHY)) + return; + + munmap(info->dma_buf, ZIP_DMA_BUF_SZ); + (void)ioctl(q->fd, ZIP_PUT_DMA_PAGES, &info->dma_page); +} + +int hisi_zip_set_queue_dio(struct wd_queue *q) +{ + struct hisi_zip_queue_info *info; + void *vaddr; + int ret; + + info = malloc(sizeof(*info)); + if (!info) + return -1; + memset((void *)info, 0, sizeof(*info)); + q->priv = info; + + vaddr = mmap(NULL, + ZIP_SQE_SIZE * ZIP_EQ_DEPTH + ZIP_CQE_SIZE * ZIP_EQ_DEPTH, + PROT_READ | PROT_WRITE, MAP_SHARED, q->fd, 4096); + if (vaddr <= 0) { + ret = (intptr_t)vaddr; + goto err_with_info; + } + info->sq_base = vaddr; + info->cq_base = vaddr + ZIP_SQE_SIZE * ZIP_EQ_DEPTH; + info->sqn = *(__u32 *)vaddr; + info->ver = *((__u32 *)vaddr + 1); + *(__u64 *)vaddr = 0; + vaddr = mmap(NULL, ZIP_IOMEM_SIZE, + PROT_READ | PROT_WRITE, MAP_SHARED, q->fd, 0); + if (vaddr <= 0) { + ret = (intptr_t)vaddr; + goto err_with_scq; + } + if (info->ver == 1) { + info->db = hacc_db_v1; + info->doorbell_base = vaddr + ZIP_DOORBELL_OFFSET_V1; + } else if (info->ver == 2) { + info->db = hacc_db_v2; + info->doorbell_base = vaddr + ZIP_DOORBELL_OFFSET_V2; + } else { + ret = -ENODEV; + goto unmap_io; + } + info->sq_tail_index = 0; + info->sq_head_index = 0; + info->cq_head_index = 0; + info->cqc_phase = 1; + info->is_sq_full = 0; + + ret = zip_get_dma_buf(q); + if (ret) + goto unmap_io; + + return 0; + +unmap_io: + munmap(vaddr, ZIP_IOMEM_SIZE); +err_with_scq: + munmap(info->sq_base, + ZIP_SQE_SIZE * ZIP_EQ_DEPTH + ZIP_CQE_SIZE * ZIP_EQ_DEPTH); +err_with_info: + free(info); + return ret; +} + +void hisi_zip_unset_queue_dio(struct wd_queue *q) +{ + struct hisi_zip_queue_info *info = q->priv; + + zip_put_dma_buf(q); + + if (info->ver == 1) + munmap(info->doorbell_base - ZIP_DOORBELL_OFFSET_V1, + ZIP_IOMEM_SIZE); + else if (info->ver == 2) + munmap(info->doorbell_base - ZIP_DOORBELL_OFFSET_V2, + ZIP_IOMEM_SIZE); + + munmap(info->cq_base, ZIP_CQE_SIZE * ZIP_EQ_DEPTH); + munmap(info->sq_base, ZIP_SQE_SIZE * ZIP_EQ_DEPTH); + free(info); + q->priv = NULL; +} + +int hisi_zip_add_to_dio_q(struct wd_queue *q, void *req) +{ + struct hisi_zip_queue_info *info = q->priv; + __u16 i; + int ret; + + if (info->is_sq_full) + return -EBUSY; + + i = info->sq_tail_index; + + ret = hisi_zip_fill_sqe(req, q, i); + if (ret) + return ret; + /*memory barrier*/ + mb(); + + if (i == (ZIP_EQ_DEPTH - 1)) + i = 0; + else + i++; + + info->db(info, DOORBELL_CMD_SQ, i, 0); + + info->sq_tail_index = i; + + if (i == info->sq_head_index) + info->is_sq_full = 1; + + return 0; +} + +int hisi_zip_get_from_dio_q(struct wd_queue *q, void **resp) +{ + struct hisi_zip_queue_info *info = q->priv; + __u16 i = info->cq_head_index; + struct cqe *cq_base = info->cq_base; + struct hisi_zip_msg *sq_base = info->sq_base; + struct cqe *cqe = cq_base + i; + struct hisi_zip_msg *sqe; + int ret; + + if (info->cqc_phase == CQE_PHASE(cqe)) { + sqe = sq_base + CQE_SQ_HEAD_INDEX(cqe); + ret = hisi_zip_recv_sqe(sqe, q, i); + if (ret < 0) + return -EIO; + + if (info->is_sq_full) + info->is_sq_full = 0; + } else { + return 0; + } + + *resp = info->req_cache[i]; + info->req_cache[i] = NULL; + + if (i == (ZIP_EQ_DEPTH - 1)) { + info->cqc_phase = !(info->cqc_phase); + i = 0; + } else + i++; + info->db(info, DOORBELL_CMD_CQ, i, 0); + info->cq_head_index = i; + info->sq_head_index = i; + + + return ret; +} + +int hisi_zip_get_capa(struct wd_capa *capa) +{ + if (capa && !strncmp(capa->alg, "zlib", 4)) { + capa->latency = 10; + capa->throughput = 10; + + /* capa->priv is to be extended */ + return 0; + } else if (capa && !strncmp(capa->alg, "gzip", 4)) { + capa->latency = 10; + capa->throughput = 10; + + /* capa->priv is to be extended */ + return 0; + } + + return -ENODEV; +} + +#if (defined(HAVE_DYNDRV) & HAVE_DYNDRV) +static struct wd_drv_dio_if zip_dio_tbl = { + .hw_type = "hisi_zip", + .open = hisi_zip_set_queue_dio, + .close = hisi_zip_unset_queue_dio, + .send = hisi_zip_add_to_dio_q, + .recv = hisi_zip_get_from_dio_q, + .get_capa = hisi_zip_get_capa, +}; + +void __attribute__ ((constructor)) wd_load_hisilicon_zip_init(void) +{ + int ret; + + ret = wd_drv_dio_tbl_set(&zip_dio_tbl); + if (ret) { + printf("Load hisilicon zip user driver fail!\n"); + return; + } +} +#endif diff --git a/samples/warpdrive/drv/hisi_zip_udrv.h b/samples/warpdrive/drv/hisi_zip_udrv.h new file mode 100644 index 000000000000..031e833739df --- /dev/null +++ b/samples/warpdrive/drv/hisi_zip_udrv.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __HISI_ZIP_DRV_H__ +#define __HISI_ZIP_DRV_H__ + +#include +#include "../wd.h" + +/* this is unnecessary big, the hardware should optimize it */ +struct hisi_zip_msg { + __u32 consumed; + __u32 produced; + __u32 comp_date_length; + __u32 dw3; + __u32 input_date_length; + __u32 lba_l; + __u32 lba_h; + __u32 dw7; /* ... */ + __u32 dw8; /* ... */ + __u32 dw9; /* ... */ + __u32 dw10; /* ... */ + __u32 priv_info; + __u32 dw12; /* ... */ + __u32 tag; + __u32 dest_avail_out; + __u32 rsvd0; + __u32 comp_head_addr_l; + __u32 comp_head_addr_h; + __u32 source_addr_l; + __u32 source_addr_h; + __u32 dest_addr_l; + __u32 dest_addr_h; + __u32 stream_ctx_addr_l; + __u32 stream_ctx_addr_h; + __u32 cipher_key1_addr_l; + __u32 cipher_key1_addr_h; + __u32 cipher_key2_addr_l; + __u32 cipher_key2_addr_h; + __u32 rsvd1[4]; +}; + +struct hisi_acc_zip_sqc { + __u16 sqn; +}; + +#define DOORBELL_CMD_SQ 0 +#define DOORBELL_CMD_CQ 1 + +int hisi_zip_set_queue_dio(struct wd_queue *q); +void hisi_zip_unset_queue_dio(struct wd_queue *q); +int hisi_zip_add_to_dio_q(struct wd_queue *q, void *req); +int hisi_zip_get_from_dio_q(struct wd_queue *q, void **resp); +int hisi_zip_get_capa(struct wd_capa *capa); + +#define ZIP_GET_DMA_PAGES _IOW('d', 3, unsigned long long) +#define ZIP_PUT_DMA_PAGES _IOW('d', 4, unsigned long long) +#endif diff --git a/samples/warpdrive/test/Makefile.am b/samples/warpdrive/test/Makefile.am new file mode 100644 index 000000000000..cce414ff4a13 --- /dev/null +++ b/samples/warpdrive/test/Makefile.am @@ -0,0 +1,9 @@ +AM_CFLAGS=-Wall -O0 -fno-strict-aliasing -ldl -lpthread + +bin_PROGRAMS=test_hisi_zip test_hisi_hpre + +test_hisi_zip_SOURCES=test_hisi_zip.c +test_hisi_hpre_SOURCES=test_hisi_hpre.c + +test_hisi_zip_LDADD=../.libs/libwd.so +test_hisi_hpre_LDADD=../.libs/libwd.so ./libcrypto.so diff --git a/samples/warpdrive/test/comp_hw.h b/samples/warpdrive/test/comp_hw.h new file mode 100644 index 000000000000..79328fd0c1a0 --- /dev/null +++ b/samples/warpdrive/test/comp_hw.h @@ -0,0 +1,23 @@ +// SPDX-License-Identifier: GPL-2.0 +/** + * This file is shared bewteen user and kernel space Wrapdrive which is + * including algorithm attibutions that both user and driver are caring for + */ + +#ifndef __VFIO_WDEV_COMP_H +#define __VFIO_WDEV_COMP_H + +/* De-compressing algorithms' parameters */ +struct vfio_wdev_comp_param { + __u32 window_size; + __u32 comp_level; + __u32 mode; + __u32 alg; +}; + +/* WD defines all the De-compressing algorithm names here */ +#define VFIO_WDEV_ZLIB "zlib" +#define VFIO_WDEV_GZIP "gzip" +#define VFIO_WDEV_LZ4 "lz4" + +#endif diff --git a/samples/warpdrive/test/test_hisi_hpre.c b/samples/warpdrive/test/test_hisi_hpre.c new file mode 100644 index 000000000000..78df945869c1 --- /dev/null +++ b/samples/warpdrive/test/test_hisi_hpre.c @@ -0,0 +1,330 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include +#include + +#include "./openssl/bn.h" +#include "./openssl/rsa.h" +#include "../wd.h" +#include "../wd_rsa.h" + +static int dbg_thrd_num; +static int key_bits = 4096; +#define TEST_MAX_THRD 8 +#define TEST_THRDS_NUM dbg_thrd_num +static pthread_t request_release_q_thrds[TEST_MAX_THRD]; + +#define DEBUG_NOIOMMU + +#define ASIZE (16 * 4096) + +#define SYS_ERR_COND(cond, msg) \ +do { \ + if (cond) { \ + perror(msg); \ + exit(EXIT_FAILURE); \ + } \ +} while (0) + +#define OP_NUMBER 4000 +#define RSA_KEY_BITS key_bits +struct test_wd_bn { + unsigned long long *d; + int top; + + /* The next are internal book keeping for bn_expand. */ + int dmax; + int neg; + int flags; +}; + +static inline __u64 test_const_bswap64(__u64 ullTmp) +{ + return ((ullTmp & 0x00000000000000ffULL) << 56) | + ((ullTmp & 0x000000000000ff00ULL) << 40) | + ((ullTmp & 0x0000000000ff0000ULL) << 24) | + ((ullTmp & 0x00000000ff000000ULL) << 8) | + ((ullTmp & 0x000000ff00000000ULL) >> 8) | + ((ullTmp & 0x0000ff0000000000ULL) >> 24) | + ((ullTmp & 0x00ff000000000000ULL) >> 40) | + ((ullTmp & 0xff00000000000000ULL) >> 56); +} + + +void test_rsa_bn_endian_swap(BIGNUM *p) +{ + struct test_wd_bn *b = (struct test_wd_bn *)p; + int i; + + for (i = 0; i < ((struct test_wd_bn *)b)->top; i++) + ((struct test_wd_bn *)b)->d[i] = + test_const_bswap64(((struct test_wd_bn *)b)->d[i]); +} +int test_rsa_key_gen(void *ctx) +{ + BIGNUM *p, *q, *e_value, *n, *e, *d, *dmp1, *dmq1, *iqmp; + int ret, bits; + RSA *test_rsa; + union wd_rsa_prikey *prikey; + struct wd_rsa_pubkey *pubkey; + + bits = wd_rsa_key_bits(ctx); + + test_rsa = RSA_new(); + if (!test_rsa || !bits) { + WD_ERR("\n RSA new fail!"); + return -1; + } + e_value = BN_new(); + if (!e_value) { + RSA_free(test_rsa); + WD_ERR("\n BN new e fail!"); + ret = -1; + return ret; + } + ret = BN_set_word(e_value, 65537); + if (ret != 1) { + WD_ERR("\n BN_set_word fail!"); + ret = -1; + goto gen_fail; + } + + ret = RSA_generate_key_ex(test_rsa, RSA_KEY_BITS, e_value, NULL); + if (ret != 1) { + WD_ERR("\n RSA_generate_key_ex fail!"); + ret = -1; + goto gen_fail; + } + RSA_get0_key((const RSA *)test_rsa, (const BIGNUM **)&n, + (const BIGNUM **)&e, (const BIGNUM **)&d); + RSA_get0_factors((const RSA *)test_rsa, (const BIGNUM **)&p, + (const BIGNUM **)&q); + RSA_get0_crt_params((const RSA *)test_rsa, (const BIGNUM **)&dmp1, + (const BIGNUM **)&dmq1, (const BIGNUM **)&iqmp); + + wd_get_rsa_pubkey(ctx, &pubkey); + wd_get_rsa_prikey(ctx, &prikey); + + if (wd_rsa_is_crt(ctx)) { + BN_bn2bin(dmp1, prikey->pkey2.dp); + BN_bn2bin(dmq1, prikey->pkey2.dq); + BN_bn2bin(p, prikey->pkey2.p); + BN_bn2bin(q, prikey->pkey2.q); + BN_bn2bin(iqmp, prikey->pkey2.qinv); + + } else { + BN_bn2bin(d, prikey->pkey1.d); + BN_bn2bin(n, prikey->pkey1.n); + + } + BN_bn2bin(e, pubkey->e); + BN_bn2bin(n, pubkey->n); + BN_free(e_value); + + return 0; + +gen_fail: + RSA_free(test_rsa); + BN_free(e_value); + + return ret; +} +void *test_q_mng_thread(void *data) +{ + struct wd_queue q; + int ret; + int container; + + container = *(int *)data; + memset(&q, 0, sizeof(q)); + q.capa.alg = "rsa"; + q.capa.throughput = 10; + q.capa.latency = 10; + q.container = container; + while (1) { + ret = wd_request_queue(&q); + if (ret) { + WD_ERR("\nwd_request_queue fail!"); + return NULL; + } + usleep(100); + wd_release_queue(&q); + } + + return NULL; +} + +int main(int argc, char *argv[]) +{ + struct wd_queue q; + struct wd_rsa_msg *msg; + void *a, *src, *dst; + int ret, i, loops; + int output_num; + struct timeval start_tval, end_tval; + float time, speed; + int mode, dir; + unsigned int pkt_len; + void *ctx = NULL; + struct wd_rsa_ctx_setup setup; + struct wd_rsa_op_data opdata; + int is_new_container = 0; + int container = 0; + + if (argv[1]) { + key_bits = strtoul(argv[1], NULL, 10); + if (key_bits != 1024 && key_bits != 2048 && + key_bits != 3072 && key_bits != 4096) + key_bits = 4096; + } else { + key_bits = 4096; + } + + if (argv[2]) + mode = strtoul(argv[2], NULL, 10); + else + mode = 0; + + if (argv[3]) + dir = strtoul(argv[3], NULL, 10); + else + dir = 0; + if (argv[4]) + TEST_THRDS_NUM = strtoul(argv[4], NULL, 10); + else + TEST_THRDS_NUM = 0; + if (argv[5]) + is_new_container = strtoul(argv[5], NULL, 10); + else + is_new_container = 0; + if (TEST_THRDS_NUM > TEST_MAX_THRD) + TEST_THRDS_NUM = TEST_MAX_THRD; + pkt_len = (RSA_KEY_BITS >> 3); + + if (is_new_container) { + container = open("/dev/vfio/vfio", O_RDWR); + if (container < 0) { + WD_ERR("Create VFIO container fail!\n"); + return -ENODEV; + } + } + memset(&q, 0, sizeof(q)); + q.capa.alg = "rsa"; + q.capa.throughput = 10; + q.capa.latency = 10; + + for (i = 0; i < TEST_THRDS_NUM; i++) { + ret = pthread_create(&request_release_q_thrds[i], NULL, + test_q_mng_thread, &container); + if (ret) { + printf("\npthread_create %dth thread fail!", i); + return -1; + } + } + ret = wd_request_queue(&q); + SYS_ERR_COND(ret, "wd_request_queue"); + printf("\npasid=%d, dma_flag=%d", q.pasid, q.dma_flag); + + setup.alg = "rsa"; + setup.key_bits = RSA_KEY_BITS; + setup.is_crt = mode; + setup.cb = NULL; + + ctx = wd_create_rsa_ctx(&q, &setup); + if (!ctx) { + WD_ERR("\ncreate rsa ctx fail!"); + goto release_q; + } + ret = test_rsa_key_gen(ctx); + if (ret) { + wd_del_rsa_ctx(ctx); + goto release_q; + } + + /* Allocate some space and setup a DMA mapping */ + a = mmap((void *)0x0, ASIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + if (!a) { + wd_del_rsa_ctx(ctx); + printf("\nmmap fail!"); + goto release_q; + } + memset(a, 0, ASIZE); + + /* set input rsa sample data */ + for (i = 0; i < ASIZE / 8; i++) + *(__u32 *)(a + i * 4) = i; +#ifndef DEBUG_NOIOMMU + ret = wd_mem_share(&q, a, ASIZE, 0); + SYS_ERR_COND(ret, "wd_mem_share err\n"); + printf("WD dma map VA=IOVA=%p successfully!\n", a); +#endif + src = a; + dst = (char *)a + (ASIZE / 2); + + msg = malloc(sizeof(*msg)); + if (!msg) { + printf("\nalloc msg fail!"); + goto alloc_msg_fail; + } + memset((void *)msg, 0, sizeof(*msg)); + loops = ASIZE / (2 * pkt_len); + gettimeofday(&start_tval, NULL); + for (i = 0; i < OP_NUMBER; i++) { + opdata.in_bytes = pkt_len; + if (dir) + opdata.op_type = WD_RSA_SIGN; + else + opdata.op_type = WD_RSA_VERIFY; + opdata.in = src + (i % loops) * pkt_len; + opdata.out = dst + (i % loops) * pkt_len; + ret = wd_do_rsa(ctx, &opdata); + if (ret) { + free(msg); + printf("\nwd_do_rsa fail!optimes=%d\n", i); + goto alloc_msg_fail; + } + } + output_num = opdata.out_bytes; + if (output_num != pkt_len) { + free(msg); + goto alloc_msg_fail; + } + gettimeofday(&end_tval, NULL); + time = (float)((end_tval.tv_sec-start_tval.tv_sec) * 1000000 + + end_tval.tv_usec - start_tval.tv_usec); + speed = 1 / (time / OP_NUMBER) * 1000; + if (mode) { + printf("\r\nPID(%d):%s CRT mode %dbits sign:", + getpid(), "rsa", RSA_KEY_BITS); + + printf("\r\ntime %0.0fus, pktlen = %d bytes, %0.3f Kops", + time, pkt_len, speed); + } else { + printf("\r\nPID(%d):%s NCRT mode %dbits sign:", + getpid(), "rsa", RSA_KEY_BITS); + printf("\r\ntime %0.0fus, pktlen = %d bytes, %0.3f Kops", + time, pkt_len, speed); + } + free(msg); + +alloc_msg_fail: + wd_del_rsa_ctx(ctx); +#ifndef DEBUG_NOIOMMU + wd_mem_unshare(&q, a, ASIZE); +#endif + munmap(a, ASIZE); +release_q: + wd_release_queue(&q); + for (i = 0; i < TEST_THRDS_NUM; i++) { + ret = pthread_join(request_release_q_thrds[i], NULL); + if (ret) { + printf("\npthread_join %dth thread fail!", i); + return -1; + } + } + return EXIT_SUCCESS; +} diff --git a/samples/warpdrive/test/test_hisi_zip.c b/samples/warpdrive/test/test_hisi_zip.c new file mode 100644 index 000000000000..7bec3a142a10 --- /dev/null +++ b/samples/warpdrive/test/test_hisi_zip.c @@ -0,0 +1,357 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#define __USE_GNU +#include +#include +#include +#include +#include +#include +#include + +#include "../wd.h" +#include "comp_hw.h" +#include "../drv/hisi_zip_udrv.h" + +#define TEST_MORE 1 + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include +# include +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#define PAGE_SHIFT 12 +#define PAGE_SIZE (1 << PAGE_SHIFT) + +#define ASIZE (8*512*4096) /*16MB*/ + +#define SYS_ERR_COND(cond, msg) \ +do { \ + if (cond) { \ + perror(msg); \ + exit(EXIT_FAILURE); \ + } \ +} while (0) + +#define ZLIB 0 +#define GZIP 1 +#define CHUNK 65535 +#define TEST_MAX_THRD 8 +#define THR_2_CPUID(i) (1 + (i) * 13) +#define CPUID_2_THR(i) (((i) - 1) / 13) + +static pthread_t request_release_q_thrds[TEST_MAX_THRD]; +static int thd_cpuid[TEST_MAX_THRD]; +static char exp_test; + +int hizip_deflate(FILE *source, FILE *dest, int type) +{ + __u64 in, out; + struct wd_queue q, *queue; + struct hisi_zip_msg msg, *recv_msg; + void *a, *b; + char *src, *dst; + int ret, total_len; + int output_num, recv_count = 0; + int fd, file_msize; + struct stat s; + struct timeval start_tval, end_tval; + float time; + + /* add zlib compress head and write head + compressed date to a file */ + char zip_head[2] = {0x78, 0x9c}; + +#ifdef TEST_MORE + struct wd_queue q1; + static int q1_tested; +#endif + memset((void *)&msg, 0, sizeof(msg)); + memset(&q, 0, sizeof(q)); + q.container = -1; + if (type == ZLIB) { + q.capa.alg = "zlib"; + msg.dw9 = 2; + } else if (type == GZIP) { + msg.dw9 = 3; + q.capa.alg = "gzip"; + } + q.capa.latency = 10; + q.capa.throughput = 0; + ret = wd_request_queue(&q); + SYS_ERR_COND(ret, "wd_request q fail!"); + fprintf(stderr, "q: node_id=%d, dma_flag=%d\n", q.node_id, q.dma_flag); +#ifdef TEST_MORE + memset(&q1, 0, sizeof(q1)); + q1.capa.alg = q.capa.alg; + q1.capa.latency = 10; + q1.capa.throughput = 0; + q1.container = -1; + ret = wd_request_queue(&q1); + SYS_ERR_COND(ret, "wd_request q1 fail!"); + fprintf(stderr, "q1: node_id=%d, dma_flag=%d\n", + q1.node_id, q1.dma_flag); +#endif + fd = fileno(source); + + fstat(fd, &s); + total_len = s.st_size; + + file_msize = !(total_len % PAGE_SIZE) ? total_len : + (total_len / PAGE_SIZE + 1) * PAGE_SIZE; + /* mmap file and DMA mapping */ + a = mmap((void *)0x0, file_msize, PROT_READ | PROT_WRITE, + MAP_PRIVATE, fd, 0); + if (!a) { + fputs("mmap file fail!\n", stderr); + goto release_q; + } + ret = wd_mem_share(&q, a, file_msize, 0); + if (ret) { + fprintf(stderr, "wd_mem_share dma a buf fail!err=%d\n", -errno); + goto unmap_file; + } + /* Allocate some space and setup a DMA mapping */ + b = mmap((void *)0x0, ASIZE, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, 0, 0); + if (!b) { + fputs("mmap b fail!\n", stderr); + goto unshare_file; + } + memset(b, 0, ASIZE); + ret = wd_mem_share(&q, b, ASIZE, 0); + if (ret) { + fputs("wd_mem_share dma b buf fail!\n", stderr); + goto unmap_mem; + } + src = (char *)a; + dst = (char *)b; + + msg.input_date_length = total_len; + msg.dest_avail_out = 0x800000; + in = (__u64)src; + out = (__u64)dst; + msg.source_addr_l = in & 0xffffffff; + msg.source_addr_h = in >> 32; + msg.dest_addr_l = out & 0xffffffff; + msg.dest_addr_h = out >> 32; + queue = &q; +test_q1: + gettimeofday(&start_tval, NULL); + ret = wd_send(queue, &msg); + if (ret < 0) { + fputs("wd send fail!\n", stderr); + goto unshare_all; + } +recv_again: + ret = wd_recv(queue, (void **)&recv_msg); + if (ret < 0) { + fprintf(stderr, "wd recevice q%d fail!\n", q1_tested); + goto unshare_all; + /* synchronous mode, if get none, then get again */ + } else if (ret == 0) { + recv_count++; + goto recv_again; + } + gettimeofday(&end_tval, NULL); + time = (float)((end_tval.tv_sec-start_tval.tv_sec) * 1000000 + + end_tval.tv_usec - start_tval.tv_usec); + output_num = recv_msg->produced; + if (output_num == 0) { + fprintf(stderr, "q%d compressing fail!\n", q1_tested); + goto unshare_all; + } + fprintf(stderr, "q%d intput %dB output %dB,re_cnt=%d, %0.0fus!\n", + q1_tested, total_len, output_num, recv_count, time); +#ifdef TEST_MORE + if (!q1_tested) { + recv_msg->produced = 0; + queue = &q1; + q1_tested = 1; + recv_count = 0; + goto test_q1; + } +#endif + + fwrite(zip_head, 1, 2, dest); + fwrite((char *)out, 1, output_num, dest); + fclose(dest); + +unshare_all: + wd_mem_unshare(&q, b, ASIZE); +unmap_mem: + munmap(b, ASIZE); +unshare_file: + wd_mem_unshare(&q, a, file_msize); +unmap_file: + munmap(a, file_msize); +release_q: + wd_release_queue(&q); + +#ifdef TEST_MORE + wd_release_queue(&q1); +#endif + return ret; +} + +/* multiple threads on queue management testing */ +void *test_q_mng_thread(void *data) +{ + struct wd_queue q; + int ret; + __u64 cnt = 0; + cpu_set_t mask; + int cpuid = *(int *)data; + + CPU_ZERO(&mask); + CPU_SET(cpuid, &mask); + if (pthread_setaffinity_np(pthread_self(), sizeof(mask), + &mask) < 0) { + perror("pthread_setaffinity_np"); + } + + while (1) { + memset(&q, 0, sizeof(q)); + q.capa.throughput = 0; + q.capa.latency = 10; + q.container = -1; + + if (cnt & 1) + q.capa.alg = "zlib"; + else + q.capa.alg = "gzip"; + ret = wd_request_queue(&q); + if (ret) { + WD_ERR("\ntid(%d) request queue at %lld fail!", + (int)syscall(__NR_gettid), cnt); + return NULL; + } + cnt++; + if (!(cnt & 0x1ff)) + fprintf(stderr, "\ntid(%d) request %lld queues nd=%d", + (int)syscall(__NR_gettid), cnt, q.node_id); + usleep(10); + wd_release_queue(&q); + if (cnt > 0xffff) { + (void)__atomic_add_fetch(&exp_test, 1, + __ATOMIC_ACQUIRE); + break; + } + } + + return NULL; +} + +void *test_devs_max_q(void) +{ + struct wd_queue q; + int ret, cnt = 0; + + while (1) { + memset(&q, 0, sizeof(q)); + q.capa.throughput = 0; + q.capa.latency = 10; + q.container = -1; + + if (cnt & 1) + q.capa.alg = "zlib"; + else + q.capa.alg = "gzip"; + ret = wd_request_queue(&q); + if (ret) { + WD_ERR("\ntid(%d) request queue at %d fail!", + (int)syscall(__NR_gettid), cnt); + return NULL; + } + cnt++; + } + + return NULL; +} + + +int main(int argc, char *argv[]) +{ + int alg_type = 0; + cpu_set_t mask; + int cpuid = 0, i, ret; + + CPU_ZERO(&mask); + if (argv[2]) { + cpuid = strtoul(argv[2], NULL, 10); + if (cpuid <= 0 || cpuid > 128) { + fputs("set cpu no affinity!\n", stderr); + goto no_affinity; + } + CPU_SET(cpuid, &mask); + if (sched_setaffinity(0, sizeof(mask), &mask) < 0) { + perror("sched_setaffinityfail!"); + return -1; + } + } +no_affinity: + /* avoid end-of-line conversions */ + SET_BINARY_MODE(stdin); + SET_BINARY_MODE(stdout); + + if (!argv[1]) { + fputs("<>\n", stderr); + goto EXIT; + } + + if (!strcmp(argv[1], "-z")) + alg_type = ZLIB; + else if (!strcmp(argv[1], "-g")) { + alg_type = GZIP; + } else if (!strcmp(argv[1], "-h")) { + fputs("[version]:1.0.2\n", stderr); + fputs("[usage]: ./test_hisi_zip [type] dest_file\n", + stderr); + fputs(" [type]:\n", stderr); + fputs(" -z = zlib\n", stderr); + fputs(" -g = gzip\n", stderr); + fputs(" -h = usage\n", stderr); + fputs("Example:\n", stderr); + fputs("./test_hisi_zip -z < test.data > out.data\n", stderr); + goto EXIT; + } else { + fputs("Unknow option\n", stderr); + fputs("<>\n", + stderr); + goto EXIT; + } + + hizip_deflate(stdin, stdout, alg_type); + + /* To test the multiple threads feature */ + if (cpuid == 0) { + for (i = 0; i < TEST_MAX_THRD; i++) { + thd_cpuid[i] = THR_2_CPUID(i); + ret = pthread_create(&request_release_q_thrds[i], NULL, + test_q_mng_thread, &thd_cpuid[i]); + if (ret) { + fprintf(stderr, + "\npthread_create %dth thread fail!", i); + return -1; + } + } + for (i = 0; i < TEST_MAX_THRD; i++) { + ret = pthread_join(request_release_q_thrds[i], NULL); + if (ret) { + fprintf(stderr, + "\npthread_join %dth thread fail!", i); + return -1; + } + } + while (exp_test < TEST_MAX_THRD) + usleep(10000); + + test_devs_max_q(); + } + +EXIT: + return EXIT_SUCCESS; +} diff --git a/samples/warpdrive/wd.c b/samples/warpdrive/wd.c new file mode 100644 index 000000000000..6441689e8c42 --- /dev/null +++ b/samples/warpdrive/wd.c @@ -0,0 +1,911 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include "config.h" +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "wd.h" +#include "wd_adapter.h" +#include "wd_util.h" + +#define SYS_CLASS_DIR "/sys/class" +#define WD_LATENCY "latency" +#define WD_THROUGHPUT "throughput" +#define _PAGE_SIZE page_size +#define _PAGE_MASK (~(_PAGE_SIZE - 1)) + +struct wd_algo_info; + +struct wd_vfio_container { + int container; + int ref; +}; + +struct wd_dev_info { + int node_id; + int numa_distance; + int iommu_type; + int dma_flag; + int group_fd; + int mdev_fd; + int ref; + int is_load; + char attr_path[PATH_STR_SIZE]; + char dev_root[PATH_STR_SIZE]; + char name[WD_NAME_SZ]; + char mdev_name[SYS_VAL_SIZE]; + char group_id[SYS_VAL_SIZE]; + char *udrv; + struct wd_algo_info *alg_list; + + TAILQ_ENTRY(wd_dev_info) next; +}; + +struct wd_algo_info { + __u32 type; + __u32 available_instances; + struct wd_capa capa; + char name[WD_NAME_SZ]; + char api[WD_NAME_SZ]; + char algo_path[PATH_STR_SIZE]; + struct wd_dev_info *dinfo; + struct wd_algo_info *next; +}; + +TAILQ_HEAD(wd_dev_list, wd_dev_info); + +static struct wd_dev_list wd_dev_cache_list = + TAILQ_HEAD_INITIALIZER(wd_dev_cache_list); + +static struct wd_lock _wd_pmutex = {0}; + +/* Global VFIO container */ +static struct wd_vfio_container container = {0, 0}; + +static unsigned long page_size; + +#if (defined(HAVE_SVA) & HAVE_SVA) + +/* Currently, PASID is exposed to user space. + * Try to avoid expose it in the next. + */ +static int _wd_bind_process(struct wd_queue *q) +{ + struct bind_data { + struct vfio_iommu_type1_bind bind; + struct vfio_iommu_type1_bind_process data; + } wd_bind; + int ret; + __u32 flags = 0; + + if (q->dma_flag & VFIO_SPIMDEV_DMA_MULTI_PROC_MAP) + flags = VFIO_IOMMU_BIND_PRIV; + else if (q->dma_flag & VFIO_SPIMDEV_DMA_SVM_NO_FAULT) + flags = VFIO_IOMMU_BIND_NOPF; + + wd_bind.bind.flags = VFIO_IOMMU_BIND_PROCESS; + wd_bind.bind.argsz = sizeof(wd_bind); + wd_bind.data.flags = flags; + ret = ioctl(q->container, VFIO_IOMMU_BIND, &wd_bind); + if (ret) + return ret; + q->pasid = wd_bind.data.pasid; + return ret; +} + +static int _wd_unbind_process(struct wd_queue *q) +{ + struct bind_data { + struct vfio_iommu_type1_bind bind; + struct vfio_iommu_type1_bind_process data; + } wd_bind; + __u32 flags = 0; + + if (q->dma_flag & VFIO_SPIMDEV_DMA_MULTI_PROC_MAP) + flags = VFIO_IOMMU_BIND_PRIV; + else if (q->dma_flag & VFIO_SPIMDEV_DMA_SVM_NO_FAULT) + flags = VFIO_IOMMU_BIND_NOPF; + + wd_bind.bind.flags = VFIO_IOMMU_BIND_PROCESS; + wd_bind.data.pasid = q->pasid; + wd_bind.data.flags = flags; + wd_bind.bind.argsz = sizeof(wd_bind); + + return ioctl(q->container, VFIO_IOMMU_UNBIND, &wd_bind); +} +#endif + +static int __alg_param_check(struct wd_algo_info *wa_info, struct wd_capa *capa) +{ + /* We think it is always matching now */ + return 0; +} + +#if (defined(HAVE_DYNDRV) & HAVE_DYNDRV) +static int _drv_load_check(char *drv) +{ + struct wd_dev_info *wd_info; + + TAILQ_FOREACH(wd_info, &wd_dev_cache_list, next) { + if (!strncmp(wd_info->udrv, drv, strlen(wd_info->udrv))) + return 1; + } + + return 0; +} +#endif + +static int __capa_check(struct wd_algo_info *ainfo, struct wd_capa *capa) +{ + struct wd_capa *alg_capa = &ainfo->capa; + + if (strncmp(ainfo->name, capa->alg, strlen(capa->alg))) + return -ENODEV; + + /* Not initiate */ + if (!alg_capa->latency || !alg_capa->throughput) + return -ENODEV; + + /* Latency check */ + if (alg_capa->latency > 0 && alg_capa->latency > capa->latency) + return -ENODEV; + + /* Throughput check */ + if (alg_capa->throughput > 0 && alg_capa->throughput < capa->throughput) + return -ENODEV; + + /* Algorithm paremeters check */ + return __alg_param_check(ainfo, capa); +} + +static void __add_alg(struct wd_algo_info *alg, struct wd_dev_info *wd_info) +{ + struct wd_algo_info *alg_list = wd_info->alg_list; + + wd_info->alg_list = alg; + alg->next = alg_list; +} + +static int _mdev_get(struct wd_dev_info *wd_info) +{ + char mdev_info[SYS_VAL_SIZE]; + int val; + + if (strlen(wd_info->group_id) > 0) + return 0; + memset(mdev_info, '\0', SYS_VAL_SIZE); + val = _get_dir_attr_str(wd_info->attr_path, SPIMDEV_MDEV_GET, + mdev_info); + if (val <= 0) + return val; + mdev_info[val - 1] = '\0'; + memcpy(wd_info->mdev_name, mdev_info, UUID_STR_SZ); + wd_info->mdev_name[UUID_STR_SZ] = '\0'; + strncpy(wd_info->group_id, &mdev_info[UUID_STR_SZ + 1], + SYS_VAL_SIZE); + + return 0; +} + +static int _get_wd_alg_info(struct wd_dev_info *dinfo, struct wd_capa *capa) +{ + char algo_path[PATH_STR_SIZE]; + DIR *drv_alg; + struct dirent *attr_file; + struct wd_algo_info *ainfo = NULL; + char *sect, *d_alg; + int cnt = 0, ret; + + strncpy(algo_path, dinfo->dev_root, PATH_STR_SIZE); + strcat(algo_path, "/device/mdev_supported_types"); + drv_alg = opendir(algo_path); + if (!drv_alg) { + WD_ERR("opendir %s fail!\n", algo_path); + return -ENODEV; + } + while ((attr_file = readdir(drv_alg)) != NULL) { + if (strncmp(attr_file->d_name, ".", 1) == 0) + continue; + if (capa && !strstr(attr_file->d_name, capa->alg)) + continue; + d_alg = attr_file->d_name; + if (!ainfo) { + ainfo = malloc(sizeof(*ainfo)); + if (!ainfo) { + WD_ERR("alloc wa fail!\n"); + closedir(drv_alg); + return -ENOMEM; + } + } + memset(ainfo, 0, sizeof(*ainfo)); + strncpy(ainfo->algo_path, algo_path, PATH_STR_SIZE); + strcat(ainfo->algo_path, "/"); + strcat(ainfo->algo_path, d_alg); + sect = strstr(d_alg, "-"); + memcpy(ainfo->api, d_alg, sect - d_alg); + strcpy(ainfo->name, sect + 1); + ainfo->dinfo = dinfo; +#if (defined(HAVE_DYNDRV) & HAVE_DYNDRV) + if (!dinfo->is_load && !_drv_load_check(ainfo->api)) { + ret = _wd_dyn_load(ainfo->api); + if (ret) { + WD_ERR("WD load %s fail!\n", ainfo->api); + goto no_alg_exit; + } + dinfo->is_load = 1; + } +#endif + ainfo->capa.alg = capa ? capa->alg : NULL; + ret = _wd_get_capa(ainfo->api, &ainfo->capa); + if (ret) { + WD_ERR("WD get capa of %s fail!\n", ainfo->api); + goto no_alg_exit; + } + if (capa && __capa_check(ainfo, capa) < 0) + goto no_alg_exit; + ainfo->available_instances = + _get_dir_attr_int(ainfo->algo_path, "available_instances"); + if (ainfo->available_instances < 0) + goto no_alg_exit; + ainfo->type = + _get_dir_attr_int(ainfo->algo_path, "type"); + __add_alg(ainfo, dinfo); + dinfo->udrv = ainfo->api; + if (capa) { + closedir(drv_alg); + return _mdev_get(dinfo); + } + cnt++; + ainfo = NULL; + } + +no_alg_exit: + if (capa) { + free_obj(ainfo); + closedir(drv_alg); + return -ENODEV; + } + + closedir(drv_alg); + return cnt; +} + +static int _get_dev_node_id(struct wd_dev_info *wd_info) +{ + int val; + + val = _get_dir_attr_int(wd_info->attr_path, SPIMDEV_NODE_ID); + if (val >= 0) + wd_info->node_id = val; +#if (defined(HAVE_NUMA) & HAVE_NUMA) + if (val < 0) { + WD_ERR("Please config NUMA in kernel!"); + return -EINVAL; + } +#endif + return 0; +} + +#if (defined(HAVE_NUMA) & HAVE_NUMA) +static int _get_dev_numa_distance(struct wd_dev_info *wd_info) +{ + return _get_dir_attr_int(wd_info->attr_path, + SPIMDEV_NUMA_DISTANCE); +} +#endif + +static int _get_dev_iommu_type(struct wd_dev_info *wd_info) +{ + int val; + + val = _get_dir_attr_int(wd_info->attr_path, SPIMDEV_IOMMU_TYPE); + if (val >= 0) + wd_info->iommu_type = val; + else + return val; + + return 0; +} + +static int _get_dev_dma_flag(struct wd_dev_info *wd_info) +{ + int val; + + val = _get_dir_attr_int(wd_info->attr_path, SPIMDEV_DMA_FLAG); + if (val >= 0) + wd_info->dma_flag = val; + else + return val; + + return 0; +} + +static int _get_wd_dev_info(struct wd_dev_info *dinfo) +{ + char *attr_path = dinfo->attr_path; + int ret; + + ret = snprintf(attr_path, PATH_STR_SIZE, "%s/%s", dinfo->dev_root, + "device/"VFIO_SPIMDEV_PDEV_ATTRS_GRP); + if (ret < 0) + return ret; + ret = _get_dev_dma_flag(dinfo); + if (ret) + return ret; + ret = _get_dev_iommu_type(dinfo); + if (ret) + return ret; + ret = _get_dev_node_id(dinfo); + if (ret) + return ret; +#if (defined(HAVE_NUMA) & HAVE_NUMA) + ret = _get_dev_numa_distance(dinfo); + if (ret < 0) + return ret; + dinfo->numa_distance = ret; +#endif + return 0; +} + +static inline struct wd_dev_info *_get_cache_dev(char *name) +{ + struct wd_dev_info *dinfo; + + TAILQ_FOREACH(dinfo, &wd_dev_cache_list, next) { + if (strncmp(dinfo->name, name, strlen(name))) + continue; + return dinfo; + } + + return NULL; +} + +static int _find_available_res(struct wd_capa *capa) +{ + DIR *wd_cls; + struct dirent *device; + struct wd_dev_info *dinfo, *wdev, *exist_mdev; + int cnt = 0; + struct wd_algo_info *alg; + + TAILQ_FOREACH(dinfo, &wd_dev_cache_list, next) { + alg = dinfo->alg_list; + while (alg && capa) { + if (__capa_check(alg, capa)) { + alg = alg->next; + continue; + } + + return 1; + } + } + dinfo = NULL; + wd_cls = opendir(SYS_CLASS_DIR"/"VFIO_SPIMDEV_CLASS_NAME); + if (!wd_cls) { + WD_ERR("WD framework is not enabled on this system!\n"); + return -ENODEV; + } + while ((device = readdir(wd_cls)) != NULL) { + if (strncmp(device->d_name, ".", 1) == 0 || + strncmp(device->d_name, "..", 2) == 0) + continue; + exist_mdev = _get_cache_dev(device->d_name); + if (exist_mdev) { + if (_get_wd_alg_info(exist_mdev, capa) < 0) { + continue; + } else { + cnt++; + break; + } + } + dinfo = malloc(sizeof(*dinfo)); + if (!dinfo) { + WD_ERR("alloc wdev fail!\n"); + closedir(wd_cls); + return -ENOMEM; + } + memset(dinfo, 0, sizeof(*dinfo)); + (void)strncpy(dinfo->dev_root, + SYS_CLASS_DIR"/"VFIO_SPIMDEV_CLASS_NAME"/", PATH_STR_SIZE); + (void)strcat(dinfo->dev_root, device->d_name); + + /* To be updated. Algorithm name should be checked at first */ + if (_get_wd_dev_info(dinfo) < 0) + continue; + + strncpy(dinfo->name, device->d_name, WD_NAME_SZ); + if (_get_wd_alg_info(dinfo, capa) < 0) + continue; + cnt++; + if (TAILQ_EMPTY(&wd_dev_cache_list)) { + TAILQ_INSERT_TAIL(&wd_dev_cache_list, dinfo, next); + dinfo = NULL; + } else { + TAILQ_FOREACH(wdev, &wd_dev_cache_list, next) { + if (dinfo->numa_distance > + wdev->numa_distance && + TAILQ_NEXT(wdev, next)) { + continue; + } else if (dinfo->numa_distance <= + wdev->numa_distance) { + TAILQ_INSERT_BEFORE(wdev, dinfo, next); + dinfo = NULL; + break; + } + TAILQ_INSERT_AFTER(&wd_dev_cache_list, + wdev, dinfo, next); + dinfo = NULL; + break; + } + } + } + closedir(wd_cls); + if (dinfo) + free(dinfo); + return cnt; +} + +int wd_dump_all_algos(void) +{ + int ret; + struct wd_dev_info *wd_info; + struct wd_algo_info *alg; + int dev_num = 0; + + wd_spinlock(&_wd_pmutex); + ret = _find_available_res(NULL); + if (ret <= 0) { + wd_unspinlock(&_wd_pmutex); + WD_ERR("No device!\n"); + return ret; + } + TAILQ_FOREACH(wd_info, &wd_dev_cache_list, next) { + alg = wd_info->alg_list; + printf("Device(%s): node_id=%d, priority=%d, iommu_type=%d\n", + wd_info->name, wd_info->node_id, + wd_info->numa_distance, wd_info->iommu_type); + while (alg) { + printf(" Alg(%s): flags=%d, available_instances=%d\n", + alg->name, alg->type, alg->available_instances); + alg = alg->next; + } + dev_num++; + } + wd_unspinlock(&_wd_pmutex); + + return dev_num; +} + +int _get_mdev_group(struct wd_queue *q, struct wd_dev_info *dinfo) +{ + sprintf(q->vfio_group_path, "/dev/vfio/%s", dinfo->group_id); + + /* open group */ + q->group = open(q->vfio_group_path, O_RDWR); + if (q->group < 0) { + if (errno == EBUSY) + return -EBUSY; + WD_ERR("open vfio group fail(%s), ret=%d\n", + q->vfio_group_path, q->group); + return q->group; + } + dinfo->group_fd = q->group; + + return 0; +} + +static void _put_algo_mdev(struct wd_queue *q) +{ + struct wd_algo_info *ainfo = q->alg_info; + struct wd_dev_info *dinfo = ainfo->dinfo; + + (void)__atomic_sub_fetch(&dinfo->ref, 1, __ATOMIC_ACQUIRE); +} + +static int _get_algo_mdev(struct wd_queue *q) +{ + int ret; + struct wd_algo_info *ainfo; + struct wd_dev_info *dinfo; + + /* Create by the order of priority of device */ + TAILQ_FOREACH(dinfo, &wd_dev_cache_list, next) { + ainfo = dinfo->alg_list; +check_next_alg: + if (!ainfo) + continue; + + if (__capa_check(ainfo, &q->capa)) { + ainfo = ainfo->next; + goto check_next_alg; + } +#if (defined(HAVE_NUMA) & HAVE_NUMA) + ret = _get_dev_numa_distance(dinfo); + if (ret < 0) + return ret; + if (ret > q->numa_dis) { + if (!TAILQ_NEXT(dinfo, next)) { + q->numa_dis++; + return _get_algo_mdev(q); + } + continue; + } +#endif + q->iommu_type = ainfo->dinfo->iommu_type; + if (dinfo->group_fd > 0) { + q->mdev = dinfo->mdev_fd; + q->group = dinfo->group_fd; + q->is_new_group = 0; + q->fd = ioctl(q->mdev, + VFIO_SPIMDEV_CMD_GET_Q, + (unsigned long)q->type); + if (q->fd < 0) + continue; + } else { + ret = _get_mdev_group(q, dinfo); + if (ret) { + WD_ERR("fail to open group: /dev/vfio/%s!\n", + dinfo->group_id); + return ret; + } + q->is_new_group = 1; + } + strncpy(q->mdev_name, dinfo->mdev_name, WD_NAME_SZ); + strncpy(q->hw_type, ainfo->api, WD_NAME_SZ); + q->alg_info = ainfo; + q->node_id = dinfo->node_id; + q->type = ainfo->type; + (void)__atomic_add_fetch(&dinfo->ref, 1, __ATOMIC_ACQUIRE); + + return 0; + } + return -ENODEV; +} + +static int _get_vfio_facility(struct wd_queue *q) +{ + struct vfio_group_status group_status = { + .argsz = sizeof(group_status) }; + struct wd_dev_info *dinfo; + int ret = 0; + int iommu_ext; + + dinfo = ((struct wd_algo_info *)q->alg_info)->dinfo; + + /* container from outside */ + if (q->container > 0) { + container.container = q->container; + q->is_ext_container = 1; + page_size = (unsigned long)getpagesize(); + } + + if (q->container <= 0) { + /* I think we need get an existing container */ + q->container = container.container; + q->is_ext_container = 1; + } + + if (q->container <= 0 && q->is_new_group) { + + /* Create a new vfio container */ + q->container = open("/dev/vfio/vfio", O_RDWR); + if (q->container < 0) { + WD_ERR("Create VFIO container fail!\n"); + return -ENODEV; + } + q->is_ext_container = 0; + container.container = q->container; + page_size = (unsigned long)getpagesize(); + + /* while using old group, cannot create new container, because + * the old group has already been added into another container. + */ + } else if (q->container <= 0 && !q->is_new_group) { + WD_ERR("%s():group %d already added into a container!\n", + __func__, q->group); + ret = -EINVAL; + return ret; + } + + /* Check the exist vfio container */ + if (ioctl(q->container, VFIO_GET_API_VERSION) != VFIO_API_VERSION) { + WD_ERR("VFIO version check fail!\n"); + ret = -EINVAL; + return ret; + } + + /* Support the IOMMU driver we want. */ + iommu_ext = dinfo->iommu_type; + if (ioctl(q->container, VFIO_CHECK_EXTENSION, iommu_ext) < 0) { + WD_ERR("VFIO iommu check fail!\n"); + ret = -EINVAL; + return ret; + } + + if (!q->is_new_group) + goto next_operation; + + ioctl(q->group, VFIO_GROUP_GET_STATUS, &group_status); + if (!(group_status.flags & VFIO_GROUP_FLAGS_VIABLE)) { + WD_ERR("VFIO group is not viable\n"); + ret = -ENODEV; + return ret; + } + + if ((ioctl(q->group, VFIO_GROUP_SET_CONTAINER, &q->container))) { + WD_ERR("VFIO group fail on VFIO_GROUP_SET_CONTAINER\n"); + ret = -ENODEV; + return ret; + } + + if (!q->is_ext_container && + ioctl(q->container, VFIO_SET_IOMMU, iommu_ext)) { + WD_ERR("VFIO fail on VFIO_SET_IOMMU(%d)\n", iommu_ext); + ret = -ENODEV; + return ret; + } + + q->mdev = ioctl(q->group, VFIO_GROUP_GET_DEVICE_FD, q->mdev_name); + if (q->mdev < 0) { + WD_ERR("VFIO GET DEVICE %s FD fail!\n", q->mdev_name); + ret = q->mdev; + return ret; + } + dinfo->mdev_fd = q->mdev; + +next_operation: + (void)__atomic_add_fetch(&container.ref, 1, __ATOMIC_ACQUIRE); + q->dma_flag = dinfo->dma_flag; +#if (defined(HAVE_SVA) & HAVE_SVA) + if (!(q->dma_flag & (VFIO_SPIMDEV_DMA_PHY | + VFIO_SPIMDEV_DMA_SINGLE_PROC_MAP))) { + ret = _wd_bind_process(q); + if (ret) { + close(q->mdev); + WD_ERR("VFIO fails to bind process!\n"); + return ret; + + } + } +#endif + + return 0; +} + +static void _put_vfio_facility(struct wd_queue *q) +{ + struct wd_algo_info *ainfo = q->alg_info; + struct wd_dev_info *dinfo = ainfo->dinfo; + +#if (defined(HAVE_SVA) & HAVE_SVA) + if (!(q->dma_flag & (VFIO_SPIMDEV_DMA_PHY | + VFIO_SPIMDEV_DMA_SINGLE_PROC_MAP))) { + if (q->pasid <= 0) { + WD_ERR("Wd queue pasid ! pasid=%d\n", q->pasid); + return; + } + if (_wd_unbind_process(q)) { + WD_ERR("VFIO fails to unbind process!\n"); + return; + } + } +#endif + if (!__atomic_load_n(&dinfo->ref, __ATOMIC_ACQUIRE)) { + if (q->mdev > 0) { + dinfo->mdev_fd = 0; + close(q->mdev); + } + if (q->group > 0) { + dinfo->group_fd = 0; + close(q->group); + } + } + if (q->container > 0 && + !__atomic_sub_fetch(&container.ref, 1, __ATOMIC_ACQUIRE)) { + close(q->container); + container.container = 0; + } +} + +int _get_queue(struct wd_queue *q) +{ + if (q->fd > 0) + return 0; + q->fd = ioctl(q->mdev, + VFIO_SPIMDEV_CMD_GET_Q, + (unsigned long)q->type); + if (q->fd < 0) + return -ENODEV; + + return 0; +} + +static void _put_queue(struct wd_queue *q) +{ + close(q->fd); +} + +int wd_request_queue(struct wd_queue *q) +{ + int ret = 0; + + wd_spinlock(&_wd_pmutex); + ret = _find_available_res(&q->capa); + if (ret <= 0) { + wd_unspinlock(&_wd_pmutex); + WD_ERR("Fail to find available algorithms!\n"); + return -ENODEV; + } + ret = _get_algo_mdev(q); + if (ret) { + wd_unspinlock(&_wd_pmutex); + WD_ERR("Fail to get mdev!\n"); + return -ENODEV; + } + ret = _get_vfio_facility(q); + if (ret) { + WD_ERR("Fail to get VFIO facility!\n"); + goto out_with_mdev; + } + ret = _get_queue(q); + if (ret) { + WD_ERR("Fail to get queue!\n"); + goto out_with_mdev; + } + wd_unspinlock(&_wd_pmutex); + ret = drv_open(q); + if (ret) { + WD_ERR("Driver queue init fail!\n"); + wd_spinlock(&_wd_pmutex); + goto out_with_queue; + } + return ret; + +out_with_queue: + _put_queue(q); +out_with_mdev: + _put_algo_mdev(q); + _put_vfio_facility(q); + wd_unspinlock(&_wd_pmutex); + + return ret; +} + +void wd_release_queue(struct wd_queue *q) +{ + drv_close(q); + wd_spinlock(&_wd_pmutex); + _put_queue(q); + _put_algo_mdev(q); + _put_vfio_facility(q); + wd_unspinlock(&_wd_pmutex); +} + +int wd_send(struct wd_queue *q, void *req) +{ + return drv_send(q, req); +} + +int wd_recv(struct wd_queue *q, void **resp) +{ + return drv_recv(q, resp); +} + +static int wd_flush_and_wait(struct wd_queue *q, __u16 ms) +{ + struct pollfd fds[1]; + + wd_flush(q); + fds[0].fd = q->fd; + fds[0].events = POLLIN; + return poll(fds, 1, ms); +} + +int wd_recv_sync(struct wd_queue *q, void **resp, __u16 ms) +{ + int ret; + + while (1) { + ret = wd_recv(q, resp); + if (ret == -EBUSY) { + ret = wd_flush_and_wait(q, ms); + if (ret) + return ret; + } else + return ret; + } +} + +void wd_flush(struct wd_queue *q) +{ + drv_flush(q); +} + +static int _wd_mem_share_type1(struct wd_queue *q, const void *addr, + size_t size, int flags) +{ + struct vfio_iommu_type1_dma_map dma_map; + + if (q->dma_flag & VFIO_SPIMDEV_DMA_PHY) + return 0; + if (q->dma_flag & VFIO_SPIMDEV_DMA_SVM_NO_FAULT) + return mlock(addr, size); + +#if (defined(HAVE_SVA) & HAVE_SVA) + else if ((q->dma_flag & VFIO_SPIMDEV_DMA_MULTI_PROC_MAP) && + (q->pasid > 0)) + dma_map.pasid = q->pasid; +#endif + else if ((q->dma_flag & VFIO_SPIMDEV_DMA_SINGLE_PROC_MAP)) + ; + else + return -1; + + size = ((size - 1) & _PAGE_MASK) + _PAGE_SIZE; + dma_map.vaddr = (__u64)addr & _PAGE_MASK; + dma_map.size = size; + dma_map.iova = dma_map.vaddr; + dma_map.flags = + VFIO_DMA_MAP_FLAG_READ | VFIO_DMA_MAP_FLAG_WRITE | flags; + dma_map.argsz = sizeof(dma_map); + + return ioctl(q->container, VFIO_IOMMU_MAP_DMA, &dma_map); +} + +static void _wd_mem_unshare_type1(struct wd_queue *q, const void *addr, + size_t size) +{ + struct vfio_iommu_type1_dma_unmap dma_unmap; + + if (q->dma_flag & VFIO_SPIMDEV_DMA_PHY) + return; + if (q->dma_flag & VFIO_SPIMDEV_DMA_SVM_NO_FAULT) { + (void)munlock(addr, size); + return; + } + +#if (defined(HAVE_SVA) & HAVE_SVA) + else if ((q->dma_flag & VFIO_SPIMDEV_DMA_MULTI_PROC_MAP) && + (q->pasid > 0)) { + dma_unmap.pasid = q->pasid; + } +#endif + else if ((q->dma_flag & VFIO_SPIMDEV_DMA_SINGLE_PROC_MAP)) { + ; + } else { + WD_ERR("%s: dma flag error!\n", __func__); + return; + } + size = ((size - 1) & _PAGE_MASK) + _PAGE_SIZE; + dma_unmap.iova = (__u64)addr & _PAGE_MASK; + dma_unmap.flags = 0; + dma_unmap.size = size; + dma_unmap.argsz = sizeof(dma_unmap); + ioctl(q->container, VFIO_IOMMU_UNMAP_DMA, &dma_unmap); +} + +int wd_mem_share(struct wd_queue *q, const void *addr, size_t size, int flags) +{ + if (drv_can_do_mem_share(q)) + return drv_share(q, addr, size, flags); + else + return _wd_mem_share_type1(q, addr, size, flags); +} + +void wd_mem_unshare(struct wd_queue *q, const void *addr, size_t size) +{ + if (drv_can_do_mem_share(q)) + drv_unshare(q, addr, size); + else + _wd_mem_unshare_type1(q, addr, size); +} + diff --git a/samples/warpdrive/wd.h b/samples/warpdrive/wd.h new file mode 100644 index 000000000000..e7d9f66c73fc --- /dev/null +++ b/samples/warpdrive/wd.h @@ -0,0 +1,92 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef __WD_H +#define __WD_H +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../include/uapi/linux/vfio.h" +#include "../../include/uapi/linux/vfio_spimdev.h" + +#define UUID_STR_SZ 36 +#define SYS_VAL_SIZE 64 +#define PATH_STR_SIZE 256 +#define WD_NAME_SZ 64 +#define WD_MAX_MEMLIST_SZ 128 + + +#ifndef dma_addr_t +#define dma_addr_t __u64 +#endif + +#ifndef true +#define true 1 +#endif + +#ifndef false +#define false 0 +#endif + +#ifndef WD_ERR +#define WD_ERR(format, args...) fprintf(stderr, format, ##args) +#endif +#define WD_CAPA_PRIV_DATA_SIZE 64 + +/* Queue Capabilities header */ +struct wd_capa { + char *alg; + __u32 throughput; + __u32 latency; + __u32 flags; + __u32 ver;/* Used for checking WD version */ + __u8 priv[WD_CAPA_PRIV_DATA_SIZE]; +}; + +struct wd_queue { + int hw_type_id; + int dma_flag; + struct wd_capa capa; + void *priv; /* private data used by the drv layer */ + void *alg_info; + void *ctx; + int container; + int group; + int mdev; + int fd; + int pasid; + int iommu_type; + int node_id; + int numa_dis; + __u16 is_new_group; + __u16 is_ext_container; + int type; + char mdev_name[WD_NAME_SZ]; + char hw_type[WD_NAME_SZ]; + char vfio_group_path[PATH_STR_SIZE]; +}; + +extern int wd_request_queue(struct wd_queue *q); +extern void wd_release_queue(struct wd_queue *q); +extern int wd_send(struct wd_queue *q, void *req); +extern int wd_recv(struct wd_queue *q, void **resp); +extern void wd_flush(struct wd_queue *q); +extern int wd_recv_sync(struct wd_queue *q, void **resp, __u16 ms); +extern int wd_mem_share(struct wd_queue *q, const void *addr, + size_t size, int flags); +extern void wd_mem_unshare(struct wd_queue *q, const void *addr, size_t size); + +/* for debug only */ +extern int wd_dump_all_algos(void); + +/* this is only for drv used */ +extern int wd_set_queue_attr(struct wd_queue *q, const char *name, + char *value); +extern int __iommu_type(struct wd_queue *q); + +#endif diff --git a/samples/warpdrive/wd_adapter.c b/samples/warpdrive/wd_adapter.c new file mode 100644 index 000000000000..c219f09ff809 --- /dev/null +++ b/samples/warpdrive/wd_adapter.c @@ -0,0 +1,171 @@ +// SPDX-License-Identifier: GPL-2.0 +#include "config.h" +#include +#include +#include +#include + +#include "wd_adapter.h" + +#if (defined(HAVE_DYNDRV) & HAVE_DYNDRV) +#define WD_MAX_DRV_NUM 16 + +static struct wd_drv_dio_if *hw_dio_tbl[WD_MAX_DRV_NUM]; + +static char *available_lib_dirs[] = { + "/usr/lib/", + "/usr/lib64/", + "/lib64/", + "/lib/", + "./", + NULL, +}; + +int wd_drv_dio_tbl_set(struct wd_drv_dio_if *tbl) +{ + int i; + + if (!tbl || !tbl->hw_type || !tbl->send || + !tbl->recv || !tbl->open || !tbl->close) + return -EINVAL; + + for (i = 0; i < WD_MAX_DRV_NUM; i++) { + if (hw_dio_tbl[i]) + continue; + if (tbl == hw_dio_tbl[i]) + return 0; + hw_dio_tbl[i] = tbl; + + return 0; + } + + return -ENODEV; +} + +/* Load the driver library */ +int _wd_dyn_load(char *drv) +{ + char fpath[PATH_STR_SIZE]; + char lib[WD_NAME_SZ]; + void *handle; + char *dir; + int i, ret; + + if (!drv) + return -EINVAL; + memset((void *)lib, 0, WD_NAME_SZ); + (void)strcat(lib, "lib"); + (void)strcat(lib, drv); + (void)strcat(lib, ".so"); + + for (i = 0; i < sizeof(available_lib_dirs); i++) { + memset((void *)fpath, 0, sizeof(fpath)); + dir = available_lib_dirs[i]; + if (dir) { + ret = snprintf(fpath, PATH_STR_SIZE, "%s/%s", dir, lib); + if (ret < 0) + return ret; + } else { + break; + } + handle = dlopen(fpath, RTLD_LAZY); + if (!handle) + continue; + return 0; + } + + return -ENODEV; +} +#else +#include "./drv/hisi_zip_udrv.h" +#include "./drv/hisi_hpre_udrv.h" + +#define WD_MAX_DRV_NUM (sizeof(hw_dio_tbl) / sizeof(struct wd_drv_dio_if *)) +static struct wd_drv_dio_if hisi_zip_dio_tbl = { + .hw_type = "hisi_zip", + .open = hisi_zip_set_queue_dio, + .close = hisi_zip_unset_queue_dio, + .send = hisi_zip_add_to_dio_q, + .recv = hisi_zip_get_from_dio_q, + .get_capa = hisi_zip_get_capa, +}; + +static struct wd_drv_dio_if hisi_hpre_dio_tbl = { + .hw_type = "hisi_hpre", + .open = hpre_set_queue_dio, + .close = hpre_unset_queue_dio, + .send = hpre_add_to_dio_q, + .recv = hpre_get_from_dio_q, + .get_capa = hpre_get_capa, +}; + +static struct wd_drv_dio_if *hw_dio_tbl[] = { + &hisi_zip_dio_tbl, + &hisi_hpre_dio_tbl, + /* Add other drivers direct IO operations here */ +}; +#endif + +int _wd_get_capa(char *drv, struct wd_capa *capa) +{ + int i; + + for (i = 0; i < WD_MAX_DRV_NUM; i++) { + if (!strcmp(drv, hw_dio_tbl[i]->hw_type)) + return hw_dio_tbl[i]->get_capa(capa); + } + + return -ENODEV; +} + +int drv_open(struct wd_queue *q) +{ + int i; + + for (i = 0; i < WD_MAX_DRV_NUM; i++) { + if (!strcmp(q->hw_type, hw_dio_tbl[i]->hw_type)) { + q->hw_type_id = i; + return hw_dio_tbl[q->hw_type_id]->open(q); + } + } + + WD_ERR("No available driver to use!\n"); + + return -ENODEV; +} + +void drv_close(struct wd_queue *q) +{ + hw_dio_tbl[q->hw_type_id]->close(q); +} + +int drv_send(struct wd_queue *q, void *req) +{ + return hw_dio_tbl[q->hw_type_id]->send(q, req); +} + +int drv_recv(struct wd_queue *q, void **req) +{ + return hw_dio_tbl[q->hw_type_id]->recv(q, req); +} + +int drv_share(struct wd_queue *q, const void *addr, size_t size, int flags) +{ + return hw_dio_tbl[q->hw_type_id]->share(q, addr, size, flags); +} + +void drv_unshare(struct wd_queue *q, const void *addr, size_t size) +{ + hw_dio_tbl[q->hw_type_id]->unshare(q, addr, size); +} + +int drv_can_do_mem_share(struct wd_queue *q) +{ + return hw_dio_tbl[q->hw_type_id]->share != NULL; +} + +void drv_flush(struct wd_queue *q) +{ + if (hw_dio_tbl[q->hw_type_id]->flush) + hw_dio_tbl[q->hw_type_id]->flush(q); +} diff --git a/samples/warpdrive/wd_adapter.h b/samples/warpdrive/wd_adapter.h new file mode 100644 index 000000000000..a354666ae028 --- /dev/null +++ b/samples/warpdrive/wd_adapter.h @@ -0,0 +1,44 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __WD_ADAPTER_H__ +#define __WD_ADAPTER_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "wd.h" + +struct wd_drv_dio_if { + char *hw_type; + int (*open)(struct wd_queue *q); + void (*close)(struct wd_queue *q); + int (*set_pasid)(struct wd_queue *q); + int (*unset_pasid)(struct wd_queue *q); + int (*send)(struct wd_queue *q, void *req); + int (*recv)(struct wd_queue *q, void **req); + void (*flush)(struct wd_queue *q); + int (*share)(struct wd_queue *q, const void *addr, + size_t size, int flags); + int (*unshare)(struct wd_queue *q, const void *addr, size_t size); + int (*get_capa)(struct wd_capa *capa); +}; + +extern int drv_open(struct wd_queue *q); +extern void drv_close(struct wd_queue *q); +extern int drv_send(struct wd_queue *q, void *req); +extern int drv_recv(struct wd_queue *q, void **req); +extern void drv_flush(struct wd_queue *q); +extern int drv_share(struct wd_queue *q, const void *addr, + size_t size, int flags); +extern void drv_unshare(struct wd_queue *q, const void *addr, size_t size); +extern int drv_can_do_mem_share(struct wd_queue *q); +extern int wd_drv_dio_tbl_set(struct wd_drv_dio_if *tbl); +extern int _wd_dyn_load(char *drv); +extern int _wd_get_capa(char *drv, struct wd_capa *capa); +#endif diff --git a/samples/warpdrive/wd_comp.c b/samples/warpdrive/wd_comp.c new file mode 100644 index 000000000000..a67aa2952fed --- /dev/null +++ b/samples/warpdrive/wd_comp.c @@ -0,0 +1,157 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "wd.h" +#include "wd_comp.h" +#include "wd_util.h" + + +struct wd_comp_udata { + void *tag; + struct wd_comp_opdata *opdata; +}; + +struct wd_comp_ctx { + struct wd_comp_msg cache_msg; + struct wd_queue *q; + wd_comp_cb cb; + char alg[32]; +}; + + +/* Before initiate this context, we should get a queue from WD */ +void *wd_create_comp_ctx(struct wd_queue *q, struct wd_comp_ctx_setup *setup) +{ + struct wd_comp_ctx *ctx; + + if (!q || !setup) { + WD_ERR("%s(): input param err!\n", __func__); + return NULL; + } + ctx = malloc(sizeof(*ctx)); + if (!ctx) { + WD_ERR("Alloc ctx memory fail!\n"); + return ctx; + } + memset(ctx, 0, sizeof(*ctx)); + ctx->q = q; + strncpy(ctx->alg, q->capa.alg, strlen(q->capa.alg)); + ctx->cache_msg.aflags = setup->aflags; + ctx->cache_msg.comp_lv = setup->comp_lv; + ctx->cache_msg.humm_type = setup->humm_type; + ctx->cache_msg.win_size = setup->win_size; + ctx->cache_msg.file_type = setup->file_type; + ctx->cache_msg.alg = ctx->alg; + ctx->cb = setup->cb; + q->ctx = ctx; + + return ctx; +} + +int wd_do_comp(void *ctx, struct wd_comp_opdata *opdata) +{ + struct wd_comp_ctx *ctxt = ctx; + struct wd_comp_msg *resp; + int ret; + + if (!ctx || !opdata) { + WD_ERR("%s(): input param err!\n", __func__); + return -1; + } + ctxt->cache_msg.cflags = *(opdata->cflush); + ctxt->cache_msg.src = (__u64)opdata->in; + ctxt->cache_msg.dst = (__u64)opdata->out; + ctxt->cache_msg.in_bytes = (__u32)opdata->in_bytes; + + ret = wd_send(ctxt->q, &ctxt->cache_msg); + if (ret) { + WD_ERR("%s():wd_send err!\n", __func__); + return ret; + } + ret = wd_recv_sync(ctxt->q, (void **)&resp, 0); + if (ret != 1) { + WD_ERR("%s():wd_recv_sync err!ret=%d\n", __func__, ret); + return ret; + } + + *(opdata->out_bytes) = resp->out_bytes; + *(opdata->comsumed) = resp->in_coms; + *(opdata->cflush) = resp->cflags; + + return 0; +} + +int wd_comp_op(void *ctx, struct wd_comp_opdata *opdata, void *tag) +{ + struct wd_comp_ctx *context = ctx; + struct wd_comp_msg *msg = &context->cache_msg; + int ret; + struct wd_comp_udata *udata; + + if (!ctx || !opdata) { + WD_ERR("%s(): input param err!\n", __func__); + return -1; + } + + msg->cflags = *(opdata->cflush); + msg->src = (__u64)opdata->in; + msg->dst = (__u64)opdata->out; + msg->in_bytes = (__u32)opdata->in_bytes; + + /* malloc now, as need performance we should rewrite mem management */ + udata = malloc(sizeof(*udata)); + if (!udata) { + WD_ERR("malloc udata fail!\n"); + return -1; + } + udata->tag = tag; + udata->opdata = opdata; + + msg->udata = (__u64)udata; + ret = wd_send(context->q, (void *)msg); + if (ret < 0) { + WD_ERR("wd send request fail!\n"); + return -1; + } + + return 0; +} + +int wd_comp_poll(struct wd_queue *q, int num) +{ + int ret, count = 0, status = 0; + struct wd_comp_msg *resp; + struct wd_comp_ctx *ctx = q->ctx; + struct wd_comp_udata *udata; + + do { + ret = wd_recv(q, (void **)&resp); + if (ret < 1) + break; + count++; + udata = (void *)resp->udata; + status = (int)resp->status; + *(udata->opdata->cflush) = resp->cflags; + *(udata->opdata->out_bytes) = resp->out_bytes; + *(udata->opdata->comsumed) = resp->in_coms; + ctx->cb(udata->tag, status, (void *)udata->opdata); + free(udata); + } while (--num); + + return count; +} + +void wd_del_comp_ctx(void *ctx) +{ + if (ctx) + free(ctx); +} diff --git a/samples/warpdrive/wd_comp.h b/samples/warpdrive/wd_comp.h new file mode 100644 index 000000000000..faaab9291c80 --- /dev/null +++ b/samples/warpdrive/wd_comp.h @@ -0,0 +1,91 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef __WD_COMP_H +#define __WD_COMP_H + +#include +#include + +#include "wd.h" + + +/* compressing algorithms' parameters */ +struct wd_comp_param { + __u32 win_size; + __u32 comp_lv; + __u32 mode; + __u32 alg; +}; + +enum wd_comp_op { + WD_COMP_INVALID, + WD_COMP_PRESS, + WD_COMP_INFLATE, + WD_COMP_PSSTHRH, +}; + +enum wd_comp_flush { + WD_INVALID_CFLUSH, + WD_NO_CFLUSH, + WD_PARTIAL_CFLUSH, + WD_FULL_CFLUSH, + WD_FINISH, +}; + +typedef void (*wd_comp_cb)(void *tag, int status, void *opdata); + +struct wd_comp_ctx_setup { + char *alg; + wd_comp_cb cb; + __u32 win_size; + __u32 aflags; + __u8 op_type; + __u8 humm_type; + __u8 comp_lv; + __u8 file_type; +}; + +struct wd_comp_opdata { + __u32 *cflush; + __u8 *in; + __u32 in_bytes; + __u32 *comsumed; + __u8 *out; + __u32 *out_bytes; +}; + +struct wd_comp_msg { + + /* First 8 bytes of the message must indicate algorithm */ + union { + char *alg; + __u64 pading; + }; + + /* address type */ + __u32 aflags; + + /* Comsumed bytes of input data */ + __u32 in_coms; + __u32 in_bytes; + __u32 out_bytes; + __u64 src; + __u64 dst; + __u8 comp_lv; + __u8 file_type; + __u8 humm_type; + __u8 op_type; + __u32 win_size; + + /* This flag indicates the output mode, from enum wd_comp_flush */ + __u32 cflags; + __u32 status; + __u64 udata; +}; + +void *wd_create_comp_ctx(struct wd_queue *q, struct wd_comp_ctx_setup *setup); + +int wd_do_comp(void *ctx, struct wd_comp_opdata *opdata); +int wd_comp_op(void *ctx, struct wd_comp_opdata *opdata, void *tag); +int wd_comp_poll(struct wd_queue *q, int num); +void wd_del_comp_ctx(void *ctx); +#endif diff --git a/samples/warpdrive/wd_dh.c b/samples/warpdrive/wd_dh.c new file mode 100644 index 000000000000..2fd613c774ad --- /dev/null +++ b/samples/warpdrive/wd_dh.c @@ -0,0 +1,166 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "wd.h" +#include "wd_dh.h" +#include "wd_util.h" + + +struct wd_dh_udata { + void *tag; + struct wd_dh_op_data *opdata; +}; + +struct wd_dh_ctx { + struct wd_dh_msg cache_msg; + struct wd_queue *q; + wd_dh_cb cb; + char alg[32]; +}; + + +/* Before initiate this context, we should get a queue from WD */ +void *wd_create_dh_ctx(struct wd_queue *q, struct wd_dh_ctx_setup *setup) +{ + struct wd_dh_ctx *ctx; + + if (!q || !setup) { + WD_ERR("%s(): input param err!\n", __func__); + return NULL; + } + ctx = malloc(sizeof(*ctx)); + if (!ctx) { + WD_ERR("Alloc ctx memory fail!\n"); + return ctx; + } + memset(ctx, 0, sizeof(*ctx)); + ctx->q = q; + strncpy(ctx->alg, q->capa.alg, strlen(q->capa.alg)); + ctx->cache_msg.aflags = setup->aflags; + + ctx->cache_msg.alg = ctx->alg; + ctx->cb = setup->cb; + q->ctx = ctx; + + return ctx; +} + +int wd_do_dh(void *ctx, struct wd_dh_op_data *opdata) +{ + struct wd_dh_ctx *ctxt = ctx; + struct wd_dh_msg *resp; + int ret; + + if (!ctx || !opdata) { + WD_ERR("%s(): input param err!\n", __func__); + return -1; + } + if (opdata->op_type == WD_DH_PHASE1 || + opdata->op_type == WD_DH_PHASE2) { + ctxt->cache_msg.p = (__u64)opdata->p; + ctxt->cache_msg.g = (__u64)opdata->g; + ctxt->cache_msg.x = (__u64)opdata->x; + ctxt->cache_msg.pbytes = (__u16)opdata->pbytes; + ctxt->cache_msg.gbytes = (__u16)opdata->gbytes; + ctxt->cache_msg.xbytes = (__u16)opdata->xbytes; + ctxt->cache_msg.pri = (__u64)opdata->pri; + } else { + WD_ERR("%s():operatinal type err!\n", __func__); + return -1; + } + + ret = wd_send(ctxt->q, &ctxt->cache_msg); + if (ret) { + WD_ERR("%s():wd_send err!\n", __func__); + return ret; + } + ret = wd_recv_sync(ctxt->q, (void **)&resp, 0); + if (ret != 1) { + WD_ERR("%s():wd_recv_sync err!ret=%d\n", __func__, ret); + return ret; + } + + *(opdata->pri_bytes) = resp->pribytes; + + return 0; +} + +int wd_dh_op(void *ctx, struct wd_dh_op_data *opdata, void *tag) +{ + struct wd_dh_ctx *context = ctx; + struct wd_dh_msg *msg = &context->cache_msg; + int ret; + struct wd_dh_udata *udata; + + + msg->status = 0; + + /* malloc now, as need performance we should rewrite mem management */ + udata = malloc(sizeof(*udata)); + if (!udata) { + WD_ERR("malloc udata fail!\n"); + return -1; + } + udata->tag = tag; + udata->opdata = opdata; + if (opdata->op_type == WD_DH_PHASE1 || + opdata->op_type == WD_DH_PHASE2) { + msg->p = (__u64)opdata->p; + msg->g = (__u64)opdata->g; + msg->x = (__u64)opdata->x; + msg->pbytes = (__u16)opdata->pbytes; + msg->gbytes = (__u16)opdata->gbytes; + msg->xbytes = (__u16)opdata->xbytes; + msg->pri = (__u64)opdata->pri; + } else { + WD_ERR("%s():operatinal type err!\n", __func__); + return -1; + } + + msg->udata = (__u64)udata; + ret = wd_send(context->q, (void *)msg); + if (ret < 0) { + WD_ERR("wd send request fail!\n"); + return -1; + } + + return 0; +} + +int wd_dh_poll(struct wd_queue *q, int num) +{ + int ret, count = 0; + struct wd_dh_msg *resp; + struct wd_dh_ctx *ctx = q->ctx; + unsigned int status; + struct wd_dh_udata *udata; + + do { + ret = wd_recv(q, (void **)&resp); + if (ret < 1) + break; + count++; + udata = (void *)resp->udata; + *(udata->opdata->pri_bytes) = (__u16)resp->pribytes; + status = resp->status; + ctx->cb(udata->tag, status, (void *)udata->opdata); + free(udata); + } while (--num); + + return count; +} + +void wd_del_dh_ctx(void *ctx) +{ + if (ctx) + free(ctx); +} diff --git a/samples/warpdrive/wd_dh.h b/samples/warpdrive/wd_dh.h new file mode 100644 index 000000000000..b4bdba26a39a --- /dev/null +++ b/samples/warpdrive/wd_dh.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef __WD_DH_H +#define __WD_DH_H + +#include +#include + +#include "../../include/uapi/linux/vfio_spimdev.h" + +enum wd_dh_op { + WD_DH_INVALID, + WD_DH_PHASE1, + WD_DH_PHASE2, +}; + +typedef void (*wd_dh_cb)(void *tag, int status, void *opdata); + +struct wd_dh_ctx_setup { + char *alg; + wd_dh_cb cb; + __u16 aflags; +}; + +struct wd_dh_op_data { + void *p; + void *x; + + /* it is PV also at phase 2 */ + void *g; + + /* phase 1&&2 output */ + void *pri; + __u16 *pri_bytes; + + __u16 pbytes; + __u16 xbytes; + __u16 gbytes; + + enum wd_dh_op op_type; +}; + +struct wd_dh_msg { + + /* First 8 bytes of the message must indicate algorithm */ + union { + char *alg; + __u64 pading; + }; + + /* address type */ + __u16 aflags; + __u8 op_type; + __u8 resv; + __u32 status; + + __u64 p; + __u64 x; + + /* is PV also at phase 2 */ + __u64 g; + + /* result address */ + __u64 pri; + + __u16 pbytes; + __u16 xbytes; + __u16 gbytes; + __u16 pribytes; + __u64 udata; +}; + +void *wd_create_dh_ctx(struct wd_queue *q, struct wd_dh_ctx_setup *setup); + +/* Synchronous mode API of DH*/ +int wd_do_dh(void *ctx, struct wd_dh_op_data *opdata); + +/* Asynchronous mode APIs of DH */ +int wd_dh_op(void *ctx, struct wd_dh_op_data *opdata, void *tag); +int wd_dh_poll(struct wd_queue *q, int num); + + +void wd_del_dh_ctx(void *ctx); +#endif diff --git a/samples/warpdrive/wd_rsa.c b/samples/warpdrive/wd_rsa.c new file mode 100644 index 000000000000..4fadc859661f --- /dev/null +++ b/samples/warpdrive/wd_rsa.c @@ -0,0 +1,263 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "wd.h" +#include "wd_rsa.h" +#include "wd_util.h" + + +struct wd_rsa_udata { + void *tag; + struct wd_rsa_op_data *opdata; +}; + +struct wd_rsa_ctx { + struct wd_rsa_msg cache_msg; + struct wd_queue *q; + char alg[32]; + wd_rsa_cb cb; + __u32 key_size; + __u32 is_crt; + struct wd_rsa_pubkey pubkey; + union wd_rsa_prikey prikey; +}; + + +/* Before initiate this context, we should get a queue from WD */ +void *wd_create_rsa_ctx(struct wd_queue *q, struct wd_rsa_ctx_setup *setup) +{ + struct wd_rsa_ctx *ctx; + __u32 prikey_size, pubkey_size; + + if (!q || !setup) { + WD_ERR("%s(): input param err!\n", __func__); + return NULL; + } + if (strncmp(setup->alg, "rsa", 3) || strncmp(q->capa.alg, "rsa", 3)) { + WD_ERR("%s(): algorithm mismatching!\n", __func__); + return NULL; + } + if (setup->is_crt) + prikey_size = 5 * (setup->key_bits >> 4); + else + prikey_size = 2 * (setup->key_bits >> 3); + pubkey_size = 2 * (setup->key_bits >> 3); + ctx = malloc(sizeof(*ctx) + pubkey_size + prikey_size); + if (!ctx) { + WD_ERR("Alloc ctx memory fail!\n"); + return ctx; + } + memset(ctx, 0, sizeof(*ctx) + pubkey_size + prikey_size); + ctx->q = q; + strncpy(ctx->alg, q->capa.alg, strlen(q->capa.alg)); + if (setup->is_crt) + ctx->cache_msg.prikey_type = WD_RSA_PRIKEY2; + else + ctx->cache_msg.prikey_type = WD_RSA_PRIKEY1; + ctx->cache_msg.aflags = setup->aflags; + ctx->cache_msg.pubkey = (__u64)&ctx->pubkey; + ctx->pubkey.e = (__u8 *)ctx + sizeof(*ctx); + ctx->pubkey.n = ctx->pubkey.e + (setup->key_bits >> 3); + ctx->cache_msg.prikey = (__u64)&ctx->prikey; + if (setup->is_crt) { + ctx->prikey.pkey2.dq = ctx->pubkey.n + (setup->key_bits >> 3); + ctx->prikey.pkey2.dp = ctx->prikey.pkey2.dq + + (setup->key_bits >> 4); + ctx->prikey.pkey2.q = ctx->prikey.pkey2.dp + + (setup->key_bits >> 4); + ctx->prikey.pkey2.p = ctx->prikey.pkey2.q + + (setup->key_bits >> 4); + ctx->prikey.pkey2.qinv = ctx->prikey.pkey2.p + + (setup->key_bits >> 4); + } else { + ctx->prikey.pkey1.d = ctx->pubkey.n + (setup->key_bits >> 3); + ctx->prikey.pkey1.n = ctx->prikey.pkey1.d + + (setup->key_bits >> 3); + } + ctx->cache_msg.nbytes = setup->key_bits >> 3; + ctx->cache_msg.alg = ctx->alg; + ctx->cb = setup->cb; + ctx->is_crt = setup->is_crt; + ctx->key_size = setup->key_bits >> 3; + q->ctx = ctx; + + return ctx; +} + +int wd_rsa_is_crt(void *ctx) +{ + if (ctx) + return ((struct wd_rsa_ctx *)ctx)->is_crt; + else + return 0; +} + +int wd_rsa_key_bits(void *ctx) +{ + if (ctx) + return (((struct wd_rsa_ctx *)ctx)->key_size) << 3; + else + return 0; +} + +int wd_set_rsa_pubkey(void *ctx, struct wd_rsa_pubkey *pubkey) +{ + void *p; + + if (ctx && pubkey && pubkey->e && pubkey->n) { + p = (void *)((struct wd_rsa_ctx *)ctx)->cache_msg.pubkey; + memcpy(p, (void *)pubkey, sizeof(struct wd_rsa_pubkey)); + return 0; + } + + return -1; +} + +void wd_get_rsa_pubkey(void *ctx, struct wd_rsa_pubkey **pubkey) +{ + if (ctx && pubkey) + *pubkey = (void *)((struct wd_rsa_ctx *)ctx)->cache_msg.pubkey; +} + +int wd_set_rsa_prikey(void *ctx, union wd_rsa_prikey *prikey) +{ + void *p; + + if (!ctx && !prikey) + return -1; + + if (wd_rsa_is_crt(ctx)) { + if (!(prikey->pkey2.dp) || !(prikey->pkey2.dq) || + !(prikey->pkey2.p) || !(prikey->pkey2.q) || + !(prikey->pkey2.qinv)) + return -1; + + } else { + if (!(prikey->pkey1.n) || !(prikey->pkey1.d)) + return -1; + + } + p = (void *)((struct wd_rsa_ctx *)ctx)->cache_msg.prikey; + memcpy(p, (void *)prikey, sizeof(union wd_rsa_prikey)); + + return 0; +} + +void wd_get_rsa_prikey(void *ctx, union wd_rsa_prikey **prikey) +{ + if (ctx && prikey) + *prikey = (void *)((struct wd_rsa_ctx *)ctx)->cache_msg.prikey; +} + +int wd_do_rsa(void *ctx, struct wd_rsa_op_data *opdata) +{ + struct wd_rsa_ctx *ctxt = ctx; + struct wd_rsa_msg *resp; + int ret; + + if (opdata->op_type == WD_RSA_SIGN || + opdata->op_type == WD_RSA_VERIFY) { + ctxt->cache_msg.in = (__u64)opdata->in; + ctxt->cache_msg.inbytes = (__u16)opdata->in_bytes; + ctxt->cache_msg.out = (__u64)opdata->out; + } + ctxt->cache_msg.op_type = (__u8)opdata->op_type; + ctxt->cache_msg.status = -1; + + ret = wd_send(ctxt->q, &ctxt->cache_msg); + if (ret) { + WD_ERR("%s():wd_send err!\n", __func__); + return ret; + } + +recv_again: + ret = wd_recv(ctxt->q, (void **)&resp); + if (!ret) { + usleep(1); + goto recv_again; + } else if (ret < 0) { + return ret; + } + opdata->out = (void *)resp->out; + opdata->out_bytes = resp->outbytes; + + return 0; +} + +int wd_rsa_op(void *ctx, struct wd_rsa_op_data *opdata, void *tag) +{ + struct wd_rsa_ctx *context = ctx; + struct wd_rsa_msg *msg = &context->cache_msg; + int ret; + struct wd_rsa_udata *udata; + + if (!ctx || !opdata) { + WD_ERR("param err!\n"); + return -1; + } + msg->status = 0; + + /* malloc now, as need performance we should rewrite mem management */ + udata = malloc(sizeof(*udata)); + if (!udata) { + WD_ERR("malloc udata fail!\n"); + return -1; + } + udata->tag = tag; + udata->opdata = opdata; + if (opdata->op_type == WD_RSA_SIGN || + opdata->op_type == WD_RSA_VERIFY) { + msg->in = (__u64)opdata->in; + msg->inbytes = (__u16)opdata->in_bytes; + msg->out = (__u64)opdata->out; + } + //msg->pubkey = (__u64)opdata->pubkey; + //msg->prikey = (__u64)opdata->prikey; + msg->udata = (__u64)udata; + msg->op_type = (__u8)opdata->op_type; + ret = wd_send(context->q, (void *)msg); + if (ret < 0) { + WD_ERR("wd send request fail!\n"); + return -1; + } + + return 0; +} + +int wd_rsa_poll(struct wd_queue *q, int num) +{ + int ret, count = 0; + struct wd_rsa_msg *resp; + struct wd_rsa_ctx *ctx = q->ctx; + unsigned int status; + struct wd_rsa_udata *udata; + + do { + ret = wd_recv(q, (void **)&resp); + if (ret < 1) + break; + count++; + udata = (void *)resp->udata; + udata->opdata->out_bytes = (__u32)resp->outbytes; + status = resp->status; + ctx->cb(udata->tag, status, udata->opdata); + free(udata); + } while (--num); + + return count; +} + +void wd_del_rsa_ctx(void *ctx) +{ + if (ctx) + free(ctx); +} diff --git a/samples/warpdrive/wd_rsa.h b/samples/warpdrive/wd_rsa.h new file mode 100644 index 000000000000..2c461eb6aa39 --- /dev/null +++ b/samples/warpdrive/wd_rsa.h @@ -0,0 +1,112 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +#ifndef __WD_RSA_H +#define __WD_RSA_H + +#include +#include + +#include "../../include/uapi/linux/vfio_spimdev.h" + +enum wd_rsa_op { + WD_RSA_INVALID, + WD_RSA_SIGN, + WD_RSA_VERIFY, + WD_RSA_GENKEY, +}; + +enum wd_rsa_prikey_type { + WD_RSA_PRIKEY1 = 1, + WD_RSA_PRIKEY2 = 2, +}; + +typedef void (*wd_rsa_cb)(void *tag, int status, void *opdata); + +struct wd_rsa_ctx_setup { + char *alg; + wd_rsa_cb cb; + __u16 aflags; + __u16 key_bits; + __u32 is_crt; +}; + +struct wd_rsa_pubkey { + __u8 *n; + __u8 *e; + __u32 bytes; +}; + +struct wd_rsa_prikey1 { + __u8 *n; + __u8 *d; + __u32 bytes; +}; + +struct wd_rsa_prikey2 { + __u8 *p; + __u8 *q; + __u8 *dp; + __u8 *dq; + __u8 *qinv; + __u32 bytes; +}; + +union wd_rsa_prikey { + struct wd_rsa_prikey1 pkey1; + struct wd_rsa_prikey2 pkey2; +}; + +struct wd_rsa_op_data { + enum wd_rsa_op op_type; + int status; + void *in; + void *out; + __u32 in_bytes; + __u32 out_bytes; +}; + +struct wd_rsa_msg { + + /* First 8 bytes of the message must indicate algorithm */ + union { + char *alg; + __u64 pading; + }; + + /* address type */ + __u16 aflags; + __u8 op_type; + __u8 prikey_type; + __u32 status; + + __u64 in; + __u64 out; + __u64 pubkey; + + /* private key */ + __u64 prikey; + + __u16 nbytes; + __u16 inbytes; + __u16 outbytes; + __u16 resv; + + __u64 udata; +}; + +int wd_rsa_is_crt(void *ctx); +int wd_rsa_key_bits(void *ctx); +void *wd_create_rsa_ctx(struct wd_queue *q, struct wd_rsa_ctx_setup *setup); +int wd_set_rsa_pubkey(void *ctx, struct wd_rsa_pubkey *pubkey); +void wd_get_rsa_pubkey(void *ctx, struct wd_rsa_pubkey **pubkey); +int wd_set_rsa_prikey(void *ctx, union wd_rsa_prikey *prikey); +void wd_get_rsa_prikey(void *ctx, union wd_rsa_prikey **prikey); + +/* this is a synchronous mode RSA API */ +int wd_do_rsa(void *ctx, struct wd_rsa_op_data *opdata); + +/* this is a pair of asynchronous mode RSA APIs */ +int wd_rsa_op(void *ctx, struct wd_rsa_op_data *opdata, void *tag); +int wd_rsa_poll(struct wd_queue *q, int num); + +void wd_del_rsa_ctx(void *ctx); +#endif diff --git a/samples/warpdrive/wd_util.c b/samples/warpdrive/wd_util.c new file mode 100644 index 000000000000..b4e8a465c4b5 --- /dev/null +++ b/samples/warpdrive/wd_util.c @@ -0,0 +1,18 @@ +// SPDX-License-Identifier: GPL-2.0+ +#include +#include +#include + +#include "wd_util.h" + +void wd_spinlock(struct wd_lock *lock) +{ + while (__atomic_test_and_set(&lock->lock, __ATOMIC_ACQUIRE)) + while (__atomic_load_n(&lock->lock, __ATOMIC_RELAXED)) + ; +} + +void wd_unspinlock(struct wd_lock *lock) +{ + __atomic_clear(&lock->lock, __ATOMIC_RELEASE); +} diff --git a/samples/warpdrive/wd_util.h b/samples/warpdrive/wd_util.h new file mode 100644 index 000000000000..882b21cf58fd --- /dev/null +++ b/samples/warpdrive/wd_util.h @@ -0,0 +1,110 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* the common drv header define the unified interface for wd */ +#ifndef __WD_UTIL_H__ +#define __WD_UTIL_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "../../include/uapi/linux/vfio.h" +#include "wd.h" + + +struct wd_lock { + __u8 lock; +}; + +#ifndef WD_ERR +#define WD_ERR(format, args...) fprintf(stderr, format, ##args) +#endif + +#define alloc_obj(objp) do { \ + objp = malloc(sizeof(*objp)); \ + memset(objp, 0, sizeof(*objp)); \ +} while (0) +#define free_obj(objp) do { if (objp)free(objp); } while (0) + +void wd_spinlock(struct wd_lock *lock); +void wd_unspinlock(struct wd_lock *lock); + +static inline void wd_reg_write(void *reg_addr, uint32_t value) +{ + *((uint32_t *)reg_addr) = value; +} + +static inline uint32_t wd_reg_read(void *reg_addr) +{ + uint32_t temp; + + temp = *((uint32_t *)reg_addr); + + return temp; +} + +static inline int _get_attr_str(const char *path, char *value) +{ + int fd, ret; + + fd = open(path, O_RDONLY); + if (fd < 0) { + WD_ERR("open %s fail\n", path); + return fd; + } + memset(value, 0, SYS_VAL_SIZE); + ret = read(fd, value, SYS_VAL_SIZE); + if (ret > 0) { + close(fd); + return ret; + } + close(fd); + + return -EINVAL; +} + +static inline int _get_attr_int(const char *path) +{ + char value[SYS_VAL_SIZE]; + + _get_attr_str(path, value); + return atoi(value); +} + +static inline int _get_dir_attr_int(const char *dir, char *attr) +{ + char attr_path[PATH_STR_SIZE]; + char value[SYS_VAL_SIZE]; + int ret; + + if (strlen(dir) + strlen(attr) > PATH_STR_SIZE - 1) + return -EINVAL; + ret = snprintf(attr_path, PATH_STR_SIZE, "%s/%s", dir, attr); + if (ret < 0) + return ret; + _get_attr_str(attr_path, value); + + return atoi(value); +} + +static inline int _get_dir_attr_str(const char *dir, char *attr, char *str) +{ + char attr_path[PATH_STR_SIZE]; + int ret; + + if (strlen(dir) + strlen(attr) > PATH_STR_SIZE - 1) + return -EINVAL; + ret = snprintf(attr_path, PATH_STR_SIZE, "%s/%s", dir, attr); + if (ret < 0) + return ret; + + return _get_attr_str(attr_path, str); +} +#endif -- GitLab