RDMA/hns: Add roce dfx files

driver inclusion
category: bugfix
bugzilla: NA
CVE: NA

This patch adds roce dfx files. These dfx files provide
the ability to adjust some specifications of roce driver
and the ability to query information of several main rdma
objects, such as QP, MR, CQ, etc.
Reviewed-by: NHuang Dongdong <hdd.huang@huawei.com>
Signed-off-by: NHu Chunzhi <huchunzhi@huawei.com>
Acked-by: NHanjun Guo <guohanjun@huawei.com>
Signed-off-by: NYang Yingliang <yangyingliang@huawei.com>

MAINTAINERS

@@ -6665,6 +6665,12 @@ S:	Maintained
 F:	drivers/infiniband/hw/hns/
 F:	Documentation/devicetree/bindings/infiniband/hisilicon-hns-roce.txt
 
+HISILICON ROCE DFX DRIVER
+M:	Huang Dongdong <hdd.huang@huawei.com>
+S:	Maintained
+F:	drivers/infiniband/hw/hns/Makefile
+F:	drivers/infiniband/hw/hns/roce-customer/
+
 HISILICON WARPDRIVE USER
 M:	Zaibo Xu <xuzaibo@huawei.com>
 M:	Zhou Wang <wangzhou1@hisilicon.com>

drivers/infiniband/hw/hns/Kconfig

@@ -29,3 +29,22 @@ config INFINIBAND_HNS_HIP08
 
 	  To compile this driver as a module, choose M here: the module
 	  will be called hns-roce-hw-v2.
+
+config INFINIBAND_HNS_DFX
+	tristate "Hisilicon Hip08 Family RoCE DFX support"
+	depends on INFINIBAND_HNS_HIP08
+	help
+	  RoCE DFX driver support for Hisilicon RoCE engine in Hisilicon Hip08
+	  SoC.
+
+	  To compile this driver as a module, choose M here: the module
+	  will be called hns-roce-cae.
+
+config INFINIBAND_HNS_TEST
+	tristate "Hisilicon Hip08 Family RoCE test support"
+	depends on INFINIBAND_HNS_HIP08
+	help
+	  This option provides the ability to adjust some parameters of roce
+	  driver for debugging.
+
+	  To turn this feature on, choose M here.

drivers/infiniband/hw/hns/Makefile

@@ -4,12 +4,27 @@
 
 ccflags-y :=  -Idrivers/net/ethernet/hisilicon/hns3
 
+PWD = $(srctree)/drivers/infiniband/hw/hns/
+ccflags-y += -I$(PWD) \
+	-I$(PWD)/roce-customer/
+ifeq ($(CONFIG_INFINIBAND_HNS_DFX), m)
+	ccflags-y += -DCONFIG_INFINIBAND_HNS_DFX
+endif
+
+ifeq ($(CONFIG_INFINIBAND_HNS_TEST), m)
+	ccflags-y += -DCONFIG_INFINIBAND_HNS_TEST
+endif
+
 obj-$(CONFIG_INFINIBAND_HNS) += hns-roce.o
 hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
 	hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \
 	hns_roce_sysfs.o \
 	hns_roce_cq.o hns_roce_alloc.o hns_roce_db.o hns_roce_srq.o hns_roce_restrack.o
+hns-roce-$(CONFIG_INFINIBAND_HNS_TEST) += roce-customer/rdfx_intf.o roce-customer/hns_roce_test.o
 obj-$(CONFIG_INFINIBAND_HNS_HIP06) += hns-roce-hw-v1.o
 hns-roce-hw-v1-objs := hns_roce_hw_v1.o
 obj-$(CONFIG_INFINIBAND_HNS_HIP08) += hns-roce-hw-v2.o
 hns-roce-hw-v2-objs := hns_roce_hw_v2.o hns_roce_hw_v2_dfx.o hns_roce_hw_sysfs_v2.o
+hns-roce-hw-v2-$(CONFIG_INFINIBAND_HNS_TEST) += roce-customer/hns_hw_v2_test.o
+obj-$(CONFIG_INFINIBAND_HNS_DFX) += hns-roce-cae.o
+hns-roce-cae-$(CONFIG_INFINIBAND_HNS_DFX) := roce-customer/rdfx_sysfs.o roce-customer/rdfx_hw_v2.o roce-customer/rdfx_main.o

drivers/infiniband/hw/hns/roce-customer/hns_hw_v2_test.c

+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2016-2017 Hisilicon Limited.
+
+#include <linux/module.h>
+
+#include "hns_roce_device.h"
+#include "hnae3.h"
+#include "hns_roce_hw_v2.h"
+#include "hns_hw_v2_test.h"
+
+unsigned int hr_cq_period = 0xa;
+module_param(hr_cq_period, uint, 0644);
+MODULE_PARM_DESC(hr_cq_period, "timeout of cqe to ceqe");
+
+unsigned int hr_cq_max_cnt = 0x1;
+module_param(hr_cq_max_cnt, uint, 0644);
+MODULE_PARM_DESC(hr_cq_max_cnt, "max cnt of cqe to ceqe");
+
+unsigned int hr_ceq_period = HNS_ROCE_V2_EQ_DEFAULT_INTERVAL;
+module_param(hr_ceq_period, uint, 0644);
+MODULE_PARM_DESC(hr_ceq_period, "timeout of ceqe to int");
+
+unsigned int hr_aeq_period = HNS_ROCE_V2_EQ_DEFAULT_INTERVAL;
+module_param(hr_aeq_period, uint, 0644);
+MODULE_PARM_DESC(hr_aeq_period, "timeout of aeqe to int");
+
+unsigned int hr_ceq_max_cnt = 0x1;
+module_param(hr_ceq_max_cnt, uint, 0644);
+MODULE_PARM_DESC(hr_ceq_max_cnt, "max cnt of ceqe to int");
+
+unsigned int hr_aeq_max_cnt = 0x1;
+module_param(hr_aeq_max_cnt, uint, 0644);
+MODULE_PARM_DESC(hr_aeq_max_cnt, "max cnt of aeqe to int");
+
+unsigned int hr_ceq_arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
+module_param(hr_ceq_arm_st, uint, 0644);
+MODULE_PARM_DESC(hr_ceq_arm_st, "arm state of ceq");
+
+unsigned int hr_aeq_arm_st = HNS_ROCE_V2_EQ_ALWAYS_ARMED;
+module_param(hr_aeq_arm_st, uint, 0644);
+MODULE_PARM_DESC(hr_aeq_arm_st, "arm state of aeq");
+
+unsigned int func_num = 0x1;
+module_param(func_num, uint, 0644);
+MODULE_PARM_DESC(func_num, "function num contain pfs and vfs");
+
+unsigned int dump_cqe_en = 0x1;
+module_param(dump_cqe_en, uint, 0644);
+MODULE_PARM_DESC(dump_cqe_en, "dump cqe while cqe status isn't success nor flush error");
+
+unsigned int v2_resv_pds;
+module_param(v2_resv_pds, uint, 0644);
+MODULE_PARM_DESC(v2_resv_pds, "reserved pd num, default is 0");
+
+unsigned int v2_resv_qps = 0x8;
+module_param(v2_resv_qps, uint, 0644);
+MODULE_PARM_DESC(v2_resv_qps, "reserved qp num, default is 8");
+
+unsigned int v2_resv_mrws = 0x1;
+module_param(v2_resv_mrws, uint, 0644);
+MODULE_PARM_DESC(v2_resv_mrws, "reserved mr/mw num, default is 1");
+
+unsigned int v2_resv_cqs;
+module_param(v2_resv_cqs, uint, 0644);
+MODULE_PARM_DESC(v2_resv_cqs, "reserved cq num, default is 0");
+
+unsigned int v2_resv_srqs;
+module_param(v2_resv_srqs, uint, 0644);
+MODULE_PARM_DESC(v2_resv_srqs, "reserved srq num, default is 0");
+
+void test_set_cqc_param(unsigned int *period, unsigned int *max_cnt)
+{
+	*period = hr_cq_period;
+	*max_cnt = hr_cq_max_cnt;
+}
+
+void test_set_eq_param(int eq_type, unsigned int *eq_period,
+		       unsigned int *eq_max_cnt, unsigned int *eq_arm_st)
+{
+	if (eq_type == HNS_ROCE_AEQ) {
+		*eq_period = hr_aeq_period;
+		*eq_max_cnt = hr_aeq_max_cnt;
+		*eq_arm_st = hr_aeq_arm_st;
+	} else {
+		*eq_period = hr_ceq_period;
+		*eq_max_cnt = hr_ceq_max_cnt;
+		*eq_arm_st = hr_ceq_arm_st;
+	}
+}
+
+unsigned int get_func_num(void)
+{
+	if (func_num > 4 || func_num == 0) {
+		pr_info("func num config err. config value - %d\n", func_num);
+		return 1;
+	}
+
+	return func_num;
+}
+
+unsigned int test_get_dump_cqe_en(void)
+{
+	return dump_cqe_en;
+}
+

drivers/infiniband/hw/hns/roce-customer/hns_hw_v2_test.h

+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _HNS_HW_V2_TEST_H
+#define _HNS_HW_V2_TEST_H
+
+void test_set_cqc_param(unsigned int *period, unsigned int *max_cnt);
+void test_set_eq_param(int eq_type, unsigned int *eq_period,
+		       unsigned int *eq_max_cnt, unsigned int *eq_arm_st);
+unsigned int get_func_num(void);
+unsigned int test_get_dump_cqe_en(void);
+unsigned int test_print_mb(void);
+
+extern unsigned int v2_resv_pds;
+extern unsigned int v2_resv_qps;
+extern unsigned int v2_resv_mrws;
+extern unsigned int v2_resv_cqs;
+extern unsigned int v2_resv_srqs;
+
+#endif

drivers/infiniband/hw/hns/roce-customer/hns_roce_test.c

+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2016-2017 Hisilicon Limited.
+
+#include <linux/module.h>
+
+#include "hns_roce_device.h"
+#include "hns_roce_test.h"
+
+unsigned int int_mr_access = 7;
+module_param(int_mr_access, uint, 0644);
+MODULE_PARM_DESC(int_mr_access, "Mr access for ft test");
+
+void test_set_mr_access(struct hns_roce_mr *mr)
+{
+	if (mr->key > 0)
+		mr->access = int_mr_access;
+}
+

drivers/infiniband/hw/hns/roce-customer/hns_roce_test.h

+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef _HNS_ROCE_TEST_H
+#define _HNS_ROCE_TEST_H
+
+void test_set_mr_access(struct hns_roce_mr *mr);
+
+#endif

drivers/infiniband/hw/hns/roce-customer/rdfx_common.h

+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __RDFX_COMMON__
+#define __RDFX_COMMON__
+
+#include <rdma/ib_verbs.h>
+#include "hns_roce_device.h"
+#include "hns_roce_common.h"
+#include "hnae3.h"
+#include "hns_roce_hw_v2.h"
+
+#define WQE_TYPE_MAX	(IB_WR_REG_SIG_MR + 1)
+#define CQ_ST_MAX	(IB_WC_GENERAL_ERR + 1)
+#define RCQE_TYPE_MAX	(RECV_SEND_WITH_INV + 1)
+#define RDFX_FUNC_MAX	(35)
+
+extern const char *rdfx_func_name[RDFX_FUNC_MAX];
+struct rdfx_info;
+
+/**************** kobject attribute ****************/
+struct rdfx_hw_sys_attr {
+	struct attribute attr;
+	int (*pub_show)(struct rdfx_info *rdfx);
+	int (*pub_store)(const char *buf, struct rdfx_info *rdfx);
+};
+
+#define rdfx_hw_file_attr_def(file_name, func_show, func_store) \
+static struct rdfx_hw_sys_attr g_sysfs_roce_##file_name##_attr = {\
+	{\
+		.name = #file_name,\
+		.mode = 0644,\
+	},\
+	.pub_show  = func_show,\
+	.pub_store = func_store,\
+}
+
+#define HW_ATTRS_LIST_MEMBER(file_name)(&g_sysfs_roce_##file_name##_attr.attr)
+
+enum{
+	RECV_RDMA_WITH_IMM,
+	RECV_SEND,
+	RECV_SEND_WITH_IMM,
+	RECV_SEND_WITH_INV,
+};
+
+#define MAX_CHAR_NUM_DEV_NAME	(12)
+struct rdfx_dev_info {
+	atomic_t fc[RDFX_FUNC_MAX];
+	char dev_name[MAX_CHAR_NUM_DEV_NAME];
+};
+
+struct rdfx_pd_info {
+	struct ib_pd		*pd;
+	unsigned long		pdn;
+	struct list_head	list;
+	void			*priv;
+	struct kref		cnt;
+};
+
+struct rdfx_pd_top_info {
+	atomic_t		alloc_pd_cnt;
+	atomic_t		dealloc_pd_cnt;
+	atomic_t		top_pd_index;
+	struct list_head	list;
+	spinlock_t		pd_lock;
+};
+
+struct rdfx_sq_info {
+	atomic_t		sq_wqe_cnt[WQE_TYPE_MAX];
+	u32			sq_wqe_size;
+	u32			sq_sge_size;
+	u32			offset;
+	u32			sq_depth;
+	atomic_t		sig_wqe_cnt;
+	atomic_t		db_cnt;
+	atomic_t		inline_cnt;
+	atomic_t		head;
+	atomic_t		tail;
+	atomic_t		last_comp;
+	u64			head_addr;
+	u64			tail_addr;
+};
+
+struct rdfx_rq_info {
+	atomic_t		rq_wqe_cnt;
+	u32			rq_wqe_size;
+	u32			offset;
+	u32			rq_depth;
+	atomic_t		db_cnt;
+	atomic_t		inline_cnt;
+	atomic_t		head;
+	atomic_t		tail;
+	u64			head_addr;
+	u64			tail_addr;
+};
+
+struct rdfx_qp_attr {
+	atomic_t		state;
+	u32			read_en;
+	u32			write_en;
+	u32			fast_reg_en;
+	u32			atomic_en;
+	u8			max_ord;
+	u8			max_ird;
+	u32			pd_id;
+	u32			err_code;
+	u32			max_sge[2];
+};
+
+struct rdfx_qp_info {
+	struct ib_qp		*qp;
+	unsigned long		qpn;
+	struct rdfx_sq_info	sq;
+	struct rdfx_rq_info	rq;
+	struct rdfx_qp_attr	attr;
+	struct list_head	list;
+	void			*priv;
+	void			*buf;
+	u32			buf_size;
+	struct kref		cnt;
+};
+
+struct rdfx_qp_top_info {
+	atomic_t		alloc_qp_cnt;
+	atomic_t		dealloc_qp_cnt;
+	atomic_t		top_qp_index;
+	atomic_t		dwqe_cnt;
+	struct list_head	list;
+	spinlock_t		qp_lock;
+};
+
+struct rdfx_cq_info {
+	struct ib_cq		*cq;
+	unsigned long		cqn;
+	atomic_t		scqe_cnt[WQE_TYPE_MAX];
+	atomic_t		rcqe_cnt[RCQE_TYPE_MAX];
+	atomic_t		arm_cnt[2];
+	atomic_t		st_cnt[CQ_ST_MAX];
+	atomic_t		ci;
+	u32			cqe_size;
+	u32			cq_depth;
+	struct list_head	list;
+	void			*priv;
+	void			*buf;
+	struct kref		cnt;
+};
+
+struct rdfx_cq_top_info {
+	atomic_t		alloc_cq_cnt;
+	atomic_t		dealloc_cq_cnt;
+	atomic_t		top_cq_index;
+	struct list_head	list;
+	spinlock_t		cq_lock;
+};
+
+struct rdfx_mr_info {
+	struct ib_mr		*mr;
+	u32			pd;
+	struct list_head	list;
+	struct kref		cnt;
+};
+
+struct rdfx_mr_top_info {
+	atomic_t		alloc_mr_cnt;
+	atomic_t		dealloc_mr_cnt;
+	atomic_t		top_mr_index;
+	struct list_head	list;
+	spinlock_t		mr_lock;
+};
+
+struct rdfx_ceq_info {
+	struct list_head	list;
+	unsigned long		ceqn;
+	atomic_t		ceqe_cnt;
+	void			*priv;
+	struct kref		cnt;
+};
+
+struct rdfx_aeq_info {
+	struct list_head	list;
+	atomic_t		aeqe_cnt;
+	void			*priv;
+	struct kref		cnt;
+};
+
+struct rdfx_eq_top_info {
+	struct list_head	ceq_list;
+	struct list_head	aeq_list;
+	spinlock_t		eq_lock;
+};
+
+struct rdfx_ops {
+	int (*add_sysfs)(struct rdfx_info *info);
+	void (*del_sysfs)(struct rdfx_info *info);
+	void (*print_sq_wqe)(void *wqe);
+	void (*print_rq_wqe)(void *wqe);
+	void *(*get_dfx)(struct ib_device *ib_dev);
+};
+
+struct rdfx_info {
+	struct rdfx_dev_info	dev;
+	struct rdfx_pd_top_info	pd;
+	struct rdfx_qp_top_info	qp;
+	struct rdfx_cq_top_info	cq;
+	struct rdfx_mr_top_info	mr;
+	struct rdfx_eq_top_info	eq;
+	struct rdfx_ops		*ops;
+
+	struct device		*drv_dev;
+	struct kobject		kobj;
+
+	void			*priv;
+};
+
+struct rdfx_dev_id {
+	char			name[20];
+	struct rdfx_ops		*ops;
+};
+
+struct rdfx_top_info {
+	struct ib_device	*dev;
+	struct rdfx_info	*rdfx;
+};
+
+void qp_release(struct kref *ref);
+void cq_release(struct kref *ref);
+
+#endif

drivers/infiniband/hw/hns/roce-customer/rdfx_hw_v2.h

+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __RDFX_HW_V2__
+#define __RDFX_HW_V2__
+
+/**
+ * struct rdfx_hw_v1 {
+ *	struct hns_roce_dev *hr_dev;
+ * };
+ */
+
+#define CNT_SNAP_PARAM_DATA_0_CNT_CLR_CE_S 0
+#define CNT_SNAP_PARAM_DATA_0_SNAP_EN_S 1
+
+struct rdfx_query_mbdb_cnt {
+	__le32 mailbox_issue_cnt;
+	__le32 mailbox_exe_cnt;
+	__le32 doorbell_issue_cnt;
+	__le32 doorbell_exe_cnt;
+	__le32 eq_doorbell_issue_cnt;
+	__le32 eq_doorbell_exe_cnt;
+};
+
+struct rdfx_query_mdb_dfx {
+	__le32 empty_info;
+	__le32 data_1;
+	__le32 rsv[2];
+};
+
+#define QUERY_MDB_DFX_EMPTY_INFO_EQDB_EMPTY_S	0
+
+#define QUERY_MDB_DFX_EMPTY_INFO_MB_EMPTY_S	1
+#define QUERY_MDB_DFX_EMPTY_INFO_MB_EMPTY_M \
+	(((1UL << 6) - 1) << QUERY_MDB_DFX_EMPTY_INFO_MB_EMPTY_S)
+
+#define QUERY_MDB_DFX_EMPTY_INFO_DB_EMPTY_S	7
+#define QUERY_MDB_DFX_EMPTY_INFO_DB_EMPTY_M \
+	(((1UL << 4) - 1) << QUERY_MDB_DFX_EMPTY_INFO_DB_EMPTY_S)
+
+#define QUERY_MDB_DFX_DATA_1_EQDB_FULL_S	0
+
+#define QUERY_MDB_DFX_DATA_1_MB_FULL_S		1
+#define QUERY_MDB_DFX_DATA_1_MB_FULL_M \
+	(((1UL << 6) - 1) << QUERY_MDB_DFX_DATA_1_MB_FULL_S)
+
+#define QUERY_MDB_DFX_DATA_1_DB_FULL_S		7
+#define QUERY_MDB_DFX_DATA_1_DB_FULL_M \
+	(((1UL << 4) - 1) << QUERY_MDB_DFX_DATA_1_DB_FULL_S)
+
+#define QUERY_MDB_DFX_DATA_1_DB_CMD_ERR_S	11
+#define QUERY_MDB_DFX_DATA_1_MB_CMD_ERR_S	12
+
+#endif

drivers/infiniband/hw/hns/roce-customer/rdfx_main.c

+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2016-2017 Hisilicon Limited.
+
+#include <linux/module.h>
+#include <linux/kernel.h>
+#include <linux/types.h>
+#include <linux/fs.h>
+#include <linux/errno.h>
+#include <linux/mm.h>
+#include <linux/sched.h>
+#include <linux/init.h>
+#include <linux/cdev.h>
+#include <linux/string.h>
+#include <linux/timer.h>
+#include <linux/slab.h>
+#include <linux/device.h>
+#include <linux/dma-mapping.h>
+#include <linux/kobject.h>
+#include <linux/io.h>
+#include <linux/uaccess.h>
+#include <linux/atomic.h>
+
+#include "rdfx_main.h"
+#include "rdfx_common.h"
+
+struct class *drv_class;
+struct device *drv_device;
+static int major;
+
+struct rdfx_top_info rdfx_top_info_list[MAX_IB_DEV];
+
+const struct file_operations chr_ops = {
+	.owner = THIS_MODULE,
+};
+
+static const struct rdfx_dev_id rdfx_dev_tbl[] = {
+	{.name = "hisi_",	.ops = NULL},
+	{.name = "hns",	.ops = &rdfx_ops_hw_v2},
+	{},
+};
+
+/*
+ * if can not find optstring, return -EINVAL;
+ * if find optstring, return 0
+ *         if there's input value, parg will be set with input
+ *         if there's no input value, parg will be set to '\0'
+ */
+int parg_getopt(char *input, char *optstring, char *parg)
+{
+	char *_input;
+	char *p;
+	int cnt = 0;
+	char _optstring[3];
+
+	if (input == NULL || optstring == NULL)
+		return -EINVAL;
+	_input = kmalloc(strlen(input) + 1, GFP_KERNEL);
+	if (!_input)
+		return -ENOMEM;
+	strcpy(_input, input);
+	_optstring[0] = '-';
+	_optstring[1] = optstring[0];
+	_optstring[2] = '\0';
+	p = strstr(_input, _optstring);
+	if (!p) {
+		kfree(_input);
+		return -EINVAL;
+	}
+	if (optstring[1] == '\0' || parg == NULL) {
+		kfree(_input);
+		return 0;
+	}
+	p += 2;
+	while (*p == ' ')
+		p++;
+	while (*p != ' ' && *p != '\0') {
+		p++;
+		cnt++;
+	}
+	*p = '\0';
+	p -= cnt;
+	strcpy(parg, p);
+	kfree(_input);
+	return 0;
+}
+
+static char *strtok_r(char *s, const char *delim, char **save_ptr)
+{
+	char *token;
+
+	if (s == NULL)
+		s = *save_ptr;
+
+	/* Scan to move after delimiters */
+	s += strspn(s, delim);
+	if (*s == (char)0)
+		return NULL;
+
+	/* Find the end of the token.  */
+	token = s;
+	s = strpbrk(token, delim);
+	if (s == NULL)/* This token finishes the string.  */
+		*save_ptr = strchr(token, 0);
+	else {/* Terminate the token and make *SAVE_PTR point past it.  */
+		*s = 0;
+		*save_ptr = s + 1;
+	}
+
+	return token;
+}
+
+char *strtok(char *s, const char *delim)
+{
+	static char *last;
+
+	return strtok_r(s, delim, &last);
+}
+
+int str_to_ll(char *p_buf, unsigned long long *pll_val, unsigned int *num)
+{
+	unsigned long long lng = 0;
+	long long convert_val;
+	char *p = NULL;
+	char delim[] = ",";
+	unsigned int idx = 0;
+	unsigned int i = 0;
+	unsigned long long *arr = NULL;
+	int ret = 0;
+
+	arr = kzalloc(sizeof(unsigned long long) *
+				SYSFS_MAX_PARA, GFP_KERNEL);
+	if (!arr)
+		return -ENOMEM;
+
+	p = strtok(p_buf, delim);
+	while (p) {
+		if (kstrtoll(p, 0, &convert_val)) {
+			pr_info("convert str failed\n");
+			ret = -EINVAL;
+			goto out;
+		}
+		lng = (unsigned long long)convert_val;
+		arr[idx] = lng;
+		pr_info("arr[%d] = 0x%llx\n", idx, arr[idx]);
+		idx++;
+		if (idx >= SYSFS_MAX_PARA) {
+			pr_err("sub string num should not bigger than 16\n");
+			ret = -EINVAL;
+			goto out;
+		}
+		p = strtok(NULL, delim);
+	}
+
+	*num = idx;
+	for (i = 0; i < idx; i++)
+		*(pll_val + i) = arr[i];
+
+out:
+	kfree(arr);
+
+	return ret;
+}
+
+int str_match(char *s, const char *delim)
+{
+	int num = strspn(s, delim);
+
+	if (num != 0)
+		return 1;
+	else
+		return 0;
+}
+
+int check_input(char *buf, unsigned long long *a_val, unsigned int max,
+		unsigned int min, unsigned int *param_num)
+{
+	int ret;
+
+	ret = str_to_ll(buf, a_val, param_num);
+	if (ret) {
+		pr_info("parse input string err.\r\n");
+		return ret;
+	}
+
+	if ((*param_num > max) || (*param_num < min)) {
+		pr_info("para num(0x%x) is invalid.\n", *param_num);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
+struct rdfx_info *rdfx_find_rdfx_info(char *dev_name)
+{
+	int i;
+
+	if (!strlen(dev_name))
+		return rdfx_top_info_list[0].rdfx;
+
+	for (i = 0; i < MAX_IB_DEV; i++) {
+		if (!rdfx_top_info_list[i].dev)
+			continue;
+		if (!memcmp(dev_name,
+			(rdfx_top_info_list[i].rdfx)->dev.dev_name,
+			strlen((rdfx_top_info_list[i].rdfx)->dev.dev_name))) {
+			pr_info("get rdfx info, name:%s, num: %d",
+				dev_name, i);
+			return rdfx_top_info_list[i].rdfx;
+		}
+	}
+
+	return NULL;
+}
+
+void *rdfx_buf_offset(struct dfx_buf *buf, int offset)
+{
+	u32 bits_per_long_val = BITS_PER_LONG;
+	u32 page_size = 1 << buf->page_shift;
+
+	if ((bits_per_long_val == 64 && buf->page_shift == PAGE_SHIFT) ||
+	    buf->nbufs == 1)
+		return (char *)(buf->direct.buf) + offset;
+	else
+		return (char *)(buf->page_list[offset >> buf->page_shift].buf) +
+		       (offset & (page_size - 1));
+}
+
+static int rdfx_info_init(struct ib_device *ib_dev, int i)
+{
+	int j;
+
+	for (j = 0; j < sizeof(rdfx_dev_tbl)/sizeof(struct rdfx_dev_id); j++) {
+		if (!memcmp(rdfx_dev_tbl[j].name, ib_dev->name,
+		    strlen(rdfx_dev_tbl[j].name))) {
+			if (rdfx_dev_tbl[j].ops)
+				rdfx_top_info_list[i].rdfx =
+					rdfx_dev_tbl[j].ops->get_dfx(ib_dev);
+			(rdfx_top_info_list[i].rdfx)->ops = rdfx_dev_tbl[j].ops;
+			(rdfx_top_info_list[i].rdfx)->drv_dev = drv_device;
+
+			memset(&((rdfx_top_info_list[i].rdfx)->kobj), 0,
+				sizeof(struct kobject));
+			strlcpy((rdfx_top_info_list[i].rdfx)->dev.dev_name,
+				ib_dev->name, IB_DEVICE_NAME_MAX);
+			pr_info("init dev %s success\n", ib_dev->name);
+			break;
+		}
+	}
+
+	if (!rdfx_top_info_list[i].rdfx) {
+		pr_err("dev(%s) not support\n",	ib_dev->name);
+		return -EINVAL;
+	}
+	return 0;
+}
+
+static void rdfx_add_device(struct ib_device *ib_dev)
+{
+	int i = 0;
+	int ret = 0;
+	struct rdfx_ops *ops = NULL;
+
+	for (i = 0; i < MAX_IB_DEV; i++)
+		if (!rdfx_top_info_list[i].dev)
+			break;
+	if (i == MAX_IB_DEV) {
+		pr_err("rdfx add device failed, rdfx top info list is full\n.");
+		return;
+	}
+
+	rdfx_top_info_list[i].dev = ib_dev;
+	pr_info("rdfx add ib device(%p), idx - %d, name - %s\n",
+		ib_dev, i, ib_dev->name);
+	ret = rdfx_info_init(ib_dev, i);
+	if (ret) {
+		pr_err("rdfx info init failed\n");
+		return;
+	}
+
+	ops = (rdfx_top_info_list[i].rdfx)->ops;
+	ret = ops->add_sysfs(rdfx_top_info_list[i].rdfx);
+	if (ret)
+		pr_err("rdfx add hw sysfs failed\n");
+}
+
+static void rdfx_remove_device(struct ib_device *ib_dev, void *client_data)
+{
+	int i = 0;
+	struct rdfx_ops *ops = NULL;
+
+	for (i = 0; i < MAX_IB_DEV; i++) {
+		if (rdfx_top_info_list[i].dev &&
+		    (rdfx_top_info_list[i].dev == ib_dev)) {
+			pr_info("rdfx rm ib device(%p), idx - %d, name - %s\n",
+				ib_dev, i, ib_dev->name);
+			ops = (rdfx_top_info_list[i].rdfx)->ops;
+			ops->del_sysfs(rdfx_top_info_list[i].rdfx);
+			memset(&rdfx_top_info_list[i], 0,
+			       sizeof(struct rdfx_top_info));
+		}
+	}
+}
+
+struct ib_client rdfx_client = {
+	.name   = "rdfx_client",
+	.add	= rdfx_add_device,
+	.remove = rdfx_remove_device,
+};
+
+static int __init rdfx_init(void)
+{
+	int ret = 0;
+
+	major = register_chrdev(0, DFX_DEVICE_NAME, &chr_ops);
+	if (major < 0) {
+		pr_err("Sorry, register the character device failed\n ");
+		return major;
+	}
+
+	/*default content:/sys/class */
+	drv_class = class_create(THIS_MODULE, DFX_DEVICE_NAME);
+	drv_device = device_create(drv_class, NULL, MKDEV(major, 0),
+				   NULL, DFX_DEVICE_NAME);
+
+	memset(rdfx_top_info_list, 0, sizeof(rdfx_top_info_list));
+
+	ret = ib_register_client(&rdfx_client);
+	if (ret) {
+		pr_err("rdfx register client failed, ret = %d\n", ret);
+		goto register_client_failed;
+	}
+
+	/*init and add kobjects*/
+	ret = rdfx_add_common_sysfs(drv_device);
+	if (ret) {
+		pr_err("rdfx add common sysfs failed, ret = %d\n", ret);
+		goto add_common_sysfs_failed;
+	}
+
+	return 0;
+
+add_common_sysfs_failed:
+	ib_unregister_client(&rdfx_client);
+register_client_failed:
+	device_unregister(drv_device);
+	class_destroy(drv_class);
+	unregister_chrdev(major, DFX_DEVICE_NAME);
+
+	return ret;
+}
+
+void __exit rdfx_exit(void)
+{
+	pr_info("rmmod rdfx module\n");
+
+	rdfx_del_common_sysfs();
+	ib_unregister_client(&rdfx_client);
+
+	device_destroy(drv_class, MKDEV(major, 0));
+	class_destroy(drv_class);
+	unregister_chrdev(major, DFX_DEVICE_NAME);
+}
+
+module_init(rdfx_init);
+module_exit(rdfx_exit);
+
+MODULE_AUTHOR("Huawei Tech. Co., Ltd.");
+MODULE_DESCRIPTION("Hisilicon Hip08 Family RoCE DFx Driver");
+MODULE_LICENSE("Dual BSD/GPL");
+

drivers/infiniband/hw/hns/roce-customer/rdfx_main.h

+/* SPDX-License-Identifier: GPL-2.0 */
+
+#ifndef __HI1610_ROCE_KTEST_H__
+#define __HI1610_ROCE_KTEST_H__
+
+#define DFX_DEVICE_NAME "rdma_dfx"
+
+#ifndef SYSFS_PAGE_SIZE
+#define SYSFS_PAGE_SIZE	(4096)           /* sysfs�ļ��Ĵ�С */
+#endif
+#define SYSFS_MAX_PARA	(16)
+#define MAX_IB_DEV	(12)
+#define CQE_SIZE	(32)
+
+#define DEF_OPT_STR_LEN	(10)
+
+int rdfx_add_common_sysfs(struct device *p_dev);
+void rdfx_del_common_sysfs(void);
+
+int parg_getopt(char *input, char *optstring, char *parg);
+char *strtok(char *s, const char *delim);
+int str_to_ll(char *p_buf, unsigned long long *pll_val, unsigned int *num);
+int str_match(char *s, const char *delim);
+int check_input(char *buf, unsigned long long *a_val, unsigned int max,
+		unsigned int min, unsigned int *param_num);
+struct rdfx_info *rdfx_find_rdfx_info(char *dev_name);
+
+struct dfx_buf_list {
+	void		*buf;
+	dma_addr_t	map;
+};
+
+struct dfx_buf {
+	struct dfx_buf_list	direct;
+	struct dfx_buf_list	*page_list;
+	int			nbufs;
+	u32			npages;
+	unsigned int			page_shift;
+};
+
+extern struct rdfx_ops rdfx_ops_hw_v2;
+
+void *rdfx_buf_offset(struct dfx_buf *buf, int offset);
+
+/**
+ * struct roce_dev_kobj {
+ *	struct device		*dev;
+ *	struct kobject		kobj;
+ * };
+ */
+
+#endif