!231 ROH: Support ROH basic functions and adapt ROH mode for RDMA/hns driver

Merge Pull Request from: @chenke1978 
 
[Description]
The ROH module driver consists of the ROH Core and ROH DRV
modules, which work with hardware to implement communication
between nodes through HCCS packets.
ROH Core is a protocol stack of the ROH architecture. It provides
related services for upper layers by invoking operation interfaces
provided by the ROH DRV.
The ROH DRV implements the lower layer functions of the ROH
featureand provides a series of interfaces for operating hardware
for the ROH Core.

This patch supports basic ROH functions, such as: sysfs file node
query, abnormal interrupt handling, reset capability and dfx
information query.

RDMA/hns supports ROH mode, mainly adapted to the device id of ROH,
and the different capabilities and features of RDMA/hns in ROH mode.

[Testing]
kernel options:
CONFIG_ROH=m
CONFIG_ROH_HNS=m

Test passed with below step:
1. Using a hardware environment that supports ROH, insmod net/hns,
 RDMA/hns and ROH related drivers:
 insmod hnae3.ko
 insmod hclge.ko
 insmod hns3.ko
 insmod roh_core.ko
 insmod hns-roh-v1.ko
 insmod hns-roce-hw-v2.ko
2. Check whether ROH generates the corresponding sysfs node:
 ls /sys/class/roh/hns3_0/
3. Check whether the abnormal interrupt information of roh is correct.
 The down or up of the network device link corresponding to roh will
 cause the roh abnormal interrupt count to increase.
 cat /proc/interrupts | grep roh
4. Configure the network port ip and check whether the ip/mac has the
 correct mapping relationship.
5. Query eid to check whether it complies with ip conversion rules:
 cat /sys/class/roh/hns3_0/node_eid
6. Use ping to check Ethernet communication is normal.
7. Use perftest to check RDMA communication is normal.
8. Use the "ethtool --reset eth1 all" command to trigger a reset. 
 
Link:https://gitee.com/openeuler/kernel/pulls/231 
Reviewed-by: Ling Mingqiang <lingmingqiang@huawei.com> 
Reviewed-by: Xie XiuQi <xiexiuqi@huawei.com> 
Reviewed-by: Zheng Zengkai <zhengzengkai@huawei.com> 
Signed-off-by: Zheng Zengkai <zhengzengkai@huawei.com> 

drivers/infiniband/hw/hns/hns_roce_device.h

@@ -955,6 +955,7 @@ struct hns_roce_dev {
 	struct hns_roce_hem_table  gmv_table;
 
 	int			cmd_mod;
+	u8			mac_type;
 	int			loop_idc;
 	u32			sdb_offset;
 	u32			odb_offset;

drivers/infiniband/hw/hns/hns_roce_hw_v2.c

@@ -476,6 +476,7 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 			     void *wqe, unsigned int *sge_idx,
 			     unsigned int owner_bit)
 {
+	struct hns_roce_dev *hr_dev = to_hr_dev(qp->ibqp.device);
 	struct hns_roce_ah *ah = to_hr_ah(ud_wr(wr)->ah);
 	struct hns_roce_v2_ud_send_wqe *ud_sq_wqe = wqe;
 	unsigned int curr_idx = *sge_idx;
@@ -509,6 +510,9 @@ static inline int set_ud_wqe(struct hns_roce_qp *qp,
 	if (ret)
 		return ret;
 
+	if (hr_dev->mac_type == HNAE3_MAC_ROH && qp->ibqp.qp_type == IB_QPT_GSI)
+		ud_sq_wqe->dmac[0] = 0xFF;
+
 	qp->sl = to_hr_ah(ud_wr(wr)->ah)->av.sl;
 
 	set_extend_sge(qp, wr->sg_list, &curr_idx, valid_num_sge);
@@ -2397,6 +2401,28 @@ static int hns_roce_query_pf_caps(struct hns_roce_dev *hr_dev)
 	return 0;
 }
 
+static void hns_roce_set_mac_type(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_cmq_desc desc;
+	int ret;
+
+	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
+		return;
+
+	hns_roce_cmq_setup_basic_desc(&desc, HNS_QUERY_MAC_TYPE, true);
+	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+	if (ret == CMD_NOT_EXIST)
+		return;
+
+	if (ret) {
+		dev_err(hr_dev->dev, "failed to get mac mod, ret = %d.\n", ret);
+		return;
+	}
+
+	if (le32_to_cpu(desc.data[0]))
+		hr_dev->mac_type = HNAE3_MAC_ROH;
+}
+
 static int config_hem_entry_size(struct hns_roce_dev *hr_dev, u32 type, u32 val)
 {
 	struct hns_roce_cmq_desc desc;
@@ -3044,6 +3070,8 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
 	if (ret)
 		return ret;
 
+	hns_roce_set_mac_type(hr_dev);
+
 	ret = get_hem_table(hr_dev);
 	if (ret)
 		return ret;
@@ -3276,6 +3304,8 @@ static int hns_roce_v2_set_gid(struct hns_roce_dev *hr_dev, int gid_index,
 			else
 				sgid_type = GID_TYPE_FLAG_ROCE_V2_IPV6;
 		} else if (attr->gid_type == IB_GID_TYPE_ROCE) {
+			if (hr_dev->mac_type == HNAE3_MAC_ROH)
+				return -EPERM;
 			sgid_type = GID_TYPE_FLAG_ROCE_V1;
 		}
 	}
@@ -6850,6 +6880,11 @@ static const struct pci_device_id hns_roce_hw_v2_pci_tbl[] = {
 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_RDMA), 0},
 	{PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_RDMA_DCB_PFC_VF),
 	 HNAE3_DEV_SUPPORT_ROCE_DCB_BITS},
+
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_ROH), 0 },
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_200G_ROH), 0 },
+	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_400G_ROH), 0 },
+
 	/* required last entry */
 	{0, }
 };

drivers/infiniband/hw/hns/hns_roce_hw_v2.h

@@ -226,6 +226,7 @@ enum {
 /* CMQ command */
 enum hns_roce_opcode_type {
 	HNS_QUERY_FW_VER				= 0x0001,
+	HNS_QUERY_MAC_TYPE				= 0x0389,
 	HNS_ROCE_OPC_QUERY_HW_VER			= 0x8000,
 	HNS_ROCE_OPC_CFG_GLOBAL_PARAM			= 0x8001,
 	HNS_ROCE_OPC_ALLOC_PF_RES			= 0x8004,

drivers/infiniband/hw/hns/hns_roce_main.c

@@ -430,6 +430,7 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
 	}
 
 	resp.cqe_size = hr_dev->caps.cqe_sz;
+	resp.mac_type = hr_dev->mac_type;
 
 	ret = ib_copy_to_udata(udata, &resp,
 			       min(udata->outlen, sizeof(resp)));
@@ -511,9 +512,14 @@ static int hns_roce_port_immutable(struct ib_device *ib_dev, u8 port_num,
 	immutable->gid_tbl_len = attr.gid_tbl_len;
 
 	immutable->max_mad_size = IB_MGMT_MAD_SIZE;
-	immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
-	if (to_hr_dev(ib_dev)->caps.flags & HNS_ROCE_CAP_FLAG_ROCE_V1_V2)
-		immutable->core_cap_flags |= RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+
+	if (to_hr_dev(ib_dev)->mac_type == HNAE3_MAC_ROH)
+		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+	else if (to_hr_dev(ib_dev)->caps.flags & HNS_ROCE_CAP_FLAG_ROCE_V1_V2)
+		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
+					    RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
+	else
+		immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE;
 
 	return 0;
 }

drivers/roh/core/Makefile

@@ -3,5 +3,5 @@
 # Makefile for the Linux kernel ROH Core drivers.
 #
 
-roh_core-objs := main.o core.o
+roh_core-objs := main.o core.o sysfs.o
 obj-$(CONFIG_ROH) += roh_core.o

drivers/roh/core/core.c

@@ -77,6 +77,8 @@ struct roh_device *roh_alloc_device(size_t size)
 
 	mutex_init(&device->unregistration_lock);
 
+	mutex_init(&device->eid_mutex);
+
 	xa_init_flags(&device->client_data, XA_FLAGS_ALLOC);
 	init_rwsem(&device->client_data_rwsem);
 	init_completion(&device->unreg_completion);
@@ -191,6 +193,20 @@ static int assign_name(struct roh_device *device)
 	return ret;
 }
 
+static int setup_device(struct roh_device *device)
+{
+	int ret;
+
+	/* Query GUID */
+	ret = device->ops.query_guid(device, &device->node_guid);
+	if (ret) {
+		pr_err("failed to query guid, ret = %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
 static void disable_device(struct roh_device *device)
 {
 	u32 cid;
@@ -237,6 +253,61 @@ static int enable_device_and_get(struct roh_device *device)
 	return ret;
 }
 
+static int roh_ipv4_event(struct notifier_block *this, unsigned long event, void *ptr)
+{
+	struct roh_eid_attr eid_attr;
+	struct in_ifaddr *ifa = ptr;
+	struct roh_device *device;
+	struct net_device *ndev;
+	struct sockaddr_in in;
+	int ret;
+
+	device = container_of(this, struct roh_device, nb);
+	ndev = ifa->ifa_dev->dev;
+	if (device->netdev != ndev) {
+		pr_warn("netdev mismatch.\n");
+		return NOTIFY_DONE;
+	}
+
+	in.sin_addr.s_addr = ifa->ifa_address;
+
+	eid_attr.base = be32_to_cpu(in.sin_addr.s_addr) & 0xffffff; /* lower 3B as src eid */
+	eid_attr.num = 1;
+	ret = roh_device_set_eid(device, &eid_attr);
+	if (ret) {
+		pr_err("failed to set eid by IP, ret = %d\n", ret);
+		return ret;
+	}
+
+	return NOTIFY_DONE;
+}
+
+static int roh_register_inetaddr_event(struct roh_device *device)
+{
+	int ret;
+
+	device->nb.notifier_call = roh_ipv4_event;
+	ret = register_inetaddr_notifier(&device->nb);
+	if (ret) {
+		pr_err("roh_core: failed to register inetaddr notifier, ret = %d\n", ret);
+		device->nb.notifier_call = NULL;
+	}
+
+	return ret;
+}
+
+static void roh_unregister_inetaddr_event(struct roh_device *device)
+{
+	int ret;
+
+	if (device->nb.notifier_call) {
+		ret = unregister_inetaddr_notifier(&device->nb);
+		if (ret)
+			pr_err("roh_core: failed to unregister inetaddr notifier, ret = %d\n", ret);
+		device->nb.notifier_call = NULL;
+	}
+}
+
 int roh_register_device(struct roh_device *device)
 {
 	int ret;
@@ -247,6 +318,12 @@ int roh_register_device(struct roh_device *device)
 		return ret;
 	}
 
+	ret = setup_device(device);
+	if (ret) {
+		pr_err("roh_core: failed to setup device, ret = %d\n", ret);
+		return ret;
+	}
+
 	dev_set_uevent_suppress(&device->dev, true);
 	ret = device_add(&device->dev);
 	if (ret) {
@@ -254,6 +331,16 @@ int roh_register_device(struct roh_device *device)
 		goto out;
 	}
 
+	ret = roh_device_register_sysfs(device);
+	if (ret)
+		goto err_dev_cleanup;
+
+	ret = roh_register_inetaddr_event(device);
+	if (ret) {
+		pr_err("roh_core: failed to register inetaddr event, ret = %d\n", ret);
+		goto err_unregister_sysfs;
+	}
+
 	ret = enable_device_and_get(device);
 	dev_set_uevent_suppress(&device->dev, false);
 	kobject_uevent(&device->dev.kobj, KOBJ_ADD);
@@ -266,6 +353,11 @@ int roh_register_device(struct roh_device *device)
 	roh_device_put(device);
 
 	return 0;
+
+err_unregister_sysfs:
+	roh_device_unregister_sysfs(device);
+err_dev_cleanup:
+	device_del(&device->dev);
 out:
 	dev_set_uevent_suppress(&device->dev, false);
 	return ret;
@@ -279,6 +371,8 @@ static void __roh_unregister_device(struct roh_device *device)
 		goto out;
 
 	disable_device(device);
+	roh_unregister_inetaddr_event(device);
+	roh_device_unregister_sysfs(device);
 	device_del(&device->dev);
 
 out:
@@ -458,6 +552,115 @@ void roh_unregister_client(struct roh_client *client)
 	remove_client_id(client);
 }
 
+static int roh_set_pf_mac_by_eid(struct roh_device *device,
+				 struct roh_eid_attr *eid_attr)
+{
+	const struct net_device_ops *ndev_ops;
+	u32 eid = eid_attr->base;
+	struct net_device *ndev;
+	struct sockaddr s_addr;
+	u8 mac[ETH_ALEN];
+	int ret;
+
+	ndev = device->netdev;
+	if (!ndev)
+		return -EINVAL;
+
+	ndev_ops = ndev->netdev_ops;
+	if (!ndev_ops->ndo_set_mac_address)
+		return -EOPNOTSUPP;
+
+	convert_eid_to_mac(mac, eid);
+
+	s_addr.sa_family = ndev->type;
+	memcpy(s_addr.sa_data, mac, ndev->addr_len);
+
+	ret = dev_set_mac_address(ndev, &s_addr, NULL);
+	if (ret) {
+		netdev_err(ndev, "failed to set dev %s mac, ret = %d\n",
+			   ndev->name, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+static int roh_set_mac_by_eid(struct roh_device *device,
+			      struct roh_eid_attr *eid_attr)
+{
+	int ret;
+
+	ret = roh_set_pf_mac_by_eid(device, eid_attr);
+	if (ret) {
+		pr_err("failed to set pf mac, ret = %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int roh_device_set_eid(struct roh_device *device, struct roh_eid_attr *attr)
+{
+	int ret;
+
+	if (!device->ops.set_eid)
+		return -EPROTONOSUPPORT;
+
+	mutex_lock(&device->eid_mutex);
+	/* Update current EID */
+	ret = device->ops.set_eid(device, attr);
+	if (ret) {
+		mutex_unlock(&device->eid_mutex);
+		return ret;
+	}
+
+	device->eid = *attr;
+	ret = roh_set_mac_by_eid(device, attr);
+	mutex_unlock(&device->eid_mutex);
+
+	return ret;
+}
+
+void roh_device_get_eid(struct roh_device *device, struct roh_eid_attr *attr)
+{
+	mutex_lock(&device->eid_mutex);
+	memcpy(attr, &device->eid, sizeof(*attr));
+	mutex_unlock(&device->eid_mutex);
+}
+
+void roh_device_query_guid(struct roh_device *device, struct roh_guid_attr *attr)
+{
+	memcpy(attr, &device->node_guid, sizeof(*attr));
+}
+
+enum roh_link_status roh_device_query_link_status(struct roh_device *device)
+{
+	return device->link_status;
+}
+
+static void roh_update_link_status(struct roh_device *device, u32 ls)
+{
+	device->link_status = ls;
+}
+
+void roh_event_notify(struct roh_event *event)
+{
+	struct roh_device *device = event->device;
+
+	switch (event->type) {
+	case ROH_EVENT_LINK_UP:
+		roh_update_link_status(device, ROH_LINK_UP);
+		break;
+	case ROH_EVENT_LINK_DOWN:
+		roh_update_link_status(device, ROH_LINK_DOWN);
+		break;
+	default:
+		pr_err("roh_core: not support event type(%d).\n", event->type);
+		break;
+	}
+}
+EXPORT_SYMBOL(roh_event_notify);
+
 int roh_core_init(void)
 {
 	int ret;

drivers/roh/core/core.h

@@ -19,6 +19,8 @@ enum roh_dev_tx {
 	ROHDEV_TX_LOCKED = 0x20 /* driver tx lock was already taken */
 };
 
+enum roh_link_status { ROH_LINK_DOWN = 0, ROH_LINK_UP };
+
 enum roh_mib_type { ROH_MIB_PUBLIC = 0, ROH_MIB_PRIVATE };
 
 static inline void convert_eid_to_mac(u8 mac[6], u32 eid)
@@ -82,6 +84,26 @@ struct roh_device {
 	refcount_t refcount;
 	struct completion unreg_completion;
 	struct mutex unregistration_lock; /* lock for unregiste */
+
+	struct roh_guid_attr node_guid;
+	struct roh_eid_attr eid;
+	struct mutex eid_mutex; /* operate eid needs to be mutexed */
+	u32 link_status;
+	struct notifier_block nb;
+
+	struct attribute_group *hw_stats_ag;
+	struct roh_mib_stats *hw_public_stats;
+	struct roh_mib_stats *hw_private_stats;
+};
+
+enum roh_event_type {
+	ROH_EVENT_LINK_DOWN = 0,
+	ROH_EVENT_LINK_UP
+};
+
+struct roh_event {
+	struct roh_device *device;
+	enum roh_event_type type;
 };
 
 static inline bool roh_device_try_get(struct roh_device *device)
@@ -97,6 +119,8 @@ void roh_dealloc_device(struct roh_device *device);
 int roh_register_device(struct roh_device *device);
 void roh_unregister_device(struct roh_device *device);
 
+void roh_event_notify(struct roh_event *event);
+
 int roh_core_init(void);
 void roh_core_cleanup(void);
 

drivers/roh/core/core_priv.h

@@ -18,4 +18,13 @@ void roh_unregister_client(struct roh_client *client);
 void roh_set_client_data(struct roh_device *device,
 			 struct roh_client *client, void *data);
 
+int roh_device_register_sysfs(struct roh_device *device);
+void roh_device_unregister_sysfs(struct roh_device *device);
+
+int roh_device_set_eid(struct roh_device *device, struct roh_eid_attr *attr);
+void roh_device_get_eid(struct roh_device *device, struct roh_eid_attr *attr);
+
+void roh_device_query_guid(struct roh_device *device, struct roh_guid_attr *attr);
+enum roh_link_status roh_device_query_link_status(struct roh_device *device);
+
 #endif /* __CORE_PRIV_H__ */

drivers/roh/core/sysfs.c

+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2020-2022 Hisilicon Limited.
+
+#include <linux/sysfs.h>
+#include <linux/stat.h>
+#include <net/sock.h>
+
+#include "core.h"
+#include "core_priv.h"
+
+#define ROH_MIB_STATS_TYPE_NUM 2
+
+static ssize_t node_eid_show(struct device *device,
+			     struct device_attribute *attr, char *buf)
+{
+	struct roh_device *dev = container_of(device, struct roh_device, dev);
+	struct roh_eid_attr eid;
+
+	roh_device_get_eid(dev, &eid);
+
+	return sprintf(buf, "base:%u num:%u\n", eid.base, eid.num);
+}
+
+static ssize_t node_guid_show(struct device *device,
+			      struct device_attribute *attr, char *buf)
+{
+	struct roh_device *dev = container_of(device, struct roh_device, dev);
+	struct roh_guid_attr guid;
+	u32 *val = (u32 *)guid.data;
+
+	roh_device_query_guid(dev, &guid);
+
+	return sprintf(buf, "%08x:%08x:%08x:%08x\n", val[0], val[1], val[2], val[3]);
+}
+
+static ssize_t node_link_status_show(struct device *device,
+				     struct device_attribute *attr, char *buf)
+{
+	struct roh_device *dev = container_of(device, struct roh_device, dev);
+
+	return sprintf(buf, "%s\n",
+		       (roh_device_query_link_status(dev) == ROH_LINK_UP) ?
+		       "UP" : "DOWN");
+}
+
+static DEVICE_ATTR_RO(node_eid);
+static DEVICE_ATTR_RO(node_guid);
+static DEVICE_ATTR_RO(node_link_status);
+
+static struct device_attribute *roh_class_attr[] = {
+	&dev_attr_node_eid,
+	&dev_attr_node_guid,
+	&dev_attr_node_link_status,
+};
+
+struct roh_hw_stats_attribute {
+	struct attribute attr;
+	ssize_t (*show)(struct kobject *kobj,
+			struct attribute *attr, char *buf);
+	ssize_t (*store)(struct kobject *kobj,
+			 struct attribute *attr,
+			 const char *buf,
+			 size_t count);
+};
+
+static void remove_device_sysfs(struct roh_device *device)
+{
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(roh_class_attr); i++)
+		device_remove_file(&device->dev, roh_class_attr[i]);
+}
+
+static const char * const roh_hw_stats_name[ROH_MIB_STATS_TYPE_NUM] = {
+	"mib_public",
+	"mib_private",
+};
+
+static ssize_t print_hw_stat(struct roh_device *dev,
+			     struct roh_mib_stats *stats, char *buf)
+{
+	int offset = 0;
+	int i;
+
+	for (i = 0; i < stats->num_counters; i++)
+		offset += sprintf(buf + offset, "%s: %llu\n",
+				  stats->names[i], stats->value[i]);
+	return offset;
+}
+
+static ssize_t node_public_mib_stats_show(struct kobject *kobj,
+					  struct attribute *attr, char *buf)
+{
+	struct roh_hw_stats_attribute *hsa;
+	struct roh_mib_stats *public_stats;
+	struct roh_device *dev;
+	int ret;
+
+	hsa = container_of(attr, struct roh_hw_stats_attribute, attr);
+	dev = container_of(kobj, struct roh_device, dev.kobj);
+	public_stats = dev->hw_public_stats;
+	mutex_lock(&public_stats->lock);
+	ret = dev->ops.get_hw_stats(dev, public_stats, ROH_MIB_PUBLIC);
+	if (ret)
+		goto unlock;
+
+	ret = print_hw_stat(dev, public_stats, buf);
+
+unlock:
+	mutex_unlock(&public_stats->lock);
+	return ret;
+}
+
+static ssize_t node_private_mib_stats_show(struct kobject *kobj,
+					   struct attribute *attr, char *buf)
+{
+	struct roh_mib_stats *private_stats;
+	struct roh_hw_stats_attribute *hsa;
+	struct roh_device *dev;
+	int ret;
+
+	hsa = container_of(attr, struct roh_hw_stats_attribute, attr);
+	dev = container_of(kobj, struct roh_device, dev.kobj);
+	private_stats = dev->hw_private_stats;
+	mutex_lock(&private_stats->lock);
+	ret = dev->ops.get_hw_stats(dev, private_stats, ROH_MIB_PRIVATE);
+	if (ret)
+		goto unlock;
+
+	ret = print_hw_stat(dev, private_stats, buf);
+
+unlock:
+	mutex_unlock(&private_stats->lock);
+	return ret;
+}
+
+static ssize_t (*show_roh_mib_hw_stats[ROH_MIB_STATS_TYPE_NUM])
+	(struct kobject *, struct attribute *, char *) = {
+	node_public_mib_stats_show,
+	node_private_mib_stats_show
+};
+
+static void free_hsag(struct kobject *kobj, struct attribute_group *attr_group)
+{
+	struct attribute **attr;
+
+	sysfs_remove_group(kobj, attr_group);
+
+	for (attr = attr_group->attrs; *attr; attr++)
+		kfree(*attr);
+	kfree(attr_group);
+}
+
+static struct attribute *alloc_hsa(const char *name,
+				   ssize_t (*show_roh_mib_hw_stats)
+				   (struct kobject *, struct attribute *, char *))
+{
+	struct roh_hw_stats_attribute *hsa;
+
+	hsa = kmalloc(sizeof(*hsa), GFP_KERNEL);
+	if (!hsa)
+		return NULL;
+
+	hsa->attr.name = (char *)name;
+	hsa->attr.mode = 0444;
+	hsa->show = show_roh_mib_hw_stats;
+	hsa->store = NULL;
+	return &hsa->attr;
+}
+
+static void setup_mib_stats(struct roh_device *device)
+{
+	struct roh_mib_stats *privite_stats, *public_stats;
+	struct attribute_group *hsag;
+	struct kobject *kobj;
+	int i, j;
+	int ret;
+
+	public_stats = device->ops.alloc_hw_stats(device, ROH_MIB_PUBLIC);
+	if (!public_stats)
+		return;
+
+	privite_stats = device->ops.alloc_hw_stats(device, ROH_MIB_PRIVATE);
+	if (!privite_stats) {
+		kfree(public_stats);
+		return;
+	}
+
+	hsag = kzalloc(sizeof(*hsag) +
+		       sizeof(void *) * (ARRAY_SIZE(roh_hw_stats_name)),
+		       GFP_KERNEL);
+	if (!hsag)
+		goto err_free_stats;
+
+	ret = device->ops.get_hw_stats(device, public_stats, ROH_MIB_PUBLIC);
+	if (ret)
+		goto err_free_hsag;
+
+	ret = device->ops.get_hw_stats(device, privite_stats, ROH_MIB_PRIVATE);
+	if (ret)
+		goto err_free_hsag;
+
+	hsag->name = "node_mib_stats";
+	hsag->attrs = (void *)hsag + sizeof(*hsag);
+
+	for (i = 0; i < ARRAY_SIZE(roh_hw_stats_name); i++) {
+		hsag->attrs[i] = alloc_hsa(roh_hw_stats_name[i],
+					   show_roh_mib_hw_stats[i]);
+		if (!hsag->attrs[i])
+			goto err;
+		sysfs_attr_init(hsag->attrs[i]);
+	}
+
+	mutex_init(&privite_stats->lock);
+	mutex_init(&public_stats->lock);
+
+	kobj = &device->dev.kobj;
+	ret = sysfs_create_group(kobj, hsag);
+	if (ret)
+		goto err;
+	device->hw_stats_ag = hsag;
+	device->hw_public_stats = public_stats;
+	device->hw_private_stats = privite_stats;
+
+	return;
+
+err:
+	for (j = i - 1; j >= 0; j--)
+		kfree(hsag->attrs[j]);
+err_free_hsag:
+	kfree(hsag);
+err_free_stats:
+	kfree(public_stats);
+	kfree(privite_stats);
+}
+
+int roh_device_register_sysfs(struct roh_device *device)
+{
+	int ret;
+	int i;
+
+	for (i = 0; i < ARRAY_SIZE(roh_class_attr); i++) {
+		ret = device_create_file(&device->dev, roh_class_attr[i]);
+		if (ret) {
+			pr_err("failed to create node %s, ret = %d.\n",
+			       roh_class_attr[i]->attr.name, ret);
+			goto err;
+		}
+	}
+
+	if (device->ops.alloc_hw_stats)
+		setup_mib_stats(device);
+
+	return 0;
+
+err:
+	remove_device_sysfs(device);
+	return ret;
+}
+
+void roh_device_unregister_sysfs(struct roh_device *device)
+{
+	if (device->hw_stats_ag)
+		free_hsag(&device->dev.kobj, device->hw_stats_ag);
+
+	kfree(device->hw_private_stats);
+	kfree(device->hw_public_stats);
+
+	remove_device_sysfs(device);
+}

drivers/roh/hw/hns3/Makefile

@@ -8,5 +8,7 @@ ccflags-y +=  -I $(srctree)/drivers/roh/core
 ccflags-y +=  -I $(srctree)/drivers/roh/hw/hns3
 
 hns-roh-v1-objs := hns3_cmdq.o	\
+		   hns3_verbs.o	\
+		   hns3_intr.o	\
 		   hns3_main.o
 obj-$(CONFIG_ROH_HNS) += hns-roh-v1.o

drivers/roh/hw/hns3/hns3_cmdq.c

@@ -8,6 +8,7 @@
 #include <linux/delay.h>
 
 #include "core.h"
+#include "hns3_device.h"
 #include "hns3_common.h"
 #include "hns3_cmdq.h"
 #include "hns3_reg.h"
@@ -315,3 +316,137 @@ int hns3_roh_cmdq_send(struct hns3_roh_device *hroh_dev, struct hns3_roh_desc *d
 
 	return ret;
 }
+
+int hns3_roh_get_link_status(struct hns3_roh_device *hroh_dev, u32 *link_status)
+{
+	struct hns3_roh_query_link_status_info *req;
+	struct hns3_roh_desc desc;
+	u32 link_val;
+	int ret;
+
+	hns3_roh_cmdq_setup_basic_desc(&desc, HNS3_ROH_OPC_QUERY_PORT_LINK_STATUS, true);
+	ret = hns3_roh_cmdq_send(hroh_dev, &desc, 1);
+	if (ret) {
+		dev_err(hroh_dev->dev, "failed to query link status, ret = %d\n", ret);
+		return ret;
+	}
+
+	req = (struct hns3_roh_query_link_status_info *)desc.data;
+	link_val = le32_to_cpu(req->query_link_status);
+
+	*link_status = link_val ? HNS3_ROH_LINK_STATUS_UP : HNS3_ROH_LINK_STATUS_DOWN;
+
+	return 0;
+}
+
+static void hns3_roh_dispatch_event(struct hns3_roh_device *hroh_dev, enum roh_event_type type)
+{
+	struct roh_event event = {0};
+
+	event.device = &hroh_dev->roh_dev;
+	event.type = type;
+	roh_event_notify(&event);
+}
+
+void hns3_roh_update_link_status(struct hns3_roh_device *hroh_dev)
+{
+	u32 state = HNS3_ROH_LINK_STATUS_DOWN;
+	enum roh_event_type type;
+	int ret;
+
+	if (test_and_set_bit(HNS3_ROH_SW_STATE_LINK_UPDATING, &hroh_dev->state))
+		return;
+
+	ret = hns3_roh_get_link_status(hroh_dev, &state);
+	if (ret) {
+		state = HNS3_ROH_LINK_STATUS_DOWN;
+		clear_bit(HNS3_ROH_SW_STATE_LINK_UPDATING, &hroh_dev->state);
+		return;
+	}
+
+	type = (state == HNS3_ROH_LINK_STATUS_DOWN) ? ROH_EVENT_LINK_DOWN : ROH_EVENT_LINK_UP;
+	hns3_roh_dispatch_event(hroh_dev, type);
+
+	clear_bit(HNS3_ROH_SW_STATE_LINK_UPDATING, &hroh_dev->state);
+}
+
+static void hns3_roh_link_fail_parse(struct hns3_roh_device *hroh_dev,
+				     u8 link_fail_code)
+{
+	switch (link_fail_code) {
+	case HNS3_ROH_LF_REF_CLOCK_LOST:
+		dev_warn(hroh_dev->dev, "reference clock lost!\n");
+		break;
+	case HNS3_ROH_LF_XSFP_TX_DISABLE:
+		dev_warn(hroh_dev->dev, "SFP tx is disabled!\n");
+		break;
+	case HNS3_ROH_LF_XSFP_ABSENT:
+		dev_warn(hroh_dev->dev, "SFP is absent!\n");
+		break;
+	default:
+		break;
+	}
+}
+
+static void hns3_roh_handle_link_change_event(struct hns3_roh_device *hroh_dev,
+					      struct hns3_roh_mbx_vf_to_pf_cmd *req)
+{
+	int link_status = req->msg.subcode;
+
+	hns3_roh_task_schedule(hroh_dev, 0);
+
+	if (link_status == HNS3_ROH_LINK_STATUS_DOWN)
+		hns3_roh_link_fail_parse(hroh_dev, req->msg.data[0]);
+}
+
+static bool hns3_roh_cmd_crq_empty(struct hns3_roh_device *hroh_dev)
+{
+	struct hns3_roh_priv *priv = (struct hns3_roh_priv *)hroh_dev->priv;
+	u32 tail = hns3_roh_read(hroh_dev, HNS3_ROH_RX_CMDQ_TAIL_REG);
+
+	return tail == priv->cmdq.crq.next_to_use;
+}
+
+void hns3_roh_mbx_handler(struct hns3_roh_device *hroh_dev)
+{
+	struct hns3_roh_priv *priv = (struct hns3_roh_priv *)hroh_dev->priv;
+	struct hns3_roh_cmdq_ring *crq = &priv->cmdq.crq;
+	struct hns3_roh_mbx_vf_to_pf_cmd *req;
+	struct hns3_roh_desc *desc;
+	unsigned int flag;
+
+	/* handle all the mailbox requests in the queue */
+	while (!hns3_roh_cmd_crq_empty(hroh_dev)) {
+		desc = &crq->desc[crq->next_to_use];
+		req = (struct hns3_roh_mbx_vf_to_pf_cmd *)desc->data;
+
+		flag = le16_to_cpu(crq->desc[crq->next_to_use].flag);
+		if (unlikely(!hns3_roh_get_bit(flag, HNS3_ROH_CMDQ_RX_OUTVLD_B))) {
+			dev_warn(hroh_dev->dev,
+				 "dropped invalid mbx message, code = %u\n",
+				 req->msg.code);
+
+			/* dropping/not processing this invalid message */
+			crq->desc[crq->next_to_use].flag = 0;
+			hns3_roh_mbx_ring_ptr_move_crq(crq);
+			continue;
+		}
+
+		switch (req->msg.code) {
+		case HNS3_ROH_MBX_PUSH_LINK_STATUS:
+			hns3_roh_handle_link_change_event(hroh_dev, req);
+			break;
+		default:
+			dev_err(hroh_dev->dev,
+				"un-supported mbx message, code = %u\n",
+				req->msg.code);
+			break;
+		}
+
+		crq->desc[crq->next_to_use].flag = 0;
+		hns3_roh_mbx_ring_ptr_move_crq(crq);
+	}
+
+	/* write back CMDQ_RQ header ptr, M7 need this ptr */
+	hns3_roh_write(hroh_dev, HNS3_ROH_RX_CMDQ_HEAD_REG, crq->next_to_use);
+}

drivers/roh/hw/hns3/hns3_cmdq.h

@@ -26,6 +26,7 @@ enum { HNS3_ROH_CMDQ_CRQ = 0, HNS3_ROH_CMDQ_CSQ };
 
 enum hns3_roh_opcode_type {
 	HNS3_ROH_OPC_GET_INTR_INFO = 0x0023,
+	HNS3_ROH_OPC_QUERY_PORT_LINK_STATUS = 0x038a,
 	HNS3_ROH_OPC_SET_EID = 0x9001,
 	HNS3_ROH_OPC_GET_GUID = 0x9002,
 	HNS3_ROH_OPC_QUERY_MIB_PUBLIC = 0x9005,
@@ -44,6 +45,24 @@ enum hns3_roh_cmd_return_status {
 	HNS3_ROH_CMD_EXEC_TIMEOUT
 };
 
+enum hns3_roh_mbx_opcode {
+	HNS3_ROH_MBX_PUSH_LINK_STATUS = 201 /* (M7 -> PF) get port link status */
+};
+
+struct hns3_roh_get_intr_info {
+	__le16 tqp_num;
+	__le16 packet_buffer_cell_cnt;
+	__le16 msixcap_localid_ba_nic;
+	__le16 msixcap_localid_number_nic;
+	__le16 pf_intr_vector_number_roce;
+	__le16 pf_own_fun_number;
+	__le16 tx_pkt_buffer_cell_cnt;
+	__le16 delay_value_cell_num;
+	__le16 tqp_number_1k;
+	__le16 pf_intr_vector_number_roh;
+	u8 rsv[4];
+};
+
 struct hns3_roh_set_eid_info {
 	__le32 base_eid;
 	__le32 num_eid;
@@ -55,6 +74,31 @@ struct hns3_roh_get_guid_info {
 	u8 rsv[8];
 };
 
+struct hns3_roh_query_link_status_info {
+	__le32 query_link_status;
+	u8 rsv[20];
+};
+
+#define HNS3_ROH_MBX_MAX_MSG_SIZE 14
+
+struct hns3_roh_vf_to_pf_msg {
+	u8 code;
+	struct {
+		u8 subcode;
+		u8 data[HNS3_ROH_MBX_MAX_MSG_SIZE];
+	};
+};
+
+struct hns3_roh_mbx_vf_to_pf_cmd {
+	u8 rsv;
+	u8 mbx_src_vfid; /* Auto filled by IMP */
+	u8 mbx_need_resp;
+	u8 rsv1;
+	u8 msg_len;
+	u8 rsv2[3];
+	struct hns3_roh_vf_to_pf_msg msg;
+};
+
 static inline void hns3_roh_mbx_ring_ptr_move_crq(struct hns3_roh_cmdq_ring *crq)
 {
 	crq->next_to_use = (crq->next_to_use + 1) % crq->desc_num;
@@ -66,5 +110,7 @@ int hns3_roh_cmdq_send(struct hns3_roh_device *hroh_dev,
 		       struct hns3_roh_desc *desc, int num);
 void hns3_roh_cmdq_setup_basic_desc(struct hns3_roh_desc *desc,
 				    enum hns3_roh_opcode_type opcode, bool is_read);
+int hns3_roh_get_link_status(struct hns3_roh_device *hroh_dev, u32 *link_status);
+void hns3_roh_update_link_status(struct hns3_roh_device *hroh_dev);
 
 #endif /* __HNS3_ROH_CMDQ_H__ */

drivers/roh/hw/hns3/hns3_common.h

@@ -8,10 +8,18 @@
 
 #define HNS3_ROH_VERSION "1.0"
 
+#define HNS3_ROH_MIN_VECTOR_NUM 2
+
 #define HNS3_ROH_NAME "roh"
+#define HNS3_ROH_INT_NAME_LEN 32
 
 #define HNS3_ROH_DESC_DATA_LEN 6
 
+#define HNS3_ROH_RD_FIRST_STATS_NUM 3
+#define HNS3_ROH_RD_OTHER_STATS_NUM 4
+
+#define HNS3_ROH_HW_RST_UNINT_DELAY 100
+
 struct hns3_roh_desc {
 	__le16 opcode;
 
@@ -51,6 +59,18 @@ struct hns3_roh_priv {
 	unsigned long state;
 };
 
+struct hns3_roh_intr_info {
+	u16 base_vecotr;
+	u16 vector_offset;
+	u16 vector_num;
+};
+
+struct hns3_roh_abn_vector {
+	u8 __iomem *addr;
+	int vector_irq;
+	char name[HNS3_ROH_INT_NAME_LEN];
+};
+
 struct hns3_roh_device {
 	struct roh_device roh_dev;
 	struct pci_dev *pdev;
@@ -61,11 +81,19 @@ struct hns3_roh_device {
 	u8 __iomem *reg_base;
 	const struct hns3_roh_hw *hw;
 	struct hns3_roh_priv *priv;
+
+	struct hns3_roh_intr_info intr_info;
+	struct hns3_roh_abn_vector abn_vector;
+	unsigned long last_processed;
+	unsigned long state;
+	struct delayed_work srv_task;
+	struct dentry *dfx_debugfs;
 };
 
 struct hns3_roh_hw {
 	int (*cmdq_init)(struct hns3_roh_device *hroh_dev);
 	void (*cmdq_exit)(struct hns3_roh_device *hroh_dev);
+	int (*get_intr_cap)(struct hns3_roh_device *hroh_dev);
 };
 
 static inline struct hns3_roh_device *to_hroh_dev(struct roh_device *rohdev)
@@ -85,4 +113,8 @@ static inline struct hns3_roh_device *to_hroh_dev(struct roh_device *rohdev)
 #define hns3_roh_get_bit(origin, shift) \
 	hns3_roh_get_field(origin, 0x1 << (shift), shift)
 
+void hns3_roh_task_schedule(struct hns3_roh_device *hroh_dev,
+			    unsigned long delay_time);
+void hns3_roh_mbx_handler(struct hns3_roh_device *hroh_dev);
+void hns3_roh_mbx_task_schedule(struct hns3_roh_device *hroh_dev);
 #endif

drivers/roh/hw/hns3/hns3_device.h

+/* SPDX-License-Identifier: GPL-2.0+ */
+// Copyright (c) 2022 Hisilicon Limited.
+
+#ifndef __HNS3_ROH_DEVICE_H__
+#define __HNS3_ROH_DEVICE_H__
+
+enum hns3_roh_state {
+	HNS3_ROH_STATE_RESETTING,
+	HNS3_ROH_STATE_INIT,
+	HNS3_ROH_STATE_INITED,
+	HNS3_ROH_STATE_DOWN
+};
+
+enum { HNS3_ROH_RST_DIRECT_RETURN = 0 };
+
+enum hns3_roh_link_type {
+	HNS3_ROH_LINK_STATUS_DOWN = 0,
+	HNS3_ROH_LINK_STATUS_UP
+};
+
+enum hns3_roh_dev_sw_state {
+	HNS3_ROH_SW_STATE_MBX_SERVICE_SCHED,
+	HNS3_ROH_SW_STATE_MBX_HANDLING,
+	HNS3_ROH_SW_STATE_LINK_UPDATING,
+	HNS3_ROH_SW_STATE_MAX
+};
+
+enum hns3_roh_event_type {
+	HNS3_ROH_VECTOR0_EVENT_MBX,
+	HNS3_ROH_VECTOR0_EVENT_OTHER
+};
+
+enum hns3_roh_link_fail_code {
+	HNS3_ROH_LF_NORMAL = 0,
+	HNS3_ROH_LF_REF_CLOCK_LOST = 1,
+	HNS3_ROH_LF_XSFP_TX_DISABLE = 2,
+	HNS3_ROH_LF_XSFP_ABSENT = 3
+};
+
+#endif

drivers/roh/hw/hns3/hns3_intr.c

+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2020-2022 Hisilicon Limited.
+
+#include <linux/interrupt.h>
+#include <linux/pci.h>
+
+#include "hns3_device.h"
+#include "hns3_reg.h"
+#include "hns3_intr.h"
+
+static u32 hns3_roh_parse_event_type(struct hns3_roh_device *hroh_dev, u32 *clear_val)
+{
+	u32 cmdq_src_reg;
+	u32 event_type;
+
+	cmdq_src_reg = hns3_roh_read(hroh_dev, HNS3_ROH_VECTOR0_CMDQ_SRC_REG);
+	if (BIT(HNS3_ROH_VECTOR0_RX_CMDQ_INT_B) & cmdq_src_reg)
+		event_type = HNS3_ROH_VECTOR0_EVENT_MBX;
+	else
+		event_type = HNS3_ROH_VECTOR0_EVENT_OTHER;
+
+	*clear_val = cmdq_src_reg;
+
+	return event_type;
+}
+
+static void hns3_roh_clear_event_type(struct hns3_roh_device *hroh_dev,
+				      u32 event_type, u32 val)
+{
+	switch (event_type) {
+	case HNS3_ROH_VECTOR0_EVENT_MBX:
+		hns3_roh_write(hroh_dev, HNS3_ROH_VECTOR0_CMDQ_SRC_REG, val);
+		break;
+	default:
+		break;
+	}
+}
+
+void hns3_roh_enable_vector(struct hns3_roh_abn_vector *vector, bool enable)
+{
+	writel(enable ? 1 : 0, vector->addr);
+}
+
+static irqreturn_t hns3_roh_abn_irq_handle(int irq, void *data)
+{
+	struct hns3_roh_device *hroh_dev = data;
+	irqreturn_t result;
+	u32 clear_val = 0;
+	u32 event_type;
+
+	hns3_roh_enable_vector(&hroh_dev->abn_vector, false);
+
+	event_type = hns3_roh_parse_event_type(hroh_dev, &clear_val);
+	switch (event_type) {
+	case HNS3_ROH_VECTOR0_EVENT_MBX:
+		/* If we are here then,
+		 * 1. Either we are not handling any mbx task and we are not
+		 *    scheduled as well
+		 *                        OR
+		 * 2. We could be handling a mbx task but nothing more is
+		 *    scheduled.
+		 * In both cases, we should schedule mbx task as there are more
+		 * mbx messages reported by this interrupt.
+		 */
+		hns3_roh_mbx_task_schedule(hroh_dev);
+
+		result = IRQ_HANDLED;
+		break;
+	default:
+		dev_warn(hroh_dev->dev, "unknown event type, type = %u\n",
+			 event_type);
+		result = IRQ_NONE;
+		break;
+	}
+
+	hns3_roh_clear_event_type(hroh_dev, event_type, clear_val);
+
+	if (!clear_val || event_type == HNS3_ROH_VECTOR0_EVENT_MBX)
+		hns3_roh_enable_vector(&hroh_dev->abn_vector, true);
+
+	return result;
+}
+
+static void hns3_roh_abn_irq_uninit(struct hns3_roh_device *hroh_dev)
+{
+	struct hns3_roh_abn_vector *abn_vector;
+
+	abn_vector = &hroh_dev->abn_vector;
+	free_irq(abn_vector->vector_irq, hroh_dev);
+}
+
+void hns3_roh_uninit_irq(struct hns3_roh_device *hroh_dev)
+{
+	hns3_roh_abn_irq_uninit(hroh_dev);
+}
+
+static int hns3_roh_abn_irq_init(struct hns3_roh_device *hroh_dev)
+{
+	struct hns3_roh_abn_vector *abn_vector = &hroh_dev->abn_vector;
+	int vector_index = hroh_dev->intr_info.vector_offset;
+	int ret;
+
+	abn_vector->vector_irq = pci_irq_vector(hroh_dev->pdev, vector_index);
+	abn_vector->addr = hroh_dev->reg_base + HNS3_ROH_VECTOR0_INT_CTRL_REG;
+
+	ret = snprintf(abn_vector->name, HNS3_ROH_INT_NAME_LEN, "%s-%s-abn",
+		       HNS3_ROH_NAME, pci_name(hroh_dev->pdev));
+	if (ret >= HNS3_ROH_INT_NAME_LEN || ret < 0) {
+		dev_err(hroh_dev->dev, "abn vector name is too long.\n");
+		return -EINVAL;
+	}
+
+	ret = request_irq(abn_vector->vector_irq, hns3_roh_abn_irq_handle, 0,
+			  abn_vector->name, hroh_dev);
+	if (ret) {
+		dev_err(hroh_dev->dev,
+			"failed to request abn irq: %d, ret = %d\n",
+			abn_vector->vector_irq, ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int hns3_roh_init_irq(struct hns3_roh_device *hroh_dev)
+{
+	int ret;
+
+	ret = hns3_roh_abn_irq_init(hroh_dev);
+	if (ret) {
+		dev_err(hroh_dev->dev, "failed to init abn irq, ret = %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}

drivers/roh/hw/hns3/hns3_intr.h

+/* SPDX-License-Identifier: GPL-2.0+ */
+// Copyright (c) 2020-2022 Hisilicon Limited.
+
+#ifndef __HNS3_ROH_INTR_H__
+#define __HNS3_ROH_INTR_H__
+
+#include "hns3_common.h"
+
+void hns3_roh_enable_vector(struct hns3_roh_abn_vector *vector, bool enable);
+int hns3_roh_init_irq(struct hns3_roh_device *hroh_dev);
+void hns3_roh_uninit_irq(struct hns3_roh_device *hroh_dev);
+
+#endif /* __HNS3_ROH_INTR_H__ */

drivers/roh/hw/hns3/hns3_main.c

@@ -4,11 +4,17 @@
 #include <linux/acpi.h>
 #include <linux/module.h>
 #include <linux/pci.h>
-
+#include <linux/debugfs.h>
 #include "core.h"
 #include "hnae3.h"
+#include "hns3_device.h"
 #include "hns3_common.h"
 #include "hns3_cmdq.h"
+#include "hns3_verbs.h"
+#include "hns3_intr.h"
+
+static struct workqueue_struct *hns3_roh_wq;
+static struct dentry *hns3_roh_dfx_root;
 
 static const struct pci_device_id hns3_roh_pci_tbl[] = {
 	{ PCI_VDEVICE(HUAWEI, HNAE3_DEV_ID_100G_ROH), 0 },
@@ -20,6 +26,37 @@ static const struct pci_device_id hns3_roh_pci_tbl[] = {
 };
 MODULE_DEVICE_TABLE(pci, hns3_roh_pci_tbl);
 
+static int hns3_roh_get_intr_cap(struct hns3_roh_device *hroh_dev)
+{
+	struct hns3_roh_get_intr_info *resp;
+	struct hns3_roh_desc desc;
+	int ret;
+
+	hns3_roh_cmdq_setup_basic_desc(&desc, HNS3_ROH_OPC_GET_INTR_INFO, true);
+
+	ret = hns3_roh_cmdq_send(hroh_dev, &desc, 1);
+	if (ret) {
+		dev_err(hroh_dev->dev, "failed to get intr info, ret = %d\n", ret);
+		return ret;
+	}
+
+	resp = (struct hns3_roh_get_intr_info *)desc.data;
+
+	hroh_dev->intr_info.vector_offset =
+		le16_to_cpu(resp->msixcap_localid_number_nic) +
+		le16_to_cpu(resp->pf_intr_vector_number_roce);
+	hroh_dev->intr_info.vector_num =
+		le16_to_cpu(resp->pf_intr_vector_number_roh);
+	if (hroh_dev->intr_info.vector_num < HNS3_ROH_MIN_VECTOR_NUM) {
+		dev_err(hroh_dev->dev,
+			"just %d intr resources, not enough(min: %d).\n",
+			hroh_dev->intr_info.vector_num, HNS3_ROH_MIN_VECTOR_NUM);
+		return -EINVAL;
+	}
+
+	return 0;
+}
+
 static void hns3_roh_unregister_device(struct hns3_roh_device *hroh_dev)
 {
 	hroh_dev->active = false;
@@ -39,6 +76,17 @@ static int hns3_roh_register_device(struct hns3_roh_device *hroh_dev)
 	rohdev->dev.parent = dev;
 	rohdev->netdev = hroh_dev->netdev;
 
+	rohdev->ops.set_eid = hns3_roh_set_eid;
+	rohdev->ops.query_guid = hns3_roh_query_guid;
+	rohdev->ops.alloc_hw_stats = hns3_roh_alloc_hw_stats;
+	rohdev->ops.get_hw_stats = hns3_roh_get_hw_stats;
+
+	ret = hns3_roh_get_link_status(hroh_dev, &rohdev->link_status);
+	if (ret) {
+		dev_err(dev, "failed to get link status, ret = %d\n", ret);
+		return ret;
+	}
+
 	ret = roh_register_device(rohdev);
 	if (ret) {
 		dev_err(dev, "failed to register roh device, ret = %d\n", ret);
@@ -50,6 +98,65 @@ static int hns3_roh_register_device(struct hns3_roh_device *hroh_dev)
 	return 0;
 }
 
+void hns3_roh_mbx_task_schedule(struct hns3_roh_device *hroh_dev)
+{
+	if (!test_and_set_bit(HNS3_ROH_SW_STATE_MBX_SERVICE_SCHED, &hroh_dev->state))
+		mod_delayed_work(hns3_roh_wq, &hroh_dev->srv_task, 0);
+}
+
+void hns3_roh_task_schedule(struct hns3_roh_device *hroh_dev, unsigned long delay_time)
+{
+	mod_delayed_work(hns3_roh_wq, &hroh_dev->srv_task, delay_time);
+}
+
+static void hns3_roh_mbx_service_task(struct hns3_roh_device *hroh_dev)
+{
+	if (!test_and_clear_bit(HNS3_ROH_SW_STATE_MBX_SERVICE_SCHED,
+				&hroh_dev->state) ||
+		test_and_set_bit(HNS3_ROH_SW_STATE_MBX_HANDLING, &hroh_dev->state))
+		return;
+
+	hns3_roh_mbx_handler(hroh_dev);
+
+	clear_bit(HNS3_ROH_SW_STATE_MBX_HANDLING, &hroh_dev->state);
+}
+
+static void hns3_roh_poll_service_task(struct hns3_roh_device *hroh_dev)
+{
+	unsigned long delta = round_jiffies_relative(HZ);
+
+	hns3_roh_update_link_status(hroh_dev);
+
+	if (time_is_after_jiffies(hroh_dev->last_processed + HZ)) {
+		delta = jiffies - hroh_dev->last_processed;
+		if (delta < round_jiffies_relative(HZ)) {
+			delta = round_jiffies_relative(HZ) - delta;
+			goto out;
+		}
+	}
+
+	hroh_dev->last_processed = jiffies;
+
+out:
+	hns3_roh_task_schedule(hroh_dev, delta);
+}
+
+static void hns3_roh_service_task(struct work_struct *work)
+{
+	struct hns3_roh_device *hroh_dev =
+		container_of(work, struct hns3_roh_device, srv_task.work);
+
+	hns3_roh_mbx_service_task(hroh_dev);
+
+	hns3_roh_poll_service_task(hroh_dev);
+}
+
+static void hns3_roh_dev_sw_state_init(struct hns3_roh_device *hroh_dev)
+{
+	clear_bit(HNS3_ROH_SW_STATE_MBX_SERVICE_SCHED, &hroh_dev->state);
+	clear_bit(HNS3_ROH_SW_STATE_MBX_HANDLING, &hroh_dev->state);
+}
+
 static int hns3_roh_init_hw(struct hns3_roh_device *hroh_dev)
 {
 	struct device *dev = hroh_dev->dev;
@@ -61,11 +168,29 @@ static int hns3_roh_init_hw(struct hns3_roh_device *hroh_dev)
 		return ret;
 	}
 
+	ret = hroh_dev->hw->get_intr_cap(hroh_dev);
+	if (ret) {
+		dev_err(dev, "failed to get intr cap, ret = %d\n", ret);
+		goto err_free_cmdq;
+	}
+
+	ret = hns3_roh_init_irq(hroh_dev);
+	if (ret) {
+		dev_err(dev, "failed to init irq, ret = %d\n", ret);
+		goto err_free_cmdq;
+	}
+
 	return 0;
+
+err_free_cmdq:
+	hroh_dev->hw->cmdq_exit(hroh_dev);
+	return ret;
 }
 
 static void hns3_roh_uninit_hw(struct hns3_roh_device *hroh_dev)
 {
+	hns3_roh_uninit_irq(hroh_dev);
+
 	hroh_dev->hw->cmdq_exit(hroh_dev);
 }
 
@@ -86,6 +211,14 @@ static int hns3_roh_init(struct hns3_roh_device *hroh_dev)
 		goto err_uninit_hw;
 	}
 
+	INIT_DELAYED_WORK(&hroh_dev->srv_task, hns3_roh_service_task);
+
+	hns3_roh_enable_vector(&hroh_dev->abn_vector, true);
+
+	hns3_roh_dev_sw_state_init(hroh_dev);
+
+	hns3_roh_task_schedule(hroh_dev, round_jiffies_relative(HZ));
+
 	dev_info(dev, "%s driver init success.\n", HNS3_ROH_NAME);
 
 	return 0;
@@ -97,6 +230,8 @@ static int hns3_roh_init(struct hns3_roh_device *hroh_dev)
 
 static void hns3_roh_exit(struct hns3_roh_device *hroh_dev)
 {
+	cancel_delayed_work_sync(&hroh_dev->srv_task);
+
 	hns3_roh_unregister_device(hroh_dev);
 
 	hns3_roh_uninit_hw(hroh_dev);
@@ -108,6 +243,7 @@ static void hns3_roh_exit(struct hns3_roh_device *hroh_dev)
 static const struct hns3_roh_hw hns3_roh_hw = {
 	.cmdq_init = hns3_roh_cmdq_init,
 	.cmdq_exit = hns3_roh_cmdq_exit,
+	.get_intr_cap = hns3_roh_get_intr_cap,
 };
 
 static void hns3_roh_get_cfg_from_frame(struct hns3_roh_device *hroh_dev,
@@ -124,6 +260,7 @@ static void hns3_roh_get_cfg_from_frame(struct hns3_roh_device *hroh_dev,
 	hroh_dev->priv->handle = handle;
 }
 
+static void hns3_roh_dfx_init(struct hns3_roh_device *hroh_dev);
 static int __hns3_roh_init_instance(struct hnae3_handle *handle)
 {
 	struct hns3_roh_device *hroh_dev;
@@ -148,8 +285,13 @@ static int __hns3_roh_init_instance(struct hnae3_handle *handle)
 		dev_err(hroh_dev->dev, "failed to init roh, ret = %d\n", ret);
 		goto err_kzalloc;
 	}
+
 	handle->priv = hroh_dev;
 
+	set_bit(HNS3_ROH_STATE_INITED, &handle->rohinfo.reset_state);
+
+	hns3_roh_dfx_init(hroh_dev);
+
 	return 0;
 
 err_kzalloc:
@@ -159,6 +301,7 @@ static int __hns3_roh_init_instance(struct hnae3_handle *handle)
 	return ret;
 }
 
+static void hns3_roh_dfx_uninit(struct hns3_roh_device *hroh_dev);
 static void __hns3_roh_uninit_instance(struct hnae3_handle *handle)
 {
 	struct hns3_roh_device *hroh_dev = (struct hns3_roh_device *)handle->priv;
@@ -166,6 +309,13 @@ static void __hns3_roh_uninit_instance(struct hnae3_handle *handle)
 	if (!hroh_dev)
 		return;
 
+	hns3_roh_dfx_uninit(hroh_dev);
+
+	if (!test_and_clear_bit(HNS3_ROH_STATE_INITED, &handle->rohinfo.reset_state))
+		netdev_warn(hroh_dev->netdev, "already uninitialized\n");
+
+	hns3_roh_enable_vector(&hroh_dev->abn_vector, false);
+
 	handle->priv = NULL;
 
 	hns3_roh_exit(hroh_dev);
@@ -206,9 +356,52 @@ static void hns3_roh_uninit_instance(struct hnae3_handle *handle, bool reset)
 	__hns3_roh_uninit_instance(handle);
 }
 
+static int hns3_roh_reset_notify_init(struct hnae3_handle *handle)
+{
+	struct device *dev = &handle->pdev->dev;
+	int ret;
+
+	ret = __hns3_roh_init_instance(handle);
+	if (ret) {
+		dev_err(dev, "failed to reinit in roh reset process, ret = %d\n", ret);
+		handle->priv = NULL;
+		clear_bit(HNS3_ROH_STATE_INITED, &handle->rohinfo.reset_state);
+	}
+
+	return 0;
+}
+
+static int hns3_roh_reset_notify_uninit(struct hnae3_handle *handle)
+{
+	msleep(HNS3_ROH_HW_RST_UNINT_DELAY);
+	__hns3_roh_uninit_instance(handle);
+
+	return 0;
+}
+
+static int hns3_roh_reset_notify(struct hnae3_handle *handle,
+				 enum hnae3_reset_notify_type type)
+{
+	int ret = 0;
+
+	switch (type) {
+	case HNAE3_INIT_CLIENT:
+		ret = hns3_roh_reset_notify_init(handle);
+		break;
+	case HNAE3_UNINIT_CLIENT:
+		ret = hns3_roh_reset_notify_uninit(handle);
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
 static const struct hnae3_client_ops hns3_roh_ops = {
 	.init_instance = hns3_roh_init_instance,
 	.uninit_instance = hns3_roh_uninit_instance,
+	.reset_notify = hns3_roh_reset_notify,
 };
 
 static struct hnae3_client hns3_roh_client = {
@@ -217,14 +410,223 @@ static struct hnae3_client hns3_roh_client = {
 	.ops = &hns3_roh_ops,
 };
 
+static ssize_t hns3_roh_dfx_cmd_read(struct file *filp, char __user *buffer,
+				     size_t count, loff_t *pos)
+{
+#define HNS3_ROH_DFX_READ_LEN 256
+	int uncopy_bytes;
+	char *buf;
+	int len;
+
+	if (*pos != 0)
+		return 0;
+
+	if (count < HNS3_ROH_DFX_READ_LEN)
+		return -ENOSPC;
+
+	buf = kzalloc(HNS3_ROH_DFX_READ_LEN, GFP_KERNEL);
+	if (!buf)
+		return -ENOMEM;
+
+	len = scnprintf(buf, HNS3_ROH_DFX_READ_LEN, "%s\n", "echo help to cmd to get help info");
+	uncopy_bytes = copy_to_user(buffer, buf, len);
+
+	kfree(buf);
+
+	if (uncopy_bytes)
+		return -EFAULT;
+
+	return (*pos = len);
+}
+
+static void hns3_roh_dfx_help(struct hns3_roh_device *hroh_dev)
+{
+	dev_info(hroh_dev->dev, "dev info\n");
+}
+
+static void hns3_roh_dfx_get_vector_cap(struct hns3_roh_device *hroh_dev)
+{
+	u16 roce_vector_num, nic_vector_num, roh_vector_num;
+	struct hns3_roh_get_intr_info *resp;
+	struct device *dev = hroh_dev->dev;
+	struct hns3_roh_desc desc;
+	int ret;
+
+	hns3_roh_cmdq_setup_basic_desc(&desc, HNS3_ROH_OPC_GET_INTR_INFO, true);
+
+	ret = hns3_roh_cmdq_send(hroh_dev, &desc, 1);
+	if (ret)
+		dev_warn(hroh_dev->dev, "failed to get intr info, ret = %d\n", ret);
+
+	resp = (struct hns3_roh_get_intr_info *)desc.data;
+
+	nic_vector_num = le16_to_cpu(resp->msixcap_localid_number_nic);
+	roce_vector_num = le16_to_cpu(resp->pf_intr_vector_number_roce);
+	roh_vector_num = le16_to_cpu(resp->pf_intr_vector_number_roh);
+
+	dev_info(dev, "NIC vector num: %d\n", nic_vector_num);
+	dev_info(dev, "RoCE vector num: %d\n", roce_vector_num);
+	dev_info(dev, "ROH vector num: %d\n", roh_vector_num);
+	dev_info(dev, "ROH vector offset: %d\n", hroh_dev->intr_info.vector_offset);
+}
+
+static void hns3_roh_dfx_dump_dev_info(struct hns3_roh_device *hroh_dev)
+{
+	struct device *dev = hroh_dev->dev;
+
+	dev_info(dev, "PCIe device id: 0x%x\n", hroh_dev->pdev->device);
+	dev_info(dev, "PCIe device name: %s\n", pci_name(hroh_dev->pdev));
+	dev_info(dev, "Network device name: %s\n", netdev_name(hroh_dev->netdev));
+	dev_info(dev, "BAR2~3 base addr: 0x%llx\n", (u64)hroh_dev->reg_base);
+
+	dev_info(dev, "Base vector: %d\n", hroh_dev->intr_info.base_vecotr);
+	hns3_roh_dfx_get_vector_cap(hroh_dev);
+
+	dev_info(dev, "ABN vector0 irq: %d\n", hroh_dev->abn_vector.vector_irq);
+	dev_info(dev, "ABN vector0 addr: 0x%llx\n", (u64)hroh_dev->abn_vector.addr);
+	dev_info(dev, "ABN vector0 name: %s\n", hroh_dev->abn_vector.name);
+}
+
+static int hns3_roh_dfx_check_cmd(struct hns3_roh_device *hroh_dev, char *cmd_buf)
+{
+	int ret = 0;
+
+	if (strncmp(cmd_buf, "help", strlen("help")) == 0)
+		hns3_roh_dfx_help(hroh_dev);
+	else if (strncmp(cmd_buf, "dev info", strlen("dev info")) == 0)
+		hns3_roh_dfx_dump_dev_info(hroh_dev);
+	else
+		ret = -EOPNOTSUPP;
+	return ret;
+}
+
+static ssize_t hns3_roh_dfx_cmd_write(struct file *filp, const char __user *buffer,
+				      size_t count, loff_t *pos)
+{
+#define HNS3_ROH_DFX_WRITE_LEN 1024
+	struct hns3_roh_device *hroh_dev = filp->private_data;
+	char *cmd_buf, *cmd_buf_tmp;
+	int uncopied_bytes;
+	int ret;
+
+	if (*pos != 0)
+		return 0;
+
+	if (count > HNS3_ROH_DFX_WRITE_LEN)
+		return -ENOSPC;
+
+	cmd_buf = kzalloc(count + 1, GFP_KERNEL);
+	if (!cmd_buf)
+		return count;
+
+	uncopied_bytes = copy_from_user(cmd_buf, buffer, count);
+	if (uncopied_bytes) {
+		kfree(cmd_buf);
+		return -EFAULT;
+	}
+
+	cmd_buf[count] = '\0';
+
+	cmd_buf_tmp = strchr(cmd_buf, '\n');
+	if (cmd_buf_tmp) {
+		*cmd_buf_tmp = '\0';
+		count = cmd_buf_tmp - cmd_buf + 1;
+	}
+
+	ret = hns3_roh_dfx_check_cmd(hroh_dev, cmd_buf);
+	if (ret)
+		hns3_roh_dfx_help(hroh_dev);
+
+	kfree(cmd_buf);
+	cmd_buf = NULL;
+
+	return count;
+}
+
+static const struct file_operations hns3_roh_dfx_cmd_fops = {
+	.owner = THIS_MODULE,
+	.open  = simple_open,
+	.read  = hns3_roh_dfx_cmd_read,
+	.write = hns3_roh_dfx_cmd_write,
+};
+
+static void hns3_roh_dfx_init(struct hns3_roh_device *hroh_dev)
+{
+	const char *name = pci_name(hroh_dev->pdev);
+	struct dentry *entry;
+
+	if (IS_ERR_OR_NULL(hns3_roh_dfx_root))
+		return;
+
+	hroh_dev->dfx_debugfs = debugfs_create_dir(name, hns3_roh_dfx_root);
+	if (IS_ERR_OR_NULL(hroh_dev->dfx_debugfs))
+		return;
+
+	entry = debugfs_create_file("hns3_roh_dfx", 0600, hroh_dev->dfx_debugfs,
+				    hroh_dev, &hns3_roh_dfx_cmd_fops);
+	if (IS_ERR_OR_NULL(entry)) {
+		debugfs_remove_recursive(hroh_dev->dfx_debugfs);
+		hroh_dev->dfx_debugfs = NULL;
+		return;
+	}
+}
+
+static void hns3_roh_dfx_uninit(struct hns3_roh_device *hroh_dev)
+{
+	if (IS_ERR_OR_NULL(hroh_dev->dfx_debugfs))
+		return;
+
+	debugfs_remove_recursive(hroh_dev->dfx_debugfs);
+	hroh_dev->dfx_debugfs = NULL;
+}
+
+static void hns3_roh_dfx_register_debugfs(const char *dir_name)
+{
+	hns3_roh_dfx_root = debugfs_create_dir(dir_name, NULL);
+	if (IS_ERR_OR_NULL(hns3_roh_dfx_root))
+		return;
+}
+
+static void hns3_roh_dfx_unregister_debugfs(void)
+{
+	if (IS_ERR_OR_NULL(hns3_roh_dfx_root))
+		return;
+
+	debugfs_remove_recursive(hns3_roh_dfx_root);
+	hns3_roh_dfx_root = NULL;
+}
+
 static int __init hns3_roh_module_init(void)
 {
-	return hnae3_register_client(&hns3_roh_client);
+	int ret;
+
+	hns3_roh_wq = alloc_workqueue("%s", 0, 0, HNS3_ROH_NAME);
+	if (!hns3_roh_wq) {
+		pr_err("%s: failed to create wq.\n", HNS3_ROH_NAME);
+		return -ENOMEM;
+	}
+
+	hns3_roh_dfx_register_debugfs(HNS3_ROH_NAME);
+
+	ret = hnae3_register_client(&hns3_roh_client);
+	if (ret)
+		goto out;
+
+	return 0;
+
+out:
+	hns3_roh_dfx_unregister_debugfs();
+	destroy_workqueue(hns3_roh_wq);
+	return ret;
 }
 
 static void __exit hns3_roh_module_cleanup(void)
 {
 	hnae3_unregister_client(&hns3_roh_client);
+
+	hns3_roh_dfx_unregister_debugfs();
+
+	destroy_workqueue(hns3_roh_wq);
 }
 
 module_init(hns3_roh_module_init);

drivers/roh/hw/hns3/hns3_verbs.c

+// SPDX-License-Identifier: GPL-2.0+
+// Copyright (c) 2020-2022 Hisilicon Limited.
+
+#include "core.h"
+
+#include "hns3_verbs.h"
+#include "hns3_cmdq.h"
+
+int hns3_roh_set_eid(struct roh_device *rohdev, struct roh_eid_attr *eid_attr)
+{
+	struct hns3_roh_device *hroh_dev = to_hroh_dev(rohdev);
+	struct hns3_roh_set_eid_info *req;
+	struct hns3_roh_desc desc;
+	int ret;
+
+	hns3_roh_cmdq_setup_basic_desc(&desc, HNS3_ROH_OPC_SET_EID, false);
+
+	req = (struct hns3_roh_set_eid_info *)desc.data;
+	req->base_eid = cpu_to_le32(eid_attr->base);
+	req->num_eid = cpu_to_le32(eid_attr->num);
+
+	ret = hns3_roh_cmdq_send(hroh_dev, &desc, 1);
+	if (ret) {
+		dev_err(hroh_dev->dev, "failed to set eid, ret = %d\n", ret);
+		return ret;
+	}
+
+	return 0;
+}
+
+int hns3_roh_query_guid(struct roh_device *rohdev, struct roh_guid_attr *guid_attr)
+{
+	struct hns3_roh_device *hroh_dev = to_hroh_dev(rohdev);
+	struct hns3_roh_get_guid_info *req;
+	struct hns3_roh_desc desc;
+	int ret;
+
+	hns3_roh_cmdq_setup_basic_desc(&desc, HNS3_ROH_OPC_GET_GUID, true);
+	ret = hns3_roh_cmdq_send(hroh_dev, &desc, 1);
+	if (ret) {
+		dev_err(hroh_dev->dev, "failed to query guid, ret = %d\n", ret);
+		return ret;
+	}
+
+	req = (struct hns3_roh_get_guid_info *)desc.data;
+	memcpy(guid_attr->data, req->guid, sizeof(req->guid));
+
+	return 0;
+}
+
+static const char * const hns3_roh_hw_stats_name_public[] = {
+	"mac_tx_packet_num",
+	"mac_rx_packet_num",
+	"reserved",
+	"mac_tx_0_min_pkt_num",
+	"mac_tx_min_64_pkt_num",
+	"mac_tx_65_127_pkt_num",
+	"mac_tx_128_255_pkt_num",
+	"mac_tx_256_511_pkt_num",
+	"mac_tx_512_1023_pkt_num",
+	"mac_tx_1024_1518_pkt_num",
+	"mac_tx_1519_2047_pkt_num",
+	"mac_tx_2048_4095_pkt_num",
+	"mac_tx_4096_max_pkt_num",
+	"mac_tx_over_max_pkt_num",
+	"mac_tx_err_pkt_num",
+	"mac_tx_tc0_pkt_num",
+	"mac_tx_tc1_pkt_num",
+	"mac_tx_tc2_pkt_num",
+	"mac_tx_tc3_pkt_num",
+	"mac_tx_tc4_pkt_num",
+	"mac_tx_tc5_pkt_num",
+	"mac_tx_tc6_pkt_num",
+	"mac_tx_tc7_pkt_num",
+	"mac_tx_tc0_oct_num",
+	"mac_tx_tc1_oct_num",
+	"mac_tx_tc2_oct_num",
+	"mac_tx_tc3_oct_num",
+	"mac_tx_tc4_oct_num",
+	"mac_tx_tc5_oct_num",
+	"mac_tx_tc6_oct_num",
+	"mac_tx_tc7_oct_num",
+	"mac_tx_rdma_pkt_num",
+	"mac_tx_ip_pkt_num",
+	"mac_tx_udmp_eid_pkt_num",
+	"mac_tx_udmp_dr_pkt_num",
+	"mac_tx_rdma_oct_num",
+	"mac_tx_ip_oct_num",
+	"mac_tx_udmp_eid_oct_num",
+	"mac_tx_udmp_dr_oct_num",
+	"mac_tx_udmp_uap_pkt_num",
+	"mac_tx_udmp_uap_oct_num",
+	"mac_rx_udmp_uap_pkt_num",
+	"mac_rx_udmp_uap_oct_num",
+	"mac_rx_0_min_pkt_num",
+	"mac_rx_min_64_pkt_num",
+	"mac_rx_65_127_pkt_num",
+	"mac_rx_128_255_pkt_num",
+	"mac_rx_256_511_pkt_num",
+	"mac_rx_512_1023_pkt_num",
+	"mac_rx_1024_1518_pkt_num",
+	"mac_rx_1519_2047_pkt_num",
+	"mac_rx_2048_4095_pkt_num",
+	"mac_rx_4096_max_pkt_num",
+	"mac_rx_over_max_pkt_num",
+	"mac_rx_err_pkt_num",
+	"mac_rx_tc0_pkt_num",
+	"mac_rx_tc1_pkt_num",
+	"mac_rx_tc2_pkt_num",
+	"mac_rx_tc3_pkt_num",
+	"mac_rx_tc4_pkt_num",
+	"mac_rx_tc5_pkt_num",
+	"mac_rx_tc6_pkt_num",
+	"mac_rx_tc7_pkt_num",
+	"mac_rx_tc0_oct_num",
+	"mac_rx_tc1_oct_num",
+	"mac_rx_tc2_oct_num",
+	"mac_rx_tc3_oct_num",
+	"mac_rx_tc4_oct_num",
+	"mac_rx_tc5_oct_num",
+	"mac_rx_tc6_oct_num",
+	"mac_rx_tc7_oct_num",
+	"mac_rx_rdma_pkt_num",
+	"mac_rx_ip_pkt_num",
+	"mac_rx_udmp_eid_pkt_num",
+	"mac_rx_udmp_dr_pkt_num",
+	"mac_rx_rdma_oct_num",
+	"mac_rx_ip_oct_num",
+	"mac_rx_udmp_eid_oct_num",
+	"mac_rx_udmp_dr_oct_num",
+};
+
+static const char * const hns3_roh_hw_stats_name_private[] = {
+	"mac_tx_block_num",
+	"mac_tx_flit_num",
+	"mac_tx_icrd_vna_total_used",
+	"mac_tx_icrd_vna_total_released",
+	"mac_tx_tc0_icrd_vn_total_used",
+	"mac_tx_tc1_icrd_vn_total_used",
+	"mac_tx_tc2_icrd_vn_total_used",
+	"mac_tx_tc3_icrd_vn_total_used",
+	"mac_tx_tc4_icrd_vn_total_used",
+	"mac_tx_tc5_icrd_vn_total_used",
+	"mac_tx_tc6_icrd_vn_total_used",
+	"mac_tx_tc7_icrd_vn_total_used",
+	"mac_tx_tc0_icrd_vn_total_released",
+	"mac_tx_tc1_icrd_vn_total_released",
+	"mac_tx_tc2_icrd_vn_total_released",
+	"mac_tx_tc3_icrd_vn_total_released",
+	"mac_tx_tc4_icrd_vn_total_released",
+	"mac_tx_tc5_icrd_vn_total_released",
+	"mac_tx_tc6_icrd_vn_total_released",
+	"mac_tx_tc7_icrd_vn_total_released",
+	"mac_tx_rdma_inc_pkt_num",
+	"mac_tx_rdma_inc_oct_num",
+	"mac_rx_rdma_inc_pkt_num",
+	"mac_rx_rdma_inc_oct_num",
+	"mac_rx_block_num",
+	"mac_rx_flit_num",
+	"mac_rx_icrd_vna_total_used",
+	"mac_rx_icrd_vna_total_released",
+	"mac_rx_tc0_icrd_vn_total_used",
+	"mac_rx_tc1_icrd_vn_total_used",
+	"mac_rx_tc2_icrd_vn_total_used",
+	"mac_rx_tc3_icrd_vn_total_used",
+	"mac_rx_tc4_icrd_vn_total_used",
+	"mac_rx_tc5_icrd_vn_total_used",
+	"mac_rx_tc6_icrd_vn_total_used",
+	"mac_rx_tc7_icrd_vn_total_used",
+	"mac_rx_tc0_icrd_vn_total_released",
+	"mac_rx_tc1_icrd_vn_total_released",
+	"mac_rx_tc2_icrd_vn_total_released",
+	"mac_rx_tc3_icrd_vn_total_released",
+	"mac_rx_tc4_icrd_vn_total_released",
+	"mac_rx_tc5_icrd_vn_total_released",
+	"mac_rx_tc6_icrd_vn_total_released",
+	"mac_rx_tc7_icrd_vn_total_released",
+};
+
+struct roh_mib_stats *hns3_roh_alloc_hw_stats(struct roh_device *rohdev, enum roh_mib_type mib_type)
+{
+	struct roh_mib_stats *stats = NULL;
+	int num_counters;
+
+	switch (mib_type) {
+	case ROH_MIB_PUBLIC:
+		num_counters = ARRAY_SIZE(hns3_roh_hw_stats_name_public);
+		stats = kzalloc(sizeof(*stats) + num_counters * sizeof(u64), GFP_KERNEL);
+		if (!stats)
+			return NULL;
+		stats->names = hns3_roh_hw_stats_name_public;
+		stats->num_counters = num_counters;
+		break;
+	case ROH_MIB_PRIVATE:
+		num_counters = ARRAY_SIZE(hns3_roh_hw_stats_name_private);
+		stats = kzalloc(sizeof(*stats) + num_counters * sizeof(u64), GFP_KERNEL);
+		if (!stats)
+			return NULL;
+		stats->names = hns3_roh_hw_stats_name_private;
+		stats->num_counters = num_counters;
+		break;
+	default:
+		break;
+	}
+
+	return stats;
+}
+
+int hns3_roh_get_hw_stats(struct roh_device *rohdev, struct roh_mib_stats *stats,
+			  enum roh_mib_type mib_type)
+{
+	struct hns3_roh_device *hroh_dev = to_hroh_dev(rohdev);
+	u64 *data = (u64 *)(stats->value);
+	enum hns3_roh_opcode_type opcode;
+	struct hns3_roh_desc *desc;
+	int start, stats_num;
+	__le64 *desc_data;
+	u32 desc_num;
+	int i, j;
+	int ret;
+
+	if (mib_type != ROH_MIB_PUBLIC && mib_type != ROH_MIB_PRIVATE) {
+		ret = -EINVAL;
+		goto err_out;
+	}
+
+	if (mib_type == ROH_MIB_PUBLIC)
+		desc_num = 1 + DIV_ROUND_UP(stats->num_counters - HNS3_ROH_RD_FIRST_STATS_NUM,
+			HNS3_ROH_RD_OTHER_STATS_NUM);
+	else
+		desc_num = 1 + DIV_ROUND_UP(stats->num_counters, HNS3_ROH_RD_OTHER_STATS_NUM);
+	desc = kcalloc(desc_num, sizeof(struct hns3_roh_desc), GFP_KERNEL);
+	if (!desc) {
+		ret = -ENOMEM;
+		goto err_out;
+	}
+
+	opcode = (mib_type == ROH_MIB_PUBLIC) ?
+		HNS3_ROH_OPC_QUERY_MIB_PUBLIC : HNS3_ROH_OPC_QUERY_MIB_PRIVATE;
+	hns3_roh_cmdq_setup_basic_desc(&desc[0], opcode, true);
+
+	ret = hns3_roh_cmdq_send(hroh_dev, desc, desc_num);
+	if (ret)
+		goto err_send_cmd;
+
+	start = (mib_type == ROH_MIB_PUBLIC) ? 0 : 1;
+	for (i = start; i < desc_num; i++) {
+		/* HNS3_ROH_OPC_QUERY_MIB_PUBLIC: only the first desc has the head
+		 * HNS3_ROH_OPC_QUERY_MIB_PRIVATE: start from command1, no head
+		 */
+		if (i == 0) {
+			desc_data = (__le64 *)(&desc[i].data[0]);
+			stats_num = HNS3_ROH_RD_FIRST_STATS_NUM;
+		} else {
+			desc_data = (__le64 *)(&desc[i]);
+			stats_num = HNS3_ROH_RD_OTHER_STATS_NUM;
+		}
+
+		for (j = 0; j < stats_num; j++) {
+			*data = le64_to_cpu(*desc_data);
+			data++;
+			desc_data++;
+		}
+	}
+
+err_send_cmd:
+	kfree(desc);
+err_out:
+	return ret;
+}

drivers/roh/hw/hns3/hns3_verbs.h

+/* SPDX-License-Identifier: GPL-2.0+ */
+// Copyright (c) 2020-2022 Hisilicon Limited.
+#ifndef __HNS3_ROH_VERBS_H__
+#define __HNS3_ROH_VERBS_H__
+
+#include "core.h"
+#include "hns3_common.h"
+
+int hns3_roh_set_eid(struct roh_device *rohdev, struct roh_eid_attr *eid_attr);
+int hns3_roh_query_guid(struct roh_device *rohdev, struct roh_guid_attr *guid_attr);
+struct roh_mib_stats *hns3_roh_alloc_hw_stats(struct roh_device *rohdev,
+					      enum roh_mib_type mib_type);
+int hns3_roh_get_hw_stats(struct roh_device *rohdev, struct roh_mib_stats *stats,
+			  enum roh_mib_type mib_type);
+
+#endif /* __HNS3_ROH_VERBS_H__ */

include/uapi/rdma/hns-abi.h

@@ -116,6 +116,8 @@ struct hns_roce_ib_alloc_ucontext_resp {
 	__u32   reserved;
 	__u32	config;
 	__u32	max_inline_data;
+	__u8	mac_type;
+	__u8	rsv1[7];
 };
 
 struct hns_roce_ib_alloc_ucontext {