!402 Fixed the following errors: The reset with stream fails, the query of AH...

!402 Fixed the following errors:  The reset with stream fails, the query of AH attr is invalid and the RoCE Bonding

Merge Pull Request from: @stinft 
 
Bugfix information:
1.RDMA/hns: Kernel notify usr space to stop ring db
 In the reset scenario, if the kernel receives the reset signal,
 it needs to notify the user space to stop ring doorbell.
 bugzilla: #I6F3ZU

2.RDMA/hns: Fix AH attr queried by query_qp
  bugzilla: #I6F3ZA

3.RDMA/hns: fix the error of RoCE VF based on RoCE Bonding PF
    In this patch, the following constraints are added:
    1. RoCE Bonding cannot be set with a PF which enables VF;
    2. A PF in RoCE Bonding cannot enable RoCE VF.
  bugzilla: #I6F1IQ 
 
Link:https://gitee.com/openeuler/kernel/pulls/402 

Reviewed-by: Yue Haibing <yuehaibing@huawei.com> 
Reviewed-by: Zheng Zengkai <zhengzengkai@huawei.com> 
Signed-off-by: Zheng Zengkai <zhengzengkai@huawei.com> 

drivers/infiniband/hw/hns/Makefile

@@ -4,6 +4,8 @@
 #
 
 ccflags-y :=  -I $(srctree)/drivers/net/ethernet/hisilicon/hns3
+ccflags-y +=  -I $(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3pf
+ccflags-y +=  -I $(srctree)/drivers/net/ethernet/hisilicon/hns3/hns3_common
 
 hns-roce-objs := hns_roce_main.o hns_roce_cmd.o hns_roce_pd.o \
 	hns_roce_ah.o hns_roce_hem.o hns_roce_mr.o hns_roce_qp.o \

drivers/infiniband/hw/hns/hns_roce_bond.c

@@ -26,12 +26,15 @@ static struct hns_roce_dev *hns_roce_get_hrdev_by_netdev(struct net_device *net_
 	return hr_dev;
 }
 
-static struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev)
+struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev)
 {
 	struct hns_roce_bond_group *bond_grp = NULL;
 	struct net_device *upper_dev;
 	struct net_device *net_dev;
 
+	if (!netif_is_lag_port(hr_dev->iboe.netdevs[0]))
+		return NULL;
+
 	rcu_read_lock();
 
 	upper_dev = netdev_master_upper_dev_get_rcu(hr_dev->iboe.netdevs[0]);
@@ -53,9 +56,6 @@ bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev)
 {
 	struct hns_roce_bond_group *bond_grp;
 
-	if (!netif_is_lag_port(hr_dev->iboe.netdevs[0]))
-		return false;
-
 	bond_grp = hns_roce_get_bond_grp(hr_dev);
 
 	if (bond_grp &&
@@ -75,9 +75,6 @@ struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev)
 	if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND))
 		return NULL;
 
-	if (!netif_is_lag_port(hr_dev->iboe.netdevs[0]))
-		return NULL;
-
 	if (!bond_grp) {
 		bond_grp = hns_roce_get_bond_grp(hr_dev);
 		if (!bond_grp)
@@ -644,7 +641,7 @@ static enum bond_support_type
 			slave_num++;
 			if (bus_num == -1)
 				bus_num = hr_dev->pci_dev->bus->number;
-			if (hr_dev->is_vf ||
+			if (hr_dev->is_vf || pci_num_vf(hr_dev->pci_dev) > 0 ||
 			    bus_num != hr_dev->pci_dev->bus->number) {
 				support = false;
 				break;

drivers/infiniband/hw/hns/hns_roce_bond.h

@@ -75,5 +75,6 @@ int hns_roce_bond_event(struct notifier_block *self,
 void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev);
 bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev);
 struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev);
+struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev);
 
 #endif

drivers/infiniband/hw/hns/hns_roce_device.h

@@ -59,6 +59,7 @@
 
 #define HNS_ROCE_CEQ				0
 #define HNS_ROCE_AEQ				1
+#define HNS_ROCE_IS_RESETTING			1
 
 #define HNS_ROCE_CEQE_SIZE 0x4
 #define HNS_ROCE_AEQE_SIZE 0x10
@@ -206,6 +207,7 @@ enum hns_roce_mmap_type {
 	HNS_ROCE_MMAP_TYPE_DB = 1,
 	HNS_ROCE_MMAP_TYPE_DWQE,
 	HNS_ROCE_MMAP_TYPE_DCA,
+	HNS_ROCE_MMAP_TYPE_RESET,
 };
 
 struct hns_user_mmap_entry {
@@ -248,6 +250,7 @@ struct hns_roce_ucontext {
 	struct list_head	page_list;
 	struct mutex		page_mutex;
 	struct hns_user_mmap_entry *db_mmap_entry;
+	struct hns_user_mmap_entry *reset_mmap_entry;
 	u32			config;
 	struct hns_roce_dca_ctx	dca_ctx;
 	void *dca_dbgfs;
@@ -1027,6 +1030,8 @@ struct hns_roce_dev {
 	int			loop_idc;
 	u32			sdb_offset;
 	u32			odb_offset;
+	struct page		*reset_page; /* store reset state */
+	void			*reset_kaddr; /* addr of reset page */
 	const struct hns_roce_hw *hw;
 	void			*priv;
 	struct workqueue_struct *irq_workq;

drivers/infiniband/hw/hns/hns_roce_hw_v2.c

@@ -43,6 +43,7 @@
 #include <rdma/uverbs_ioctl.h>
 
 #include "hnae3.h"
+#include "hclge_main.h"
 #include "hns_roce_common.h"
 #include "hns_roce_device.h"
 #include "hns_roce_cmd.h"
@@ -2812,6 +2813,31 @@ static void free_dip_list(struct hns_roce_dev *hr_dev)
 	spin_unlock_irqrestore(&hr_dev->dip_list_lock, flags);
 }
 
+static int hns_roce_v2_get_reset_page(struct hns_roce_dev *hr_dev)
+{
+	hr_dev->reset_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
+	if (!hr_dev->reset_page)
+		return -ENOMEM;
+
+	hr_dev->reset_kaddr = vmap(&hr_dev->reset_page, 1, VM_MAP, PAGE_KERNEL);
+	if (!hr_dev->reset_kaddr)
+		goto err_with_vmap;
+
+	return 0;
+
+err_with_vmap:
+	put_page(hr_dev->reset_page);
+	return -ENOMEM;
+}
+
+static void hns_roce_v2_put_reset_page(struct hns_roce_dev *hr_dev)
+{
+	vunmap(hr_dev->reset_kaddr);
+	hr_dev->reset_kaddr = NULL;
+	put_page(hr_dev->reset_page);
+	hr_dev->reset_page = NULL;
+}
+
 static struct ib_pd *free_mr_init_pd(struct hns_roce_dev *hr_dev)
 {
 	struct hns_roce_v2_priv *priv = hr_dev->priv;
@@ -3168,16 +3194,23 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
 {
 	int ret;
 
+	ret = hns_roce_v2_get_reset_page(hr_dev);
+	if (ret) {
+		dev_err(hr_dev->dev,
+			"reset state init failed, ret = %d.\n", ret);
+		return ret;
+	}
+
 	/* The hns ROCEE requires the extdb info to be cleared before using */
 	ret = hns_roce_clear_extdb_list_info(hr_dev);
 	if (ret)
-		return ret;
+		goto err_clear_extdb_failed;
 
 	hns_roce_set_mac_type(hr_dev);
 
 	ret = get_hem_table(hr_dev);
 	if (ret)
-		return ret;
+		goto err_clear_extdb_failed;
 
 	if (hr_dev->is_vf)
 		return 0;
@@ -3192,6 +3225,8 @@ static int hns_roce_v2_init(struct hns_roce_dev *hr_dev)
 
 err_llm_init_failed:
 	put_hem_table(hr_dev);
+err_clear_extdb_failed:
+	hns_roce_v2_put_reset_page(hr_dev);
 
 	return ret;
 }
@@ -3203,6 +3238,8 @@ static void hns_roce_v2_exit(struct hns_roce_dev *hr_dev)
 	if (!hr_dev->is_vf)
 		hns_roce_free_link_table(hr_dev);
 
+	hns_roce_v2_put_reset_page(hr_dev);
+
 	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP09)
 		free_dip_list(hr_dev);
 }
@@ -5775,6 +5812,8 @@ static int hns_roce_v2_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
 
 		rdma_ah_set_sl(&qp_attr->ah_attr,
 			       hr_reg_read(&context, QPC_SL));
+		rdma_ah_set_port_num(&qp_attr->ah_attr, hr_qp->port + 1);
+		rdma_ah_set_ah_flags(&qp_attr->ah_attr, IB_AH_GRH);
 		grh->flow_label = hr_reg_read(&context, QPC_FL);
 		grh->sgid_index = hr_reg_read(&context, QPC_GMV_IDX);
 		grh->hop_limit = hr_reg_read(&context, QPC_HOPLIMIT);
@@ -7160,6 +7199,34 @@ static void hns_roce_hw_v2_get_cfg(struct hns_roce_dev *hr_dev,
 	priv->handle = handle;
 }
 
+static bool check_vf_support(struct pci_dev *vf)
+{
+	struct hns_roce_bond_group *bond_grp;
+	struct pci_dev *pf = pci_physfn(vf);
+	struct hnae3_ae_dev *ae_dev;
+	struct hnae3_handle *handle;
+	struct hns_roce_dev *hr_dev;
+	struct hclge_dev *hdev;
+
+	/* pf == vf means that the driver is running in VM. */
+	if (pf == vf)
+		return true;
+
+	ae_dev = pci_get_drvdata(pf);
+	hdev = ae_dev->priv;
+	handle = &hdev->vport[0].roce;
+	hr_dev = handle->priv;
+
+	if (!hr_dev)
+		return false;
+
+	bond_grp = hns_roce_get_bond_grp(hr_dev);
+	if (bond_grp)
+		return false;
+
+	return true;
+}
+
 static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
 {
 	struct hns_roce_dev *hr_dev;
@@ -7177,6 +7244,11 @@ static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
 
 	hns_roce_hw_v2_get_cfg(hr_dev, handle);
 
+	if (hr_dev->is_vf && !check_vf_support(hr_dev->pci_dev)) {
+		ret = -EOPNOTSUPP;
+		goto error_failed_kzalloc;
+	}
+
 	ret = hns_roce_init(hr_dev);
 	if (ret) {
 		dev_err(hr_dev->dev, "RoCE Engine init failed!\n");
@@ -7252,6 +7324,10 @@ int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
 	ret = __hns_roce_hw_v2_init_instance(handle);
 	if (ret) {
 		handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
+
+		if (ret == -EOPNOTSUPP)
+			return ret;
+
 		dev_err(dev, "RoCE instance init failed! ret = %d\n", ret);
 		if (ops->ae_dev_resetting(handle) ||
 		    ops->get_hw_reset_stat(handle))
@@ -7282,6 +7358,18 @@ void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, bool reset)
 
 	handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
 }
+
+static void hns_roce_v2_reset_notify_user(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_v2_reset_state *state;
+
+	state = (struct hns_roce_v2_reset_state *)hr_dev->reset_kaddr;
+
+	state->reset_state = HNS_ROCE_IS_RESETTING;
+	/* Ensure reset state was flushed in memory */
+	wmb();
+}
+
 static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
 {
 	struct hns_roce_dev *hr_dev;
@@ -7300,6 +7388,9 @@ static int hns_roce_hw_v2_reset_notify_down(struct hnae3_handle *handle)
 
 	hr_dev->active = false;
 	hr_dev->dis_db = true;
+
+	hns_roce_v2_reset_notify_user(hr_dev);
+
 	hr_dev->state = HNS_ROCE_DEVICE_STATE_RST_DOWN;
 
 	return 0;

drivers/infiniband/hw/hns/hns_roce_hw_v2.h

@@ -1340,6 +1340,10 @@ struct hns_roce_link_table {
 #define HNS_ROCE_EXT_LLM_ENTRY(addr, id) (((id) << (64 - 12)) | ((addr) >> 12))
 #define HNS_ROCE_EXT_LLM_MIN_PAGES(que_num) ((que_num) * 4 + 2)
 
+struct hns_roce_v2_reset_state {
+	u32 reset_state; /* stored to use in user space */
+};
+
 struct hns_roce_v2_free_mr {
 	struct hns_roce_qp *rsv_qp[HNS_ROCE_FREE_MR_USED_QP_NUM];
 	struct hns_roce_cq *rsv_cq;

drivers/infiniband/hw/hns/hns_roce_main.c

@@ -367,6 +367,7 @@ hns_roce_user_mmap_entry_insert(struct ib_ucontext *ucontext, u64 address,
 		break;
 	case HNS_ROCE_MMAP_TYPE_DWQE:
 	case HNS_ROCE_MMAP_TYPE_DCA:
+	case HNS_ROCE_MMAP_TYPE_RESET:
 		ret = rdma_user_mmap_entry_insert_range(
 				ucontext, &entry->rdma_entry, length, 1,
 				U32_MAX);
@@ -408,6 +409,26 @@ static int hns_roce_alloc_uar_entry(struct ib_ucontext *uctx)
 	return 0;
 }
 
+static void hns_roce_dealloc_reset_entry(struct hns_roce_ucontext *context)
+{
+	if (context->reset_mmap_entry)
+		rdma_user_mmap_entry_remove(&context->reset_mmap_entry->rdma_entry);
+}
+
+static int hns_roce_alloc_reset_entry(struct ib_ucontext *uctx)
+{
+	struct hns_roce_ucontext *context = to_hr_ucontext(uctx);
+	struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device);
+
+	context->reset_mmap_entry = hns_roce_user_mmap_entry_insert(uctx,
+		(u64)hr_dev->reset_kaddr, PAGE_SIZE, HNS_ROCE_MMAP_TYPE_RESET);
+
+	if (!context->reset_mmap_entry)
+		return -ENOMEM;
+
+	return 0;
+}
+
 static void ucontext_set_resp(struct ib_ucontext *uctx,
 			      struct hns_roce_ib_alloc_ucontext_resp *resp)
 {
@@ -425,6 +446,11 @@ static void ucontext_set_resp(struct ib_ucontext *uctx,
 		rdma_entry = &context->dca_ctx.dca_mmap_entry->rdma_entry;
 		resp->dca_mmap_key = rdma_user_mmap_get_offset(rdma_entry);
 	}
+
+	if (context->reset_mmap_entry) {
+		rdma_entry = &context->reset_mmap_entry->rdma_entry;
+		resp->reset_mmap_key = rdma_user_mmap_get_offset(rdma_entry);
+	}
 }
 
 static u32 get_udca_max_qps(struct hns_roce_dev *hr_dev,
@@ -503,6 +529,10 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
 	hns_roce_register_udca(hr_dev, get_udca_max_qps(hr_dev, &ucmd),
 			       context);
 
+	ret = hns_roce_alloc_reset_entry(uctx);
+	if (ret)
+		goto error_fail_reset_entry;
+
 	ucontext_set_resp(uctx, &resp);
 	ret = ib_copy_to_udata(udata, &resp, min(udata->outlen, sizeof(resp)));
 	if (ret)
@@ -518,7 +548,9 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
 
 error_fail_copy_to_udata:
 	hns_roce_unregister_udca(hr_dev, context);
+	hns_roce_dealloc_reset_entry(context);
 
+error_fail_reset_entry:
 	hns_roce_dealloc_uar_entry(context);
 
 error_fail_uar_entry:
@@ -542,6 +574,7 @@ static void hns_roce_dealloc_ucontext(struct ib_ucontext *ibcontext)
 	hns_roce_unregister_udca(hr_dev, context);
 
 	hns_roce_dealloc_uar_entry(context);
+	hns_roce_dealloc_reset_entry(context);
 
 	ida_free(&hr_dev->uar_ida.ida, (int)context->uar.logic_idx);
 }
@@ -578,6 +611,7 @@ static int mmap_dca(struct ib_ucontext *context, struct vm_area_struct *vma)
 
 static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma)
 {
+	struct hns_roce_dev *hr_dev = to_hr_dev(uctx->device);
 	struct rdma_user_mmap_entry *rdma_entry;
 	struct hns_user_mmap_entry *entry;
 	phys_addr_t pfn;
@@ -599,8 +633,19 @@ static int hns_roce_mmap(struct ib_ucontext *uctx, struct vm_area_struct *vma)
 	case HNS_ROCE_MMAP_TYPE_DCA:
 		ret = mmap_dca(uctx, vma);
 		goto out;
+	case HNS_ROCE_MMAP_TYPE_RESET:
+		if (vma->vm_flags & (VM_WRITE | VM_EXEC)) {
+			ret = -EINVAL;
+			goto out;
+		}
+
+		ret = remap_pfn_range(vma, vma->vm_start,
+				      page_to_pfn(hr_dev->reset_page),
+				      PAGE_SIZE, vma->vm_page_prot);
+		goto out;
 	default:
-		return -EINVAL;
+		ret = -EINVAL;
+		goto out;
 	}
 
 	ret = rdma_user_mmap_io(uctx, vma, pfn, rdma_entry->npages * PAGE_SIZE,

include/uapi/rdma/hns-abi.h

@@ -127,6 +127,7 @@ struct hns_roce_ib_alloc_ucontext_resp {
 	__u32	dca_qps;
 	__u32	dca_mmap_size;
 	__aligned_u64 dca_mmap_key;
+	__aligned_u64 reset_mmap_key;
 };
 
 enum hns_roce_uctx_comp_mask {