!639 Fix errors related to bond and rq inline

Merge Pull Request from: @stinft 
 
1. #I6Z4E9 
2. #I6ZACP 
3. #I6ZAIM 
4. #I6ZBLL 
 
Link:https://gitee.com/openeuler/kernel/pulls/639 

Reviewed-by: Chengchang Tang <tangchengchang@huawei.com> 
Signed-off-by: Jialin Zhang <zhangjialin11@huawei.com> 

drivers/infiniband/hw/hns/hns_roce_bond.h

@@ -10,7 +10,9 @@
 #include <net/bonding.h>
 
 #define ROCE_BOND_FUNC_MAX 4
-#define ROCE_BOND_NAME_ID_IDX 9
+#define ROCE_BOND_NUM_MAX 2
+
+#define BOND_ID(id) BIT(id)
 
 enum {
 	BOND_MODE_1,
@@ -45,14 +47,11 @@ enum hns_roce_bond_cmd_type {
 struct hns_roce_func_info {
 	struct net_device *net_dev;
 	struct hnae3_handle *handle;
-	struct netdev_lag_lower_state_info state;
 };
 
 struct hns_roce_bond_group {
 	struct net_device *upper_dev;
-	struct net_device *main_net_dev;
 	struct hns_roce_dev *main_hr_dev;
-	u8 slave_num;
 	u8 active_slave_num;
 	u32 slave_map;
 	u32 active_slave_map;
@@ -67,12 +66,18 @@ struct hns_roce_bond_group {
 	 */
 	struct mutex bond_mutex;
 	struct hns_roce_func_info bond_func_info[ROCE_BOND_FUNC_MAX];
+	struct delayed_work bond_work;
+};
+
+struct hns_roce_die_info {
+	u8 bond_id_mask;
+	struct hns_roce_bond_group *bgrps[ROCE_BOND_NUM_MAX];
 };
 
 int hns_roce_bond_init(struct hns_roce_dev *hr_dev);
 int hns_roce_bond_event(struct notifier_block *self,
 			unsigned long event, void *ptr);
-void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev);
+void hns_roce_cleanup_bond(struct hns_roce_bond_group *bond_grp);
 bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev);
 struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev);
 struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev);

drivers/infiniband/hw/hns/hns_roce_device.h

@@ -143,8 +143,7 @@ enum hns_roce_qp_create_flags {
 enum {
 	HNS_ROCE_CAP_FLAG_REREG_MR		= BIT(0),
 	HNS_ROCE_CAP_FLAG_ROCE_V1_V2		= BIT(1),
-	/* discard this bit, reserved for compatibility */
-	HNS_ROCE_CAP_FLAG_DISCARD		= BIT(2),
+	HNS_ROCE_CAP_FLAG_RQ_INLINE		= BIT(2),
 	HNS_ROCE_CAP_FLAG_CQ_RECORD_DB		= BIT(3),
 	HNS_ROCE_CAP_FLAG_QP_RECORD_DB		= BIT(4),
 	HNS_ROCE_CAP_FLAG_SRQ			= BIT(5),
@@ -159,7 +158,6 @@ enum {
 	HNS_ROCE_CAP_FLAG_DCA_MODE		= BIT(15),
 	HNS_ROCE_CAP_FLAG_STASH			= BIT(17),
 	HNS_ROCE_CAP_FLAG_CQE_INLINE		= BIT(19),
-	HNS_ROCE_CAP_FLAG_RQ_INLINE		= BIT(20),
 	HNS_ROCE_CAP_FLAG_BOND			= BIT(21),
 };
 
@@ -1124,9 +1122,6 @@ struct hns_roce_dev {
 	u64 dwqe_page;
 
 	struct notifier_block bond_nb;
-	struct delayed_work bond_work;
-	struct hns_roce_bond_group *bond_grp;
-	struct netdev_lag_lower_state_info slave_state;
 	struct hns_roce_port port_data[HNS_ROCE_MAX_PORTS];
 	atomic64_t *dfx_cnt;
 };
@@ -1402,7 +1397,7 @@ void hns_roce_srq_event(struct hns_roce_dev *hr_dev, u32 srqn, int event_type);
 u8 hns_get_gid_index(struct hns_roce_dev *hr_dev, u8 port, int gid_index);
 void hns_roce_handle_device_err(struct hns_roce_dev *hr_dev);
 int hns_roce_init(struct hns_roce_dev *hr_dev);
-void hns_roce_exit(struct hns_roce_dev *hr_dev);
+void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup);
 int hns_roce_fill_res_cq_entry(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_cq_entry_raw(struct sk_buff *msg, struct ib_cq *ib_cq);
 int hns_roce_fill_res_qp_entry(struct sk_buff *msg, struct ib_qp *ib_qp);

drivers/infiniband/hw/hns/hns_roce_hw_v2.c

@@ -1493,7 +1493,7 @@ static inline enum hns_roce_opcode_type
 		return HNS_ROCE_OPC_CLEAR_BOND_INFO;
 }
 
-int hns_roce_cmd_bond(struct hns_roce_dev *hr_dev,
+int hns_roce_cmd_bond(struct hns_roce_bond_group *bond_grp,
 		      enum hns_roce_bond_cmd_type bond_type)
 {
 	enum hns_roce_opcode_type opcode = get_bond_opcode(bond_type);
@@ -1504,33 +1504,33 @@ int hns_roce_cmd_bond(struct hns_roce_dev *hr_dev,
 	slave_info = (struct hns_roce_bond_info *)desc.data;
 	hns_roce_cmq_setup_basic_desc(&desc, opcode, false);
 
-	slave_info->bond_id = cpu_to_le32(hr_dev->bond_grp->bond_id);
+	slave_info->bond_id = cpu_to_le32(bond_grp->bond_id);
 	if (bond_type == HNS_ROCE_CLEAR_BOND)
 		goto out;
 
-	if (hr_dev->bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
+	if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
 		slave_info->bond_mode = cpu_to_le32(BOND_MODE_1);
-		if (hr_dev->bond_grp->active_slave_num != 1)
-			ibdev_err(&hr_dev->ib_dev,
+		if (bond_grp->active_slave_num != 1)
+			ibdev_err(&bond_grp->main_hr_dev->ib_dev,
 				  "active slave cnt(%d) in Mode 1 is invalid.\n",
-				  hr_dev->bond_grp->active_slave_num);
+				  bond_grp->active_slave_num);
 	} else {
 		slave_info->bond_mode = cpu_to_le32(BOND_MODE_2_4);
 		slave_info->hash_policy =
-			cpu_to_le32(hr_dev->bond_grp->bond->params.xmit_policy);
+			cpu_to_le32(bond_grp->bond->params.xmit_policy);
 	}
 
 	slave_info->active_slave_cnt =
-		cpu_to_le32(hr_dev->bond_grp->active_slave_num);
+		cpu_to_le32(bond_grp->active_slave_num);
 	slave_info->active_slave_mask =
-		cpu_to_le32(hr_dev->bond_grp->active_slave_map);
+		cpu_to_le32(bond_grp->active_slave_map);
 	slave_info->slave_mask =
-		cpu_to_le32(hr_dev->bond_grp->slave_map);
+		cpu_to_le32(bond_grp->slave_map);
 
 out:
-	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+	ret = hns_roce_cmq_send(bond_grp->main_hr_dev, &desc, 1);
 	if (ret)
-		ibdev_err(&hr_dev->ib_dev,
+		ibdev_err(&bond_grp->main_hr_dev->ib_dev,
 			  "cmq bond type(%d) failed, ret = %d.\n",
 			  bond_type, ret);
 
@@ -7341,7 +7341,7 @@ static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
 	return 0;
 
 error_failed_roce_init:
-	hns_roce_exit(hr_dev);
+	hns_roce_exit(hr_dev, true);
 
 error_failed_cfg:
 	kfree(hr_dev->priv);
@@ -7353,7 +7353,7 @@ static int __hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
 }
 
 static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
-					   bool reset)
+					   bool reset, bool bond_cleanup)
 {
 	struct hns_roce_dev *hr_dev = handle->priv;
 
@@ -7368,12 +7368,12 @@ static void __hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
 	if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08)
 		free_mr_exit(hr_dev);
 
-	hns_roce_exit(hr_dev);
+	hns_roce_exit(hr_dev, bond_cleanup);
 	kfree(hr_dev->priv);
 	ib_dealloc_device(&hr_dev->ib_dev);
 }
 
-int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
+static int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
 {
 	const struct hnae3_ae_ops *ops = handle->ae_algo->ops;
 	const struct pci_device_id *id;
@@ -7420,14 +7420,45 @@ int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle)
 	return -EBUSY;
 }
 
-void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, bool reset)
+static void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle,
+					   bool reset)
 {
 	if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
 		return;
 
 	handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
 
-	__hns_roce_hw_v2_uninit_instance(handle, reset);
+	__hns_roce_hw_v2_uninit_instance(handle, reset, true);
+
+	handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
+}
+
+struct hns_roce_dev
+	*hns_roce_bond_init_client(struct hns_roce_bond_group *bond_grp,
+				   int func_idx)
+{
+	struct hnae3_handle *handle;
+	int ret;
+
+	handle = bond_grp->bond_func_info[func_idx].handle;
+	ret = hns_roce_hw_v2_init_instance(handle);
+	if (ret)
+		return NULL;
+
+	return handle->priv;
+}
+
+void hns_roce_bond_uninit_client(struct hns_roce_bond_group *bond_grp,
+				 int func_idx)
+{
+	struct hnae3_handle *handle = bond_grp->bond_func_info[func_idx].handle;
+
+	if (handle->rinfo.instance_state != HNS_ROCE_STATE_INITED)
+		return;
+
+	handle->rinfo.instance_state = HNS_ROCE_STATE_UNINIT;
+
+	__hns_roce_hw_v2_uninit_instance(handle, false, false);
 
 	handle->rinfo.instance_state = HNS_ROCE_STATE_NON_INIT;
 }
@@ -7507,7 +7538,7 @@ static int hns_roce_hw_v2_reset_notify_uninit(struct hnae3_handle *handle)
 	handle->rinfo.reset_state = HNS_ROCE_STATE_RST_UNINIT;
 	dev_info(&handle->pdev->dev, "In reset process RoCE client uninit.\n");
 	msleep(HNS_ROCE_V2_HW_RST_UNINT_DELAY);
-	__hns_roce_hw_v2_uninit_instance(handle, false);
+	__hns_roce_hw_v2_uninit_instance(handle, false, false);
 
 	return 0;
 }
@@ -7539,6 +7570,7 @@ static void hns_roce_hw_v2_link_status_change(struct hnae3_handle *handle,
 {
 	struct net_device *netdev = handle->rinfo.netdev;
 	struct hns_roce_dev *hr_dev = handle->priv;
+	struct hns_roce_bond_group *bond_grp;
 	struct ib_event event;
 	unsigned long flags;
 	u8 phy_port;
@@ -7546,6 +7578,15 @@ static void hns_roce_hw_v2_link_status_change(struct hnae3_handle *handle,
 	if (linkup || !hr_dev)
 		return;
 
+	/* For bond device, the link status depends on the upper netdev,
+	 * and the upper device's link status depends on all the slaves'
+	 * netdev but not only one. So bond device cannot get a correct
+	 * link status from this path.
+	 */
+	bond_grp = hns_roce_get_bond_grp(hr_dev);
+	if (bond_grp)
+		return;
+
 	for (phy_port = 0; phy_port < hr_dev->caps.num_ports; phy_port++)
 		if (netdev == hr_dev->iboe.netdevs[phy_port])
 			break;

drivers/infiniband/hw/hns/hns_roce_hw_v2.h

@@ -1594,15 +1594,17 @@ struct hns_roce_bond_info {
 	__le32 hash_policy;
 };
 
-int hns_roce_hw_v2_init_instance(struct hnae3_handle *handle);
-void hns_roce_hw_v2_uninit_instance(struct hnae3_handle *handle, bool reset);
-
+struct hns_roce_dev
+	*hns_roce_bond_init_client(struct hns_roce_bond_group *bond_grp,
+				   int func_idx);
+void hns_roce_bond_uninit_client(struct hns_roce_bond_group *bond_grp,
+				 int func_idx);
 int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
 
 int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
 				  struct hns_roce_qp *hr_qp,
 				  struct ib_udata *udata);
-int hns_roce_cmd_bond(struct hns_roce_dev *hr_dev,
+int hns_roce_cmd_bond(struct hns_roce_bond_group *bond_grp,
 		      enum hns_roce_bond_cmd_type bond_type);
 
 static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],

drivers/infiniband/hw/hns/hns_roce_main.c

@@ -118,17 +118,29 @@ static int hns_roce_del_gid(const struct ib_gid_attr *attr, void **context)
 	return ret;
 }
 
-static int handle_en_event(struct hns_roce_dev *hr_dev, u8 port,
-			   unsigned long dev_event)
+static enum ib_port_state get_upper_port_state(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_bond_group *bond_grp;
+	struct net_device *upper;
+
+	bond_grp = hns_roce_get_bond_grp(hr_dev);
+	upper = bond_grp ? bond_grp->upper_dev : NULL;
+	if (upper)
+		return get_port_state(upper);
+
+	return IB_PORT_ACTIVE;
+}
+
+static int handle_en_event(struct net_device *netdev,
+			   struct hns_roce_dev *hr_dev,
+			   u32 port, unsigned long dev_event)
 {
 	struct device *dev = hr_dev->dev;
 	enum ib_port_state port_state;
-	struct net_device *netdev;
 	struct ib_event event;
 	unsigned long flags;
 	int ret = 0;
 
-	netdev = hr_dev->iboe.netdevs[port];
 	if (!netdev) {
 		dev_err(dev, "Can't find netdev on port(%u)!\n", port);
 		return -ENODEV;
@@ -176,17 +188,24 @@ static int hns_roce_netdev_event(struct notifier_block *self,
 				 unsigned long event, void *ptr)
 {
 	struct net_device *dev = netdev_notifier_info_to_dev(ptr);
+	struct hns_roce_bond_group *bond_grp;
 	struct hns_roce_ib_iboe *iboe = NULL;
 	struct hns_roce_dev *hr_dev = NULL;
+	struct net_device *upper = NULL;
 	int ret;
 	u8 port;
 
 	hr_dev = container_of(self, struct hns_roce_dev, iboe.nb);
 	iboe = &hr_dev->iboe;
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) {
+		bond_grp = hns_roce_get_bond_grp(hr_dev);
+		upper = bond_grp ? bond_grp->upper_dev : NULL;
+	}
 
 	for (port = 0; port < hr_dev->caps.num_ports; port++) {
-		if (dev == iboe->netdevs[port]) {
-			ret = handle_en_event(hr_dev, port, event);
+		if ((!upper && dev == iboe->netdevs[port]) ||
+		    (upper && dev == upper)) {
+			ret = handle_en_event(dev, hr_dev, port, event);
 			if (ret)
 				return NOTIFY_DONE;
 			break;
@@ -301,6 +320,11 @@ static int hns_roce_query_port(struct ib_device *ib_dev, u8 port_num,
 	mtu = iboe_get_mtu(net_dev->mtu);
 	props->active_mtu = mtu ? min(props->max_mtu, mtu) : IB_MTU_256;
 	props->state = get_port_state(net_dev);
+
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND &&
+	    props->state == IB_PORT_ACTIVE)
+		props->state = get_upper_port_state(hr_dev);
+
 	props->phys_state = props->state == IB_PORT_ACTIVE ?
 				    IB_PORT_PHYS_STATE_LINK_UP :
 				    IB_PORT_PHYS_STATE_DISABLED;
@@ -503,12 +527,14 @@ static int hns_roce_alloc_ucontext(struct ib_ucontext *uctx,
 
 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_RQ_INLINE) {
 		context->config |= ucmd.config & HNS_ROCE_RQ_INLINE_FLAGS;
-		resp.config |= HNS_ROCE_RSP_RQ_INLINE_FLAGS;
+		if (context->config & HNS_ROCE_RQ_INLINE_FLAGS)
+			resp.config |= HNS_ROCE_RSP_RQ_INLINE_FLAGS;
 	}
 
 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_CQE_INLINE) {
 		context->config |= ucmd.config & HNS_ROCE_CQE_INLINE_FLAGS;
-		resp.config |= HNS_ROCE_RSP_CQE_INLINE_FLAGS;
+		if (context->config & HNS_ROCE_CQE_INLINE_FLAGS)
+			resp.config |= HNS_ROCE_RSP_CQE_INLINE_FLAGS;
 	}
 
 	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DCA_MODE) {
@@ -822,12 +848,24 @@ static int hns_roce_get_hw_stats(struct ib_device *device,
 	return hw_counters + HNS_ROCE_DFX_CNT_TOTAL;
 }
 
-static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev)
+static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev,
+				       bool bond_cleanup)
 {
 	struct hns_roce_ib_iboe *iboe = &hr_dev->iboe;
+	struct hns_roce_v2_priv *priv = hr_dev->priv;
+	struct hns_roce_bond_group *bond_grp;
 
-	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND)
-		hns_roce_cleanup_bond(hr_dev);
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) {
+		unregister_netdevice_notifier(&hr_dev->bond_nb);
+		bond_grp = hns_roce_get_bond_grp(hr_dev);
+		if (bond_grp) {
+			if (bond_cleanup)
+				hns_roce_cleanup_bond(bond_grp);
+			else if (priv->handle->rinfo.reset_state ==
+				 HNS_ROCE_STATE_RST_UNINIT)
+				bond_grp->main_hr_dev = NULL;
+		}
+	}
 
 	hr_dev->active = false;
 	unregister_netdevice_notifier(&iboe->nb);
@@ -1444,10 +1482,10 @@ int hns_roce_init(struct hns_roce_dev *hr_dev)
 	return ret;
 }
 
-void hns_roce_exit(struct hns_roce_dev *hr_dev)
+void hns_roce_exit(struct hns_roce_dev *hr_dev, bool bond_cleanup)
 {
 	hns_roce_unregister_sysfs(hr_dev);
-	hns_roce_unregister_device(hr_dev);
+	hns_roce_unregister_device(hr_dev, bond_cleanup);
 	hns_roce_unregister_debugfs(hr_dev);
 
 	if (hr_dev->hw->hw_exit)