RDMA/hns: Remove the struct member 'bond_grp' from hns_roce_dev

driver inclusion
category: bugfix
bugzilla: https://gitee.com/openeuler/kernel/issues/I6Z4E9

---------------------------------------------------------------

Currently, bond_grp is attached to only one hr_dev corresponding to a
slave in RoCE bonding, which is called main_hr_dev in the driver. When
a non-main_hr_dev try to obtain its bond_grp, the driver has to find
the main_hr_dev, and then obtain the bond_grp, which leads to a
complicated code.

Applying this patch, bond_grp is removed from struct hns_roce_dev. hr_dev
can obtain bond_grp by XArray, where die_info and bond_grp are stored
according to bus number. With this change, hr_dev can get its bond_grp
directly without depending on main_hr_dev, and the code logic can be
simplified.
Signed-off-by: NJunxian Huang <huangjunxian6@hisilicon.com>

drivers/infiniband/hw/hns/hns_roce_bond.c

@@ -27,30 +27,74 @@ static struct hns_roce_dev *hns_roce_get_hrdev_by_netdev(struct net_device *net_
 	return hr_dev;
 }
 
-struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev)
+static struct net_device *get_upper_dev_from_ndev(struct net_device *net_dev)
 {
-	struct hns_roce_bond_group *bond_grp = NULL;
 	struct net_device *upper_dev;
-	struct net_device *net_dev;
-
-	if (!netif_is_lag_port(hr_dev->iboe.netdevs[0]))
-		return NULL;
 
 	rcu_read_lock();
+	upper_dev = netdev_master_upper_dev_get_rcu(net_dev);
+	rcu_read_unlock();
 
-	upper_dev = netdev_master_upper_dev_get_rcu(hr_dev->iboe.netdevs[0]);
+	return upper_dev;
+}
 
-	for_each_netdev_in_bond_rcu(upper_dev, net_dev) {
-		hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);
-		if (hr_dev && hr_dev->bond_grp) {
-			bond_grp = hr_dev->bond_grp;
-			break;
-		}
-	}
+static bool is_netdev_bond_slave(struct net_device *net_dev,
+				 struct hns_roce_bond_group *bond_grp)
+{
+	int i;
 
-	rcu_read_unlock();
+	if (!net_dev || !bond_grp)
+		return false;
 
-	return bond_grp;
+	for (i = 0; i < ROCE_BOND_FUNC_MAX; i++)
+		if (net_dev == bond_grp->bond_func_info[i].net_dev)
+			return true;
+
+	return false;
+}
+
+static bool is_hrdev_bond_slave(struct hns_roce_dev *hr_dev,
+				struct net_device *upper_dev)
+{
+	struct hns_roce_bond_group *bond_grp;
+
+	if (!hr_dev || !upper_dev)
+		return false;
+
+	if (!netif_is_lag_master(upper_dev))
+		return false;
+
+	if (upper_dev == get_upper_dev_from_ndev(hr_dev->iboe.netdevs[0]))
+		return true;
+
+	bond_grp = hns_roce_get_bond_grp(hr_dev);
+	if (bond_grp && upper_dev == bond_grp->upper_dev)
+		return true;
+
+	return false;
+}
+
+struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev)
+{
+	struct hns_roce_die_info *die_info =
+		xa_load(&roce_bond_xa, hr_dev->pci_dev->bus->number);
+	struct hns_roce_bond_group *bond_grp;
+	int i;
+
+	if (!die_info)
+		return NULL;
+
+	for (i = 0; i < ROCE_BOND_NUM_MAX; i++) {
+		bond_grp = die_info->bgrps[i];
+		if (!bond_grp)
+			continue;
+		if (is_netdev_bond_slave(hr_dev->iboe.netdevs[0], bond_grp) ||
+		    bond_grp->upper_dev ==
+		    get_upper_dev_from_ndev(hr_dev->iboe.netdevs[0]))
+			return bond_grp;
+	}
+
+	return NULL;
 }
 
 bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev)
@@ -67,18 +111,15 @@ bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev)
 
 struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev)
 {
-	struct hns_roce_bond_group *bond_grp = hr_dev->bond_grp;
+	struct hns_roce_bond_group *bond_grp = hns_roce_get_bond_grp(hr_dev);
 	struct net_device *net_dev = NULL;
 	int i;
 
 	if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND))
 		return NULL;
 
-	if (!bond_grp) {
-		bond_grp = hns_roce_get_bond_grp(hr_dev);
-		if (!bond_grp)
-			return NULL;
-	}
+	if (!bond_grp)
+		return NULL;
 
 	mutex_lock(&bond_grp->bond_mutex);
 
@@ -162,7 +203,7 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp)
 {
 	u8 main_func_idx = PCI_FUNC(bond_grp->main_hr_dev->pci_dev->devfn);
 	struct net_device *main_net_dev = bond_grp->main_net_dev;
-	struct hns_roce_dev *hr_dev;
+	struct hns_roce_dev *hr_dev = NULL;
 	struct net_device *net_dev;
 	int ret;
 	int i;
@@ -185,10 +226,8 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp)
 		if (net_dev && net_dev != main_net_dev) {
 			hr_dev = hns_roce_bond_init_client(bond_grp, i);
 			if (hr_dev) {
-				bond_grp->main_hr_dev->bond_grp = NULL;
 				bond_grp->main_hr_dev = hr_dev;
 				bond_grp->main_net_dev = net_dev;
-				hr_dev->bond_grp = bond_grp;
 				break;
 			}
 		}
@@ -198,7 +237,7 @@ static void hns_roce_set_bond(struct hns_roce_bond_group *bond_grp)
 
 	hns_roce_bond_uninit_client(bond_grp, main_func_idx);
 	hns_roce_bond_get_active_slave(bond_grp);
-	ret = hns_roce_cmd_bond(hr_dev, HNS_ROCE_SET_BOND);
+	ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_SET_BOND);
 	if (ret) {
 		ibdev_err(&hr_dev->ib_dev, "failed to set RoCE bond!\n");
 		return;
@@ -226,7 +265,7 @@ static void hns_roce_clear_bond(struct hns_roce_bond_group *bond_grp)
 			hns_roce_bond_init_client(bond_grp, i);
 	}
 
-	ret = hns_roce_cmd_bond(bond_grp->main_hr_dev, HNS_ROCE_CLEAR_BOND);
+	ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CLEAR_BOND);
 	if (ret)
 		return;
 	handle = bond_grp->bond_func_info[main_func_idx].handle;
@@ -251,7 +290,7 @@ static void hns_roce_slave_changestate(struct hns_roce_bond_group *bond_grp)
 
 	hns_roce_bond_get_active_slave(bond_grp);
 
-	ret = hns_roce_cmd_bond(bond_grp->main_hr_dev, HNS_ROCE_CHANGE_BOND);
+	ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND);
 	if (ret) {
 		ibdev_err(&bond_grp->main_hr_dev->ib_dev,
 			  "failed to change RoCE bond slave state!\n");
@@ -277,7 +316,7 @@ static void hns_roce_slave_inc(struct hns_roce_bond_group *bond_grp)
 	}
 
 	hns_roce_bond_get_active_slave(bond_grp);
-	ret = hns_roce_cmd_bond(bond_grp->main_hr_dev, HNS_ROCE_CHANGE_BOND);
+	ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND);
 	if (ret) {
 		ibdev_err(&bond_grp->main_hr_dev->ib_dev,
 			  "failed to increase RoCE bond slave!\n");
@@ -300,7 +339,7 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp)
 	int i;
 
 	if (dec_slave_map & (1 << main_func_idx)) {
-		hns_roce_cmd_bond(hr_dev, HNS_ROCE_CLEAR_BOND);
+		hns_roce_cmd_bond(bond_grp, HNS_ROCE_CLEAR_BOND);
 		for (i = 0; i < ROCE_BOND_FUNC_MAX; i++) {
 			net_dev = bond_grp->bond_func_info[i].net_dev;
 			if (!(dec_slave_map & (1 << i)) && net_dev) {
@@ -309,7 +348,6 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp)
 				if (hr_dev) {
 					bond_grp->main_hr_dev = hr_dev;
 					bond_grp->main_net_dev = net_dev;
-					hr_dev->bond_grp = bond_grp;
 					break;
 				}
 			}
@@ -328,10 +366,9 @@ static void hns_roce_slave_dec(struct hns_roce_bond_group *bond_grp)
 
 	hns_roce_bond_get_active_slave(bond_grp);
 	if (bond_grp->slave_map_diff & (1 << main_func_idx))
-		ret = hns_roce_cmd_bond(hr_dev, HNS_ROCE_SET_BOND);
+		ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_SET_BOND);
 	else
-		ret = hns_roce_cmd_bond(bond_grp->main_hr_dev,
-					HNS_ROCE_CHANGE_BOND);
+		ret = hns_roce_cmd_bond(bond_grp, HNS_ROCE_CHANGE_BOND);
 	if (ret) {
 		ibdev_err(&bond_grp->main_hr_dev->ib_dev,
 			  "failed to decrease RoCE bond slave!\n");
@@ -465,32 +502,31 @@ static int remove_bond_id(int bus_num, u8 bond_id)
 	return 0;
 }
 
-void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev)
+void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev,
+			   struct hns_roce_bond_group *bond_grp)
 {
 	int ret;
 
 	unregister_netdevice_notifier(&hr_dev->bond_nb);
 
-	if (hr_dev->bond_grp && hr_dev == hr_dev->bond_grp->main_hr_dev) {
-		cancel_delayed_work(&hr_dev->bond_grp->bond_work);
+	if (bond_grp && hr_dev == bond_grp->main_hr_dev) {
+		cancel_delayed_work(&bond_grp->bond_work);
 		ret = remove_bond_id(hr_dev->pci_dev->bus->number,
-				     hr_dev->bond_grp->bond_id);
+				     bond_grp->bond_id);
 		if (ret)
 			ibdev_err(&hr_dev->ib_dev,
 				  "failed to remove bond ID %d, ret = %d.\n",
-				  hr_dev->bond_grp->bond_id, ret);
-		kfree(hr_dev->bond_grp);
+				  bond_grp->bond_id, ret);
+		kfree(bond_grp);
 	}
-
-	hr_dev->bond_grp = NULL;
 }
 
 static bool hns_roce_bond_lowerstate_event(struct hns_roce_dev *hr_dev,
+					   struct hns_roce_bond_group *bond_grp,
 					   struct netdev_notifier_changelowerstate_info *info)
 {
 	struct net_device *net_dev =
 		netdev_notifier_info_to_dev((struct netdev_notifier_info *)info);
-	struct hns_roce_bond_group *bond_grp = hr_dev->bond_grp;
 	struct netdev_lag_lower_state_info *bond_lower_info;
 	int i;
 
@@ -568,10 +604,9 @@ static void hns_roce_bond_info_record(struct hns_roce_bond_group *bond_grp,
 	rcu_read_unlock();
 }
 
-static bool hns_roce_bond_upper_event(struct hns_roce_dev *hr_dev,
+static bool hns_roce_bond_upper_event(struct hns_roce_bond_group *bond_grp,
 				      struct netdev_notifier_changeupper_info *info)
 {
-	struct hns_roce_bond_group *bond_grp = hr_dev->bond_grp;
 	struct netdev_lag_upper_info *bond_upper_info = NULL;
 	struct net_device *upper_dev = info->upper_dev;
 	bool changed = false;
@@ -646,47 +681,12 @@ static struct hns_roce_bond_group *hns_roce_alloc_bond_grp(struct hns_roce_dev *
 	return bond_grp;
 }
 
-static struct net_device *get_upper_dev_from_ndev(struct net_device *net_dev)
-{
-	struct net_device *upper_dev;
-
-	rcu_read_lock();
-	upper_dev = netdev_master_upper_dev_get_rcu(net_dev);
-	rcu_read_unlock();
-
-	return upper_dev;
-}
-
-static bool hns_roce_is_slave(struct net_device *upper_dev,
-			      struct hns_roce_dev *hr_dev)
-{
-	return (hr_dev->bond_grp && upper_dev == hr_dev->bond_grp->upper_dev) ||
-		upper_dev == get_upper_dev_from_ndev(hr_dev->iboe.netdevs[0]);
-}
-
-static bool hns_roce_is_bond_grp_exist(struct net_device *upper_dev)
-{
-	struct hns_roce_dev *hr_dev;
-	struct net_device *net_dev;
-
-	rcu_read_lock();
-	for_each_netdev_in_bond_rcu(upper_dev, net_dev) {
-		hr_dev = hns_roce_get_hrdev_by_netdev(net_dev);
-		if (hr_dev && hr_dev->bond_grp) {
-			rcu_read_unlock();
-			return true;
-		}
-	}
-	rcu_read_unlock();
-
-	return false;
-}
-
 static enum bond_support_type
 	check_bond_support(struct hns_roce_dev *hr_dev,
 			   struct net_device **upper_dev,
 			   struct netdev_notifier_changeupper_info *info)
 {
+	struct hns_roce_bond_group *bond_grp = hns_roce_get_bond_grp(hr_dev);
 	struct netdev_lag_upper_info *bond_upper_info = NULL;
 	bool bond_grp_exist = false;
 	struct net_device *net_dev;
@@ -695,7 +695,7 @@ static enum bond_support_type
 	int bus_num = -1;
 
 	*upper_dev = info->upper_dev;
-	if (hr_dev->bond_grp || hns_roce_is_bond_grp_exist(*upper_dev))
+	if (bond_grp && *upper_dev == bond_grp->upper_dev)
 		bond_grp_exist = true;
 
 	if (!info->linking && !bond_grp_exist)
@@ -739,6 +739,7 @@ int hns_roce_bond_event(struct notifier_block *self,
 	struct hns_roce_dev *hr_dev =
 		container_of(self, struct hns_roce_dev, bond_nb);
 	enum bond_support_type support = BOND_SUPPORT;
+	struct hns_roce_bond_group *bond_grp;
 	struct net_device *upper_dev;
 	bool changed;
 
@@ -753,34 +754,34 @@ int hns_roce_bond_event(struct notifier_block *self,
 		upper_dev = get_upper_dev_from_ndev(net_dev);
 	}
 
-	if (upper_dev && !hns_roce_is_slave(upper_dev, hr_dev))
+	if (upper_dev && !is_hrdev_bond_slave(hr_dev, upper_dev))
 		return NOTIFY_DONE;
 	else if (!upper_dev && hr_dev != hns_roce_get_hrdev_by_netdev(net_dev))
 		return NOTIFY_DONE;
 
+	bond_grp = hns_roce_get_bond_grp(hr_dev);
 	if (event == NETDEV_CHANGEUPPER) {
-		if (!hr_dev->bond_grp) {
-			if (hns_roce_is_bond_grp_exist(upper_dev))
-				return NOTIFY_DONE;
-			hr_dev->bond_grp = hns_roce_alloc_bond_grp(hr_dev,
-								   upper_dev);
-			if (!hr_dev->bond_grp) {
+		if (!bond_grp) {
+			bond_grp = hns_roce_alloc_bond_grp(hr_dev, upper_dev);
+			if (!bond_grp) {
 				ibdev_err(&hr_dev->ib_dev,
 					  "failed to alloc RoCE bond_grp!\n");
 				return NOTIFY_DONE;
 			}
+		} else if (hr_dev != bond_grp->main_hr_dev) {
+			return NOTIFY_DONE;
 		}
 		if (support == BOND_EXISTING_NOT_SUPPORT) {
-			hr_dev->bond_grp->bond_ready = false;
-			hns_roce_queue_bond_work(hr_dev->bond_grp, HZ);
+			bond_grp->bond_ready = false;
+			hns_roce_queue_bond_work(bond_grp, HZ);
 			return NOTIFY_DONE;
 		}
-		changed = hns_roce_bond_upper_event(hr_dev, ptr);
+		changed = hns_roce_bond_upper_event(bond_grp, ptr);
 	} else {
-		changed = hns_roce_bond_lowerstate_event(hr_dev, ptr);
+		changed = hns_roce_bond_lowerstate_event(hr_dev, bond_grp, ptr);
 	}
 	if (changed)
-		hns_roce_queue_bond_work(hr_dev->bond_grp, HZ);
+		hns_roce_queue_bond_work(bond_grp, HZ);
 
 	return NOTIFY_DONE;
 }

drivers/infiniband/hw/hns/hns_roce_bond.h

@@ -80,7 +80,8 @@ struct hns_roce_die_info {
 int hns_roce_bond_init(struct hns_roce_dev *hr_dev);
 int hns_roce_bond_event(struct notifier_block *self,
 			unsigned long event, void *ptr);
-void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev);
+void hns_roce_cleanup_bond(struct hns_roce_dev *hr_dev,
+			   struct hns_roce_bond_group *bond_grp);
 bool hns_roce_bond_is_active(struct hns_roce_dev *hr_dev);
 struct net_device *hns_roce_get_bond_netdev(struct hns_roce_dev *hr_dev);
 struct hns_roce_bond_group *hns_roce_get_bond_grp(struct hns_roce_dev *hr_dev);

drivers/infiniband/hw/hns/hns_roce_device.h

@@ -1124,7 +1124,6 @@ struct hns_roce_dev {
 	u64 dwqe_page;
 
 	struct notifier_block bond_nb;
-	struct hns_roce_bond_group *bond_grp;
 	struct netdev_lag_lower_state_info slave_state;
 	struct hns_roce_port port_data[HNS_ROCE_MAX_PORTS];
 	atomic64_t *dfx_cnt;

drivers/infiniband/hw/hns/hns_roce_hw_v2.c

@@ -1493,7 +1493,7 @@ static inline enum hns_roce_opcode_type
 		return HNS_ROCE_OPC_CLEAR_BOND_INFO;
 }
 
-int hns_roce_cmd_bond(struct hns_roce_dev *hr_dev,
+int hns_roce_cmd_bond(struct hns_roce_bond_group *bond_grp,
 		      enum hns_roce_bond_cmd_type bond_type)
 {
 	enum hns_roce_opcode_type opcode = get_bond_opcode(bond_type);
@@ -1504,33 +1504,33 @@ int hns_roce_cmd_bond(struct hns_roce_dev *hr_dev,
 	slave_info = (struct hns_roce_bond_info *)desc.data;
 	hns_roce_cmq_setup_basic_desc(&desc, opcode, false);
 
-	slave_info->bond_id = cpu_to_le32(hr_dev->bond_grp->bond_id);
+	slave_info->bond_id = cpu_to_le32(bond_grp->bond_id);
 	if (bond_type == HNS_ROCE_CLEAR_BOND)
 		goto out;
 
-	if (hr_dev->bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
+	if (bond_grp->tx_type == NETDEV_LAG_TX_TYPE_ACTIVEBACKUP) {
 		slave_info->bond_mode = cpu_to_le32(BOND_MODE_1);
-		if (hr_dev->bond_grp->active_slave_num != 1)
-			ibdev_err(&hr_dev->ib_dev,
+		if (bond_grp->active_slave_num != 1)
+			ibdev_err(&bond_grp->main_hr_dev->ib_dev,
 				  "active slave cnt(%d) in Mode 1 is invalid.\n",
-				  hr_dev->bond_grp->active_slave_num);
+				  bond_grp->active_slave_num);
 	} else {
 		slave_info->bond_mode = cpu_to_le32(BOND_MODE_2_4);
 		slave_info->hash_policy =
-			cpu_to_le32(hr_dev->bond_grp->bond->params.xmit_policy);
+			cpu_to_le32(bond_grp->bond->params.xmit_policy);
 	}
 
 	slave_info->active_slave_cnt =
-		cpu_to_le32(hr_dev->bond_grp->active_slave_num);
+		cpu_to_le32(bond_grp->active_slave_num);
 	slave_info->active_slave_mask =
-		cpu_to_le32(hr_dev->bond_grp->active_slave_map);
+		cpu_to_le32(bond_grp->active_slave_map);
 	slave_info->slave_mask =
-		cpu_to_le32(hr_dev->bond_grp->slave_map);
+		cpu_to_le32(bond_grp->slave_map);
 
 out:
-	ret = hns_roce_cmq_send(hr_dev, &desc, 1);
+	ret = hns_roce_cmq_send(bond_grp->main_hr_dev, &desc, 1);
 	if (ret)
-		ibdev_err(&hr_dev->ib_dev,
+		ibdev_err(&bond_grp->main_hr_dev->ib_dev,
 			  "cmq bond type(%d) failed, ret = %d.\n",
 			  bond_type, ret);
 

drivers/infiniband/hw/hns/hns_roce_hw_v2.h

@@ -1602,7 +1602,7 @@ int hns_roce_v2_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata);
 int hns_roce_v2_destroy_qp_common(struct hns_roce_dev *hr_dev,
 				  struct hns_roce_qp *hr_qp,
 				  struct ib_udata *udata);
-int hns_roce_cmd_bond(struct hns_roce_dev *hr_dev,
+int hns_roce_cmd_bond(struct hns_roce_bond_group *bond_grp,
 		      enum hns_roce_bond_cmd_type bond_type);
 
 static inline void hns_roce_write64(struct hns_roce_dev *hr_dev, __le32 val[2],

drivers/infiniband/hw/hns/hns_roce_main.c

@@ -825,9 +825,13 @@ static int hns_roce_get_hw_stats(struct ib_device *device,
 static void hns_roce_unregister_device(struct hns_roce_dev *hr_dev)
 {
 	struct hns_roce_ib_iboe *iboe = &hr_dev->iboe;
+	struct hns_roce_bond_group *bond_grp;
 
-	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND)
-		hns_roce_cleanup_bond(hr_dev);
+	if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_BOND) {
+		bond_grp = hns_roce_get_bond_grp(hr_dev);
+		if (bond_grp)
+			hns_roce_cleanup_bond(hr_dev, bond_grp);
+	}
 
 	hr_dev->active = false;
 	unregister_netdevice_notifier(&iboe->nb);