diff --git a/net/dsa/legacy.c b/net/dsa/legacy.c
index b6c88fd33d4f0eab2b5313b593f70aecdcaf1ed7..93c1c43bcc58352c9aa5d9aa4b9e06cfecc40f91 100644
--- a/net/dsa/legacy.c
+++ b/net/dsa/legacy.c
@@ -272,10 +272,6 @@ static void dsa_switch_destroy(struct dsa_switch *ds)
 		if (!(dsa_is_cpu_port(ds, port) || dsa_is_dsa_port(ds, port)))
 			continue;
 		dsa_cpu_dsa_destroy(&ds->ports[port]);
-
-		/* Clearing a bit which is not set does no harm */
-		ds->cpu_port_mask |= ~(1 << port);
-		ds->dsa_port_mask |= ~(1 << port);
 	}
 
 	if (ds->slave_mii_bus && ds->ops->phy_read)