diff --git a/net/dsa/slave.c b/net/dsa/slave.c
index 528380a3e296001b9c727e9f35e07f403d7f050b..da64ba9d825d1ae1841b6d337ed994fb9391fc48 100644
--- a/net/dsa/slave.c
+++ b/net/dsa/slave.c
@@ -555,6 +555,9 @@ static void dsa_slave_phy_setup(struct dsa_slave_priv *p,
 	 */
 	if (!p->phy) {
 		p->phy = ds->slave_mii_bus->phy_map[p->port];
+		if (!p->phy)
+			return;
+
 		phy_connect_direct(slave_dev, p->phy, dsa_slave_adjust_link,
 				   p->phy_interface);
 	} else {