diff --git a/drivers/usb/musb/musb_gadget.c b/drivers/usb/musb/musb_gadget.c
index 66c8b32b16bb49ba89ac041fd66b2f8bb38b6848..6cb9514ef340590590717b34aef3f7a09c3fab65 100644
--- a/drivers/usb/musb/musb_gadget.c
+++ b/drivers/usb/musb/musb_gadget.c
@@ -1854,6 +1854,8 @@ static int musb_gadget_start(struct usb_gadget *g,
 
 	if (musb->xceiv)
 		otg_set_peripheral(musb->xceiv->otg, &musb->g);
+	else
+		phy_set_mode(musb->phy, PHY_MODE_USB_DEVICE);
 
 	musb_set_state(musb, OTG_STATE_B_IDLE);
 	spin_unlock_irqrestore(&musb->lock, flags);
@@ -1905,6 +1907,8 @@ static int musb_gadget_stop(struct usb_gadget *g)
 
 	if (musb->xceiv)
 		otg_set_peripheral(musb->xceiv->otg, NULL);
+	else
+		phy_set_mode(musb->phy, PHY_MODE_INVALID);
 
 	musb->is_active = 0;
 	musb->gadget_driver = NULL;
diff --git a/drivers/usb/musb/musb_host.c b/drivers/usb/musb/musb_host.c
index 8ad39ecd3b6fcebf2dd6fe13662cad6909ac0549..a02c29216955a551cb5772dc70913f97e13eddb3 100644
--- a/drivers/usb/musb/musb_host.c
+++ b/drivers/usb/musb/musb_host.c
@@ -2726,6 +2726,8 @@ int musb_host_setup(struct musb *musb, int power_budget)
 	if (musb->xceiv) {
 		otg_set_host(musb->xceiv->otg, &hcd->self);
 		musb->xceiv->otg->host = &hcd->self;
+	} else {
+		phy_set_mode(musb->phy, PHY_MODE_USB_HOST);
 	}
 
 	/* don't support otg protocols */