diff --git a/drivers/usb/Makefile b/drivers/usb/Makefile
index c691eea515375ac1e38f2afaec9ebc8509a2c256..f5ed3d75fa5a2ce615f32b10b7f0461f839e5757 100644
--- a/drivers/usb/Makefile
+++ b/drivers/usb/Makefile
@@ -46,7 +46,7 @@ obj-$(CONFIG_USB_MICROTEK)	+= image/
 obj-$(CONFIG_USB_SERIAL)	+= serial/
 
 obj-$(CONFIG_USB)		+= misc/
-obj-$(CONFIG_USB)		+= phy/
+obj-$(CONFIG_USB_COMMON)	+= phy/
 obj-$(CONFIG_EARLY_PRINTK_DBGP)	+= early/
 
 obj-$(CONFIG_USB_ATM)		+= atm/
diff --git a/drivers/usb/phy/Kconfig b/drivers/usb/phy/Kconfig
index 3cfabcba7447d2ccaf55d77038b43357d8cbaeec..e7cf84f0751a2a6f1d8ea27614c59abf8a7303fa 100644
--- a/drivers/usb/phy/Kconfig
+++ b/drivers/usb/phy/Kconfig
@@ -2,11 +2,11 @@
 # Physical Layer USB driver configuration
 #
 comment "USB Physical Layer drivers"
-	depends on USB
+	depends on USB || USB_GADGET
 
 config USB_ISP1301
 	tristate "NXP ISP1301 USB transceiver support"
-	depends on USB
+	depends on USB || USB_GADGET
 	depends on I2C
 	help
 	  Say Y here to add support for the NXP ISP1301 USB transceiver driver.