diff --git a/drivers/usb/host/Makefile b/drivers/usb/host/Makefile
index 65b0b6a5859977c087b046f1450d8815a8f24c37..972a74a6f428faa816d2c2cac587a4f468733c3d 100644
--- a/drivers/usb/host/Makefile
+++ b/drivers/usb/host/Makefile
@@ -24,7 +24,9 @@ endif
 
 obj-$(CONFIG_USB_WHCI_HCD)	+= whci/
 
-obj-$(CONFIG_PCI)		+= pci-quirks.o
+ifneq ($(CONFIG_USB), )
+	obj-$(CONFIG_PCI)	+= pci-quirks.o
+endif
 
 obj-$(CONFIG_USB_XHCI_PCI)	+= xhci-pci.o
 obj-$(CONFIG_USB_XHCI_PLATFORM) += xhci-plat-hcd.o