diff --git a/drivers/Makefile b/drivers/Makefile
index e06f7f633f73ff285c64e19b89041d17e2be23ce..8189b1edec005dfc93cb7d5d720d4a40af61a97f 100644
--- a/drivers/Makefile
+++ b/drivers/Makefile
@@ -16,10 +16,7 @@ obj-$(CONFIG_PINCTRL)		+= pinctrl/
 obj-$(CONFIG_GPIOLIB)		+= gpio/
 obj-y				+= pwm/
 
-obj-$(CONFIG_PCI)		+= pci/
-obj-$(CONFIG_PCI_ENDPOINT)	+= pci/endpoint/
-# PCI dwc controller drivers
-obj-y				+= pci/dwc/
+obj-y				+= pci/
 
 obj-$(CONFIG_PARISC)		+= parisc/
 obj-$(CONFIG_RAPIDIO)		+= rapidio/
diff --git a/drivers/pci/Kconfig b/drivers/pci/Kconfig
index bda151788f3f723198be6c3401393f3876f5babf..7eeb969ab86a231b030db0a3d41131336f509f68 100644
--- a/drivers/pci/Kconfig
+++ b/drivers/pci/Kconfig
@@ -125,6 +125,7 @@ config PCI_PASID
 
 config PCI_LABEL
 	def_bool y if (DMI || ACPI)
+	depends on PCI
 	select NLS
 
 config PCI_HYPERV
diff --git a/drivers/pci/Makefile b/drivers/pci/Makefile
index c7819b973df7e4bbd07cd49e3bf29eea92eafa07..ddb5aa6640d70912d6a46fec75ef2d12c4d2907e 100644
--- a/drivers/pci/Makefile
+++ b/drivers/pci/Makefile
@@ -3,12 +3,15 @@
 # Makefile for the PCI bus specific drivers.
 #
 
-obj-y		+= access.o bus.o probe.o host-bridge.o remove.o pci.o \
+obj-$(CONFIG_PCI)	+= access.o bus.o probe.o host-bridge.o remove.o pci.o \
 			pci-driver.o search.o pci-sysfs.o rom.o setup-res.o \
 			irq.o vpd.o setup-bus.o vc.o mmap.o setup-irq.o
 
+ifdef CONFIG_PCI
 obj-$(CONFIG_PROC_FS) += proc.o
 obj-$(CONFIG_SYSFS) += slot.o
+obj-$(CONFIG_OF) += of.o
+endif
 
 obj-$(CONFIG_PCI_QUIRKS) += quirks.o
 
@@ -44,10 +47,14 @@ obj-$(CONFIG_PCI_ECAM) += ecam.o
 
 obj-$(CONFIG_XEN_PCIDEV_FRONTEND) += xen-pcifront.o
 
-obj-$(CONFIG_OF) += of.o
-
 ccflags-$(CONFIG_PCI_DEBUG) := -DDEBUG
 
 # PCI host controller drivers
 obj-y += host/
 obj-y += switch/
+
+obj-$(CONFIG_PCI_ENDPOINT)	+= endpoint/
+
+# Endpoint library must be initialized before its users
+# pcie-hisi.o quirks are needed even without CONFIG_PCIE_DW
+obj-y				+= dwc/
diff --git a/drivers/pci/dwc/Makefile b/drivers/pci/dwc/Makefile
index 41ba499c96ee97c11c4d5fda39abf50f2e0f7692..cb35e45cd18685d76742804d6110da0556ef8a84 100644
--- a/drivers/pci/dwc/Makefile
+++ b/drivers/pci/dwc/Makefile
@@ -27,4 +27,6 @@ obj-$(CONFIG_PCIE_HISI_STB) += pcie-histb.o
 # ARM64 and use internal ifdefs to only build the pieces we need
 # depending on whether ACPI, the DT driver, or both are enabled.
 
+ifdef CONFIG_PCI
 obj-$(CONFIG_ARM64) += pcie-hisi.o
+endif
diff --git a/drivers/pci/host/Makefile b/drivers/pci/host/Makefile
index 34ec1d88f961b3185e0f621f4d96eaa5d75a655f..3b105919086750b0db88e1ff4ecee11f5e37990c 100644
--- a/drivers/pci/host/Makefile
+++ b/drivers/pci/host/Makefile
@@ -34,6 +34,8 @@ obj-$(CONFIG_VMD) += vmd.o
 # ARM64 and use internal ifdefs to only build the pieces we need
 # depending on whether ACPI, the DT driver, or both are enabled.
 
+ifdef CONFIG_PCI
 obj-$(CONFIG_ARM64) += pci-thunder-ecam.o
 obj-$(CONFIG_ARM64) += pci-thunder-pem.o
 obj-$(CONFIG_ARM64) += pci-xgene.o
+endif