diff --git a/drivers/serial/Makefile b/drivers/serial/Makefile
index cac9a8b31212f4a4fe34cdd732f6ea74a140bb39..6937ef96286896891829773b5082cdf498d9e6b1 100644
--- a/drivers/serial/Makefile
+++ b/drivers/serial/Makefile
@@ -5,11 +5,27 @@
 # SPDX-License-Identifier:	GPL-2.0+
 #
 
+ifdef CONFIG_SPL_BUILD
+
+ifeq ($(CONFIG_$(SPL_TPL_)BUILD)$(CONFIG_$(SPL_TPL_)DM_SERIAL),yy)
+obj-y += serial-uclass.o
+else
+obj-y += serial.o
+endif
+
+else
+
 ifdef CONFIG_DM_SERIAL
-obj-$(CONFIG_$(SPL_TPL_)DM_SERIAL) += serial-uclass.o
-obj-$(CONFIG_PL01X_SERIAL) += serial_pl01x.o
+obj-y += serial-uclass.o
 else
 obj-y += serial.o
+endif
+
+endif
+
+ifdef CONFIG_DM_SERIAL
+obj-$(CONFIG_PL01X_SERIAL) += serial_pl01x.o
+else
 obj-$(CONFIG_PL010_SERIAL) += serial_pl01x.o
 obj-$(CONFIG_PL011_SERIAL) += serial_pl01x.o
 obj-$(CONFIG_SYS_NS16550_SERIAL) += serial_ns16550.o