diff --git a/Makefile b/Makefile
index 50bb50defe187c1c9a7ec3ebd50a4f2366f2eccd..f4238b2739e8c8f5feb11922929c881c218894da 100644
--- a/Makefile
+++ b/Makefile
@@ -1512,8 +1512,9 @@ quiet_cmd_depmod = DEPMOD  $(KERNELRELEASE)
 	fi
 
 # Create temporary dir for module support files
-cmd_crmodverdir = $(Q)mkdir -p $(MODVERDIR); rm -f $(MODVERDIR)/*
-
+# clean it up only when building all modules
+cmd_crmodverdir = $(Q)mkdir -p $(MODVERDIR) \
+                  $(if $(KBUILD_MODULES),; rm -f $(MODVERDIR)/*)
 
 a_flags = -Wp,-MD,$(depfile) $(KBUILD_AFLAGS) $(AFLAGS_KERNEL) \
 	  $(NOSTDINC_FLAGS) $(KBUILD_CPPFLAGS) \