diff --git a/arch/x86/kernel/ldt.c b/arch/x86/kernel/ldt.c
index 500e90e44f8669b0249dc0b16d815c4f77d269fc..26d713ecad34a847e650638d02442eda04379f5e 100644
--- a/arch/x86/kernel/ldt.c
+++ b/arch/x86/kernel/ldt.c
@@ -426,7 +426,8 @@ static int write_ldt(void __user *ptr, unsigned long bytecount, int oldmode)
 		 * already installed then the PTE page is already
 		 * populated. Mop up a half populated page table.
 		 */
-		free_ldt_pgtables(mm);
+		if (!WARN_ON_ONCE(old_ldt))
+			free_ldt_pgtables(mm);
 		free_ldt_struct(new_ldt);
 		goto out_unlock;
 	}