diff --git a/mm/memory.c b/mm/memory.c
index 9b135a35f99c27bffddbcdd131097ded81e67ba2..1893751cf6c0498a310c7ae93bf44b929ccf6fbd 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -375,15 +375,6 @@ void tlb_remove_table(struct mmu_gather *tlb, void *table)
 {
 	struct mmu_table_batch **batch = &tlb->batch;
 
-	/*
-	 * When there's less then two users of this mm there cannot be a
-	 * concurrent page-table walk.
-	 */
-	if (atomic_read(&tlb->mm->mm_users) < 2) {
-		__tlb_remove_table(table);
-		return;
-	}
-
 	if (*batch == NULL) {
 		*batch = (struct mmu_table_batch *)__get_free_page(GFP_NOWAIT | __GFP_NOWARN);
 		if (*batch == NULL) {