diff --git a/fs/aio.c b/fs/aio.c
index 06d7d4390fe7dd8562e8433ce3a29b2b9ee4e241..4f641abac3c0927f2036f81f3f70106a57a09e22 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -567,6 +567,10 @@ static void use_mm(struct mm_struct *mm)
 	atomic_inc(&mm->mm_count);
 	tsk->mm = mm;
 	tsk->active_mm = mm;
+	/*
+	 * Note that on UML this *requires* PF_BORROWED_MM to be set, otherwise
+	 * it won't work. Update it accordingly if you change it here
+	 */
 	activate_mm(active_mm, mm);
 	task_unlock(tsk);
 
diff --git a/include/asm-um/mmu_context.h b/include/asm-um/mmu_context.h
index 095bb627b96a1d52be4cc1291f48c308449f65a1..2edb4f1f789cdb96e0208da409622db6f1eaa70b 100644
--- a/include/asm-um/mmu_context.h
+++ b/include/asm-um/mmu_context.h
@@ -20,7 +20,15 @@ extern void force_flush_all(void);
 
 static inline void activate_mm(struct mm_struct *old, struct mm_struct *new)
 {
-	if (old != new)
+	/*
+	 * This is called by fs/exec.c and fs/aio.c. In the first case, for an
+	 * exec, we don't need to do anything as we're called from userspace
+	 * and thus going to use a new host PID. In the second, we're called
+	 * from a kernel thread, and thus need to go doing the mmap's on the
+	 * host. Since they're very expensive, we want to avoid that as far as
+	 * possible.
+	 */
+	if (old != new && (current->flags & PF_BORROWED_MM))
 		force_flush_all();
 }