diff --git a/fs/aio.c b/fs/aio.c
index 06d7d43..4f641ab 100644
--- a/fs/aio.c
+++ b/fs/aio.c
@@ -567,6 +567,10 @@
 	atomic_inc(&mm->mm_count);
 	tsk->mm = mm;
 	tsk->active_mm = mm;
+	/*
+	 * Note that on UML this *requires* PF_BORROWED_MM to be set, otherwise
+	 * it won't work. Update it accordingly if you change it here
+	 */
 	activate_mm(active_mm, mm);
 	task_unlock(tsk);
 
diff --git a/include/asm-um/mmu_context.h b/include/asm-um/mmu_context.h
index 095bb62..2edb4f1 100644
--- a/include/asm-um/mmu_context.h
+++ b/include/asm-um/mmu_context.h
@@ -20,7 +20,15 @@
 
 static inline void activate_mm(struct mm_struct *old, struct mm_struct *new)
 {
-	if (old != new)
+	/*
+	 * This is called by fs/exec.c and fs/aio.c. In the first case, for an
+	 * exec, we don't need to do anything as we're called from userspace
+	 * and thus going to use a new host PID. In the second, we're called
+	 * from a kernel thread, and thus need to go doing the mmap's on the
+	 * host. Since they're very expensive, we want to avoid that as far as
+	 * possible.
+	 */
+	if (old != new && (current->flags & PF_BORROWED_MM))
 		force_flush_all();
 }