diff --git a/fs/io-wq.c b/fs/io-wq.c
index 807ccdf4640080bfcdf49cbe884653d712785e54..acc036f50ff97065dea7d3050a8df06db323ef24 100644
--- a/fs/io-wq.c
+++ b/fs/io-wq.c
@@ -446,10 +446,14 @@ static void io_worker_handle_work(struct io_worker *worker)
 			task_unlock(current);
 		}
 		if ((work->flags & IO_WQ_WORK_NEEDS_USER) && !worker->mm &&
-		    wq->mm && mmget_not_zero(wq->mm)) {
-			use_mm(wq->mm);
-			set_fs(USER_DS);
-			worker->mm = wq->mm;
+		    wq->mm) {
+			if (mmget_not_zero(wq->mm)) {
+				use_mm(wq->mm);
+				set_fs(USER_DS);
+				worker->mm = wq->mm;
+			} else {
+				work->flags |= IO_WQ_WORK_CANCEL;
+			}
 		}
 		if (!worker->creds)
 			worker->creds = override_creds(wq->creds);