diff --git a/mm/swapfile.c b/mm/swapfile.c
index 0341c5700e346fa62401e9815720ed85af3a20c9..6d6d28c0a72f63751656eeb2ced06bbfbed610c7 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -2149,8 +2149,13 @@ SYSCALL_DEFINE2(swapon, const char __user *, specialfile, int, swap_flags)
 	p->flags = 0;
 	spin_unlock(&swap_lock);
 	vfree(swap_map);
-	if (swap_file)
+	if (swap_file) {
+		if (did_down) {
+			mutex_unlock(&inode->i_mutex);
+			did_down = 0;
+		}
 		filp_close(swap_file, NULL);
+	}
 out:
 	if (page && !IS_ERR(page)) {
 		kunmap(page);