diff --git a/fs/userfaultfd.c b/fs/userfaultfd.c
index 0d19adb40dc2e481a4336eabc1cdc0e966555899..978fc69c4dbb1effe0f3344769c3e4bc18dd58f5 100644
--- a/fs/userfaultfd.c
+++ b/fs/userfaultfd.c
@@ -884,7 +884,7 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
 	struct vm_area_struct *vma, *prev;
 	/* len == 0 means wake all */
 	struct userfaultfd_wake_range range = { .len = 0, };
-	unsigned long new_flags;
+	unsigned long new_flags, userfault_flags;
 	bool still_valid;
 
 	WRITE_ONCE(ctx->released, true);
@@ -904,14 +904,18 @@ static int userfaultfd_release(struct inode *inode, struct file *file)
 	still_valid = mmget_still_valid(mm);
 	prev = NULL;
 	for (vma = mm->mmap; vma; vma = vma->vm_next) {
+		userfault_flags = VM_UFFD_MISSING | VM_UFFD_WP;
+#ifdef CONFIG_USERSWAP
+		userfault_flags |= VM_USWAP;
+#endif
 		cond_resched();
 		BUG_ON(!!vma->vm_userfaultfd_ctx.ctx ^
-		       !!(vma->vm_flags & (VM_UFFD_MISSING | VM_UFFD_WP)));
+		       !!(vma->vm_flags & userfault_flags));
 		if (vma->vm_userfaultfd_ctx.ctx != ctx) {
 			prev = vma;
 			continue;
 		}
-		new_flags = vma->vm_flags & ~(VM_UFFD_MISSING | VM_UFFD_WP);
+		new_flags = vma->vm_flags & ~userfault_flags;
 		if (still_valid) {
 			prev = vma_merge(mm, prev, vma->vm_start, vma->vm_end,
 					 new_flags, vma->anon_vma,
@@ -1333,6 +1337,8 @@ static int userfaultfd_register(struct userfaultfd_ctx *ctx,
 	 */
 	if (uffdio_register.mode & UFFDIO_REGISTER_MODE_USWAP) {
 		uffdio_register.mode &= ~UFFDIO_REGISTER_MODE_USWAP;
+		if (!uffdio_register.mode)
+			goto out;
 		vm_flags |= VM_USWAP;
 		end = uffdio_register.range.start + uffdio_register.range.len - 1;
 		vma = find_vma(mm, uffdio_register.range.start);
diff --git a/mm/memory.c b/mm/memory.c
index d470bcd7d2ed48be796686421ce8943927739ff9..a1fefe2b368d3e409b9968d69f346774d5f923ad 100644
--- a/mm/memory.c
+++ b/mm/memory.c
@@ -2782,7 +2782,6 @@ vm_fault_t do_swap_page(struct vm_fault *vmf)
 					vmf->address, vma->vm_flags);
 			goto skip_uswap;
 		}
-		BUG_ON(!(vma->vm_flags & VM_UFFD_MISSING));
 		ret = handle_userfault(vmf, VM_UFFD_MISSING | VM_USWAP);
 		return ret;
 	}