diff --git a/arch/x86/kvm/mmu.c b/arch/x86/kvm/mmu.c
index 1594ee06c920cd5f25192832d28fbeb5e6aace8b..5c4c16662c6831752412c024a83ff9522454dac9 100644
--- a/arch/x86/kvm/mmu.c
+++ b/arch/x86/kvm/mmu.c
@@ -549,7 +549,8 @@ static void rmap_remove(struct kvm *kvm, u64 *spte)
 		return;
 	sp = page_header(__pa(spte));
 	page = spte_to_page(*spte);
-	mark_page_accessed(page);
+	if (*spte & PT_ACCESSED_MASK)
+		mark_page_accessed(page);
 	if (is_writeble_pte(*spte))
 		kvm_release_page_dirty(page);
 	else