diff --git a/mm/swapfile.c b/mm/swapfile.c
index 2619729400d3296415eec11d6675b1689b93e5f4..ba97c8f635b0082421707f9e792015dbfb05bd14 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -851,6 +851,10 @@ static int scan_swap_map_slots(struct swap_info_struct *si,
 scan:
 	spin_unlock(&si->lock);
 	while (++offset <= READ_ONCE(si->highest_bit)) {
+		if (unlikely(--latency_ration < 0)) {
+			cond_resched();
+			latency_ration = LATENCY_LIMIT;
+		}
 		if (!si->swap_map[offset]) {
 			spin_lock(&si->lock);
 			goto checks;
@@ -860,13 +864,13 @@ static int scan_swap_map_slots(struct swap_info_struct *si,
 			spin_lock(&si->lock);
 			goto checks;
 		}
+	}
+	offset = si->lowest_bit;
+	while (offset < scan_base) {
 		if (unlikely(--latency_ration < 0)) {
 			cond_resched();
 			latency_ration = LATENCY_LIMIT;
 		}
-	}
-	offset = si->lowest_bit;
-	while (offset < scan_base) {
 		if (!si->swap_map[offset]) {
 			spin_lock(&si->lock);
 			goto checks;
@@ -876,10 +880,6 @@ static int scan_swap_map_slots(struct swap_info_struct *si,
 			spin_lock(&si->lock);
 			goto checks;
 		}
-		if (unlikely(--latency_ration < 0)) {
-			cond_resched();
-			latency_ration = LATENCY_LIMIT;
-		}
 		offset++;
 	}
 	spin_lock(&si->lock);