diff --git a/mm/swapfile.c b/mm/swapfile.c
index 7faa30f460e40c6c68e4a5c32b441a17f5702056..f4e45e84061c833cbb039f499cf90e374df2fd28 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -944,6 +944,11 @@ static int scan_swap_map_slots(struct swap_info_struct *si,
 scan:
 	spin_unlock(&si->lock);
 	while (++offset <= READ_ONCE(si->highest_bit)) {
+		if (unlikely(--latency_ration < 0)) {
+			cond_resched();
+			latency_ration = LATENCY_LIMIT;
+			scanned_many = true;
+		}
 		if (data_race(!si->swap_map[offset])) {
 			spin_lock(&si->lock);
 			goto checks;
@@ -953,14 +958,14 @@ static int scan_swap_map_slots(struct swap_info_struct *si,
 			spin_lock(&si->lock);
 			goto checks;
 		}
+	}
+	offset = si->lowest_bit;
+	while (offset < scan_base) {
 		if (unlikely(--latency_ration < 0)) {
 			cond_resched();
 			latency_ration = LATENCY_LIMIT;
 			scanned_many = true;
 		}
-	}
-	offset = si->lowest_bit;
-	while (offset < scan_base) {
 		if (data_race(!si->swap_map[offset])) {
 			spin_lock(&si->lock);
 			goto checks;
@@ -970,11 +975,6 @@ static int scan_swap_map_slots(struct swap_info_struct *si,
 			spin_lock(&si->lock);
 			goto checks;
 		}
-		if (unlikely(--latency_ration < 0)) {
-			cond_resched();
-			latency_ration = LATENCY_LIMIT;
-			scanned_many = true;
-		}
 		offset++;
 	}
 	spin_lock(&si->lock);