diff --git a/modules/prediction/scenario/prioritization/obstacles_prioritizer.cc b/modules/prediction/scenario/prioritization/obstacles_prioritizer.cc
index 0332a898b8b3bb6f4af915a84643996abc443a01..5e8152de75f5c64fb18a2ef824c81e828b00dff8 100644
--- a/modules/prediction/scenario/prioritization/obstacles_prioritizer.cc
+++ b/modules/prediction/scenario/prioritization/obstacles_prioritizer.cc
@@ -343,10 +343,10 @@ void ObstaclesPrioritizer::AssignCautionLevelByEgoReferenceLine() {
   // insert ego_back_lane_id
   accumulated_s = 0.0;
   for (const std::string& lane_id : lane_ids) {
-    ego_back_lane_id_set_.insert(lane_id);
     if (lane_id == ego_lane_id) {
       break;
     }
+    ego_back_lane_id_set_.insert(lane_id);
   }
 
   // then loop through lane_ids to AssignCaution for obstacle vehicles
@@ -361,7 +361,9 @@ void ObstaclesPrioritizer::AssignCautionLevelByEgoReferenceLine() {
       continue;
     }
     accumulated_s += lane_info_ptr->total_length();
-    AssignCautionByMerge(lane_info_ptr);
+    if (lane_id != ego_lane_id) {
+      AssignCautionByMerge(lane_info_ptr);
+    }
     AssignCautionByOverlap(lane_info_ptr);
     if (accumulated_s > FLAGS_caution_search_distance_ahead + ego_vehicle_s) {
       break;