diff --git a/tools/infer/predict_system.py b/tools/infer/predict_system.py
index e0f2c41fa2aba23491efee920afbd76db1ec84e0..f12fd626c804bacd2607381f6bbd6d2179e20032 100755
--- a/tools/infer/predict_system.py
+++ b/tools/infer/predict_system.py
@@ -116,7 +116,10 @@ def sorted_boxes(dt_boxes):
         sorted boxes(array) with shape [4, 2]
     """
     num_boxes = dt_boxes.shape[0]
-    sorted_boxes = sorted(dt_boxes, key=lambda x: (x[0][1], x[0][0]))
+    if abs(num_boxes - 2) < 1e-4:
+        sorted_boxes = sorted(dt_boxes, key=lambda x: (x[1], x[0]))
+    else:
+        sorted_boxes = sorted(dt_boxes, key=lambda x: (x[0][1], x[0][0]))
     _boxes = list(sorted_boxes)
 
     for i in range(num_boxes - 1):