diff --git a/tools/infer/predict_system.py b/tools/infer/predict_system.py
index 73b7155baa9f869da928b5be03692c08115489ee..3f322ecc6f44da58491f2fef06cda71f00da4f46 100755
--- a/tools/infer/predict_system.py
+++ b/tools/infer/predict_system.py
@@ -120,11 +120,14 @@ def sorted_boxes(dt_boxes):
     _boxes = list(sorted_boxes)
 
     for i in range(num_boxes - 1):
-        if abs(_boxes[i + 1][0][1] - _boxes[i][0][1]) < 10 and \
-                (_boxes[i + 1][0][0] < _boxes[i][0][0]):
-            tmp = _boxes[i]
-            _boxes[i] = _boxes[i + 1]
-            _boxes[i + 1] = tmp
+        for j in range(i, 0, -1):
+            if abs(_boxes[j + 1][0][1] - _boxes[j][0][1]) < 10 and \
+                    (_boxes[j + 1][0][0] < _boxes[j][0][0]):
+                tmp = _boxes[j]
+                _boxes[j] = _boxes[j + 1]
+                _boxes[j + 1] = tmp
+            else:
+                break
     return _boxes