diff --git a/ppdet/data/source/coco.py b/ppdet/data/source/coco.py
index d5691c7be86d82db8fdc85eaa2987bbbfb4c1535..53e31714d123a23a5432ae00a0f495cbc042f0c5 100644
--- a/ppdet/data/source/coco.py
+++ b/ppdet/data/source/coco.py
@@ -90,6 +90,13 @@ class COCODataSet(DetDataset):
                                    im_w, im_h, img_id))
                 continue
 
+            coco_rec = {
+                'im_file': im_path,
+                'im_id': np.array([img_id]),
+                'h': im_h,
+                'w': im_w,
+            } if 'image' in self.data_fields else {}
+
             if not self.load_image_only:
                 ins_anno_ids = coco.getAnnIds(imgIds=[img_id], iscrowd=False)
                 instances = coco.loadAnns(ins_anno_ids)
@@ -143,13 +150,6 @@ class COCODataSet(DetDataset):
                 if has_segmentation and not any(gt_poly):
                     continue
 
-                coco_rec = {
-                    'im_file': im_path,
-                    'im_id': np.array([img_id]),
-                    'h': im_h,
-                    'w': im_w,
-                } if 'image' in self.data_fields else {}
-
                 gt_rec = {
                     'is_crowd': is_crowd,
                     'gt_class': gt_class,