diff --git a/dygraph/ppdet/data/source/coco.py b/dygraph/ppdet/data/source/coco.py
index 387229136ef5470d988e19e63c912992c3e1a801..c78b13a0ebf17c5fecb63663e1c2574c1ab96dba 100644
--- a/dygraph/ppdet/data/source/coco.py
+++ b/dygraph/ppdet/data/source/coco.py
@@ -79,6 +79,13 @@ class COCODataSet(DetDataset):
                                    im_w, im_h, img_id))
                 continue
 
+            coco_rec = {
+                'im_file': im_path,
+                'im_id': np.array([img_id]),
+                'h': im_h,
+                'w': im_w,
+            } if 'image' in self.data_fields else {}
+
             if not self.load_image_only:
                 ins_anno_ids = coco.getAnnIds(imgIds=[img_id], iscrowd=False)
                 instances = coco.loadAnns(ins_anno_ids)
@@ -132,13 +139,6 @@ class COCODataSet(DetDataset):
                 if has_segmentation and not any(gt_poly):
                     continue
 
-                coco_rec = {
-                    'im_file': im_path,
-                    'im_id': np.array([img_id]),
-                    'h': im_h,
-                    'w': im_w,
-                } if 'image' in self.data_fields else {}
-
                 gt_rec = {
                     'is_crowd': is_crowd,
                     'gt_class': gt_class,