diff --git a/ppdet/data/source/coco.py b/ppdet/data/source/coco.py
index 67c561786446722e2b538d78cf8cf0488a168691..e393a4609ab5ed4d8558fa1548cb907aeecaa8e6 100644
--- a/ppdet/data/source/coco.py
+++ b/ppdet/data/source/coco.py
@@ -146,6 +146,8 @@ class COCODataSet(DataSet):
                             'x1: {}, y1: {}, x2: {}, y2: {}.'.format(
                                 img_id, x1, y1, x2, y2))
                 num_bbox = len(bboxes)
+                if num_bbox <= 0:
+                    continue
 
                 gt_bbox = np.zeros((num_bbox, 4), dtype=np.float32)
                 gt_class = np.zeros((num_bbox, 1), dtype=np.int32)
@@ -154,6 +156,7 @@ class COCODataSet(DataSet):
                 difficult = np.zeros((num_bbox, 1), dtype=np.int32)
                 gt_poly = [None] * num_bbox
 
+                has_segmentation = False
                 for i, box in enumerate(bboxes):
                     catid = box['category_id']
                     gt_class[i][0] = catid2clsid[catid]
@@ -161,6 +164,10 @@ class COCODataSet(DataSet):
                     is_crowd[i][0] = box['iscrowd']
                     if 'segmentation' in box:
                         gt_poly[i] = box['segmentation']
+                        has_segmentation = True
+
+                if has_segmentation and not any(gt_poly):
+                    continue
 
                 coco_rec.update({
                     'is_crowd': is_crowd,