filter illegal data (#646)

* filter illegal data

* update warning message

ppdet/data/source/coco.py

@@ -106,6 +106,16 @@ class COCODataSet(DataSet):
 
             im_fname = os.path.join(image_dir,
                                     im_fname) if image_dir else im_fname
+            if not os.path.exists(im_fname):
+                logger.warn('Illegal image file: {}, and it will be '
+                            'ignored'.format(im_fname))
+                continue
+
+            if im_w < 0 or im_h < 0:
+                logger.warn('Illegal width: {} or height: {} in annotation, '
+                            'and im_id: {} will be ignored'.format(im_w, im_h,
+                                                                   img_id))
+                continue
 
             coco_rec = {
                 'im_file': im_fname,

ppdet/data/source/voc.py

@@ -110,7 +110,14 @@ class VOCDataSet(DataSet):
                     break
                 img_file, xml_file = [os.path.join(image_dir, x) \
                         for x in line.strip().split()[:2]]
+                if not os.path.exists(img_file):
+                    logger.warn(
+                        'Illegal image file: {}, and it will be ignored'.format(
+                            img_file))
+                    continue
                 if not os.path.isfile(xml_file):
+                    logger.warn('Illegal xml file: {}, and it will be ignored'.
+                                format(xml_file))
                     continue
                 tree = ET.parse(xml_file)
                 if tree.find('id') is None:
@@ -121,14 +128,18 @@ class VOCDataSet(DataSet):
                 objs = tree.findall('object')
                 im_w = float(tree.find('size').find('width').text)
                 im_h = float(tree.find('size').find('height').text)
-                gt_bbox = np.zeros((len(objs), 4), dtype=np.float32)
-                gt_class = np.zeros((len(objs), 1), dtype=np.int32)
-                gt_score = np.ones((len(objs), 1), dtype=np.float32)
-                is_crowd = np.zeros((len(objs), 1), dtype=np.int32)
-                difficult = np.zeros((len(objs), 1), dtype=np.int32)
+                if im_w < 0 or im_h < 0:
+                    logger.warn(
+                        'Illegal width: {} or height: {} in annotation, '
+                        'and {} will be ignored'.format(im_w, im_h, xml_file))
+                    continue
+                gt_bbox = []
+                gt_class = []
+                gt_score = []
+                is_crowd = []
+                difficult = []
                 for i, obj in enumerate(objs):
                     cname = obj.find('name').text
-                    gt_class[i][0] = cname2cid[cname]
                     _difficult = int(obj.find('difficult').text)
                     x1 = float(obj.find('bndbox').find('xmin').text)
                     y1 = float(obj.find('bndbox').find('ymin').text)
@@ -138,9 +149,22 @@ class VOCDataSet(DataSet):
                     y1 = max(0, y1)
                     x2 = min(im_w - 1, x2)
                     y2 = min(im_h - 1, y2)
-                    gt_bbox[i] = [x1, y1, x2, y2]
-                    is_crowd[i][0] = 0
-                    difficult[i][0] = _difficult
+                    if x2 > x1 and y2 > y1:
+                        gt_bbox.append([x1, y1, x2, y2])
+                        gt_class.append([cname2cid[cname]])
+                        gt_score.append([1.])
+                        is_crowd.append([0])
+                        difficult.append([_difficult])
+                    else:
+                        logger.warn(
+                            'Found an invalid bbox in annotations: xml_file: {}'
+                            ', x1: {}, y1: {}, x2: {}, y2: {}.'.format(
+                                xml_file, x1, y1, x2, y2))
+                gt_bbox = np.array(gt_bbox).astype('float32')
+                gt_class = np.array(gt_class).astype('int32')
+                gt_score = np.array(gt_score).astype('float32')
+                is_crowd = np.array(is_crowd).astype('int32')
+                difficult = np.array(difficult).astype('int32')
                 voc_rec = {
                     'im_file': img_file,
                     'im_id': im_id,

ppdet/data/source/widerface.py

@@ -79,6 +79,9 @@ class WIDERFaceDataSet(DataSet):
                     h = float(temp_info_box[3])
                     # Filter out wrong labels
                     if w < 0 or h < 0:
+                        logger.warn('Illegal box with w: {}, h: {} in '
+                                    'img: {}, and it will be ignored'.format(
+                                        w, h, im_fname))
                         continue
                     xmin = max(0, xmin)
                     ymin = max(0, ymin)

ppdet/data/transform/operators.py

@@ -121,8 +121,21 @@ class DecodeImage(BaseOperator):
 
         if 'h' not in sample:
             sample['h'] = im.shape[0]
+        elif sample['h'] != im.shape[0]:
+            logger.warn(
+                "The actual image height: {} is not equal to the "
+                "height: {} in annotation, and update sample['h'] by actual "
+                "image height.".format(im.shape[0], sample['h']))
+            sample['h'] = im.shape[0]
         if 'w' not in sample:
             sample['w'] = im.shape[1]
+        elif sample['w'] != im.shape[1]:
+            logger.warn(
+                "The actual image width: {} is not equal to the "
+                "width: {} in annotation, and update sample['w'] by actual "
+                "image width.".format(im.shape[1], sample['w']))
+            sample['w'] = im.shape[1]
+
         # make default im_info with [h, w, 1]
         sample['im_info'] = np.array(
             [im.shape[0], im.shape[1], 1.], dtype=np.float32)