From e3fce2910a953ff810283c8f81edbe806ec29003 Mon Sep 17 00:00:00 2001 From: wangguanzhong Date: Tue, 12 May 2020 17:33:48 +0800 Subject: [PATCH] filter illegal data (#646) * filter illegal data * update warning message --- ppdet/data/source/coco.py | 10 ++++++++ ppdet/data/source/voc.py | 42 ++++++++++++++++++++++++------- ppdet/data/source/widerface.py | 3 +++ ppdet/data/transform/operators.py | 13 ++++++++++ 4 files changed, 59 insertions(+), 9 deletions(-) diff --git a/ppdet/data/source/coco.py b/ppdet/data/source/coco.py index a4a0e74e3..aaeaed58e 100644 --- a/ppdet/data/source/coco.py +++ b/ppdet/data/source/coco.py @@ -106,6 +106,16 @@ class COCODataSet(DataSet): im_fname = os.path.join(image_dir, im_fname) if image_dir else im_fname + if not os.path.exists(im_fname): + logger.warn('Illegal image file: {}, and it will be ' + 'ignored'.format(im_fname)) + continue + + if im_w < 0 or im_h < 0: + logger.warn('Illegal width: {} or height: {} in annotation, ' + 'and im_id: {} will be ignored'.format(im_w, im_h, + img_id)) + continue coco_rec = { 'im_file': im_fname, diff --git a/ppdet/data/source/voc.py b/ppdet/data/source/voc.py index 77410d73f..560ed17ea 100644 --- a/ppdet/data/source/voc.py +++ b/ppdet/data/source/voc.py @@ -110,7 +110,14 @@ class VOCDataSet(DataSet): break img_file, xml_file = [os.path.join(image_dir, x) \ for x in line.strip().split()[:2]] + if not os.path.exists(img_file): + logger.warn( + 'Illegal image file: {}, and it will be ignored'.format( + img_file)) + continue if not os.path.isfile(xml_file): + logger.warn('Illegal xml file: {}, and it will be ignored'. + format(xml_file)) continue tree = ET.parse(xml_file) if tree.find('id') is None: @@ -121,14 +128,18 @@ class VOCDataSet(DataSet): objs = tree.findall('object') im_w = float(tree.find('size').find('width').text) im_h = float(tree.find('size').find('height').text) - gt_bbox = np.zeros((len(objs), 4), dtype=np.float32) - gt_class = np.zeros((len(objs), 1), dtype=np.int32) - gt_score = np.ones((len(objs), 1), dtype=np.float32) - is_crowd = np.zeros((len(objs), 1), dtype=np.int32) - difficult = np.zeros((len(objs), 1), dtype=np.int32) + if im_w < 0 or im_h < 0: + logger.warn( + 'Illegal width: {} or height: {} in annotation, ' + 'and {} will be ignored'.format(im_w, im_h, xml_file)) + continue + gt_bbox = [] + gt_class = [] + gt_score = [] + is_crowd = [] + difficult = [] for i, obj in enumerate(objs): cname = obj.find('name').text - gt_class[i][0] = cname2cid[cname] _difficult = int(obj.find('difficult').text) x1 = float(obj.find('bndbox').find('xmin').text) y1 = float(obj.find('bndbox').find('ymin').text) @@ -138,9 +149,22 @@ class VOCDataSet(DataSet): y1 = max(0, y1) x2 = min(im_w - 1, x2) y2 = min(im_h - 1, y2) - gt_bbox[i] = [x1, y1, x2, y2] - is_crowd[i][0] = 0 - difficult[i][0] = _difficult + if x2 > x1 and y2 > y1: + gt_bbox.append([x1, y1, x2, y2]) + gt_class.append([cname2cid[cname]]) + gt_score.append([1.]) + is_crowd.append([0]) + difficult.append([_difficult]) + else: + logger.warn( + 'Found an invalid bbox in annotations: xml_file: {}' + ', x1: {}, y1: {}, x2: {}, y2: {}.'.format( + xml_file, x1, y1, x2, y2)) + gt_bbox = np.array(gt_bbox).astype('float32') + gt_class = np.array(gt_class).astype('int32') + gt_score = np.array(gt_score).astype('float32') + is_crowd = np.array(is_crowd).astype('int32') + difficult = np.array(difficult).astype('int32') voc_rec = { 'im_file': img_file, 'im_id': im_id, diff --git a/ppdet/data/source/widerface.py b/ppdet/data/source/widerface.py index ddf4b2cb4..311430559 100644 --- a/ppdet/data/source/widerface.py +++ b/ppdet/data/source/widerface.py @@ -79,6 +79,9 @@ class WIDERFaceDataSet(DataSet): h = float(temp_info_box[3]) # Filter out wrong labels if w < 0 or h < 0: + logger.warn('Illegal box with w: {}, h: {} in ' + 'img: {}, and it will be ignored'.format( + w, h, im_fname)) continue xmin = max(0, xmin) ymin = max(0, ymin) diff --git a/ppdet/data/transform/operators.py b/ppdet/data/transform/operators.py index 899b82097..6d04454e5 100644 --- a/ppdet/data/transform/operators.py +++ b/ppdet/data/transform/operators.py @@ -121,8 +121,21 @@ class DecodeImage(BaseOperator): if 'h' not in sample: sample['h'] = im.shape[0] + elif sample['h'] != im.shape[0]: + logger.warn( + "The actual image height: {} is not equal to the " + "height: {} in annotation, and update sample['h'] by actual " + "image height.".format(im.shape[0], sample['h'])) + sample['h'] = im.shape[0] if 'w' not in sample: sample['w'] = im.shape[1] + elif sample['w'] != im.shape[1]: + logger.warn( + "The actual image width: {} is not equal to the " + "width: {} in annotation, and update sample['w'] by actual " + "image width.".format(im.shape[1], sample['w'])) + sample['w'] = im.shape[1] + # make default im_info with [h, w, 1] sample['im_info'] = np.array( [im.shape[0], im.shape[1], 1.], dtype=np.float32) -- GitLab