From b06830587aa2a23338271178a2b1f793685f1845 Mon Sep 17 00:00:00 2001 From: walloollaw <37680514+walloollaw@users.noreply.github.com> Date: Thu, 25 Jul 2019 14:41:13 +0800 Subject: [PATCH] Fix bug in unittests of ppdet.data.tests (#2894) --- ppdet/data/source/coco_loader.py | 3 +++ ppdet/data/tests/coco.yml | 2 +- ppdet/data/tests/rcnn_dataset.yml | 2 +- ppdet/data/tests/set_env.py | 4 ---- ppdet/data/tests/test_loader.py | 8 ++++---- ppdet/data/tests/test_operator.py | 16 ++++++++-------- ppdet/data/tests/test_reader.py | 2 +- ppdet/data/tests/test_roidb_source.py | 2 +- ppdet/data/tests/test_transformer.py | 10 +++++----- ppdet/data/transform/__init__.py | 6 +++++- ppdet/data/transform/operators.py | 4 +++- 11 files changed, 32 insertions(+), 27 deletions(-) diff --git a/ppdet/data/source/coco_loader.py b/ppdet/data/source/coco_loader.py index 38bef3bf8..3935c1e0c 100644 --- a/ppdet/data/source/coco_loader.py +++ b/ppdet/data/source/coco_loader.py @@ -13,6 +13,9 @@ # limitations under the License. import numpy as np +import matplotlib +matplotlib.use('Agg') + from pycocotools.coco import COCO import logging diff --git a/ppdet/data/tests/coco.yml b/ppdet/data/tests/coco.yml index 609732f22..345e98e04 100644 --- a/ppdet/data/tests/coco.yml +++ b/ppdet/data/tests/coco.yml @@ -23,7 +23,7 @@ TRANSFORM: - OP: ResizeImage TARGET_SIZE: 800 MAX_SIZE: 1333 - - OP: Rgb2Bgr + - OP: Permute TO_BGR: False - OP: ArrangeRCNN BATCH_SIZE: 1 diff --git a/ppdet/data/tests/rcnn_dataset.yml b/ppdet/data/tests/rcnn_dataset.yml index 2d438c772..5f72530f0 100644 --- a/ppdet/data/tests/rcnn_dataset.yml +++ b/ppdet/data/tests/rcnn_dataset.yml @@ -19,7 +19,7 @@ TRANSFORM: - OP: ResizeImage TARGET_SIZE: 800 MAX_SIZE: 1333 - - OP: Rgb2Bgr + - OP: Permute TO_BGR: False - OP: ArrangeRCNN BATCH_SIZE: 1 diff --git a/ppdet/data/tests/set_env.py b/ppdet/data/tests/set_env.py index 4531681f3..2ded06a5d 100644 --- a/ppdet/data/tests/set_env.py +++ b/ppdet/data/tests/set_env.py @@ -3,10 +3,6 @@ import os import six import logging -path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../') -if path not in sys.path: - sys.path.insert(0, path) - prefix = os.path.dirname(os.path.abspath(__file__)) #coco data for testing diff --git a/ppdet/data/tests/test_loader.py b/ppdet/data/tests/test_loader.py index 1b9f35ff5..dc835aa0f 100644 --- a/ppdet/data/tests/test_loader.py +++ b/ppdet/data/tests/test_loader.py @@ -51,7 +51,7 @@ class TestLoader(unittest.TestCase): def test_load_coco_in_json(self): """ test loading COCO data in json file """ - from data.source.coco_loader import load + from ppdet.data.source.coco_loader import load if not os.path.exists(self.anno_path): logging.warn('not found %s, so skip this test' % (self.anno_path)) return @@ -71,7 +71,7 @@ class TestLoader(unittest.TestCase): return samples = 10 - from data.source.loader import load_roidb + from ppdet.data.source.loader import load_roidb records, cname2cid = load_roidb(anno_path, samples) self.assertEqual(len(records), samples) self.assertGreater(len(cname2cid), 0) @@ -79,7 +79,7 @@ class TestLoader(unittest.TestCase): def test_load_voc_in_xml(self): """ test loading VOC data in xml files """ - from data.source.voc_loader import load + from ppdet.data.source.voc_loader import load if not os.path.exists(self.anno_path1): logging.warn('not found %s, so skip this test' % (self.anno_path1)) return @@ -98,7 +98,7 @@ class TestLoader(unittest.TestCase): return samples = 3 - from loader import load_roidb + from ppdet.data.source.loader import load_roidb records, cname2cid = load_roidb(anno_path, samples) self.assertEqual(len(records), samples) self.assertGreater(len(cname2cid), 0) diff --git a/ppdet/data/tests/test_operator.py b/ppdet/data/tests/test_operator.py index 73df5601e..05544bd22 100644 --- a/ppdet/data/tests/test_operator.py +++ b/ppdet/data/tests/test_operator.py @@ -3,7 +3,7 @@ import unittest import logging import numpy as np import set_env -from data import transform as tf +import ppdet.data.transform as tf logging.basicConfig(level=logging.INFO) @@ -47,7 +47,7 @@ class TestBase(unittest.TestCase): 'target_size': 300, 'max_size': 1333 }] - mapper = tf.build(ops_conf) + mapper = tf.build_mapper(ops_conf) self.assertTrue(mapper is not None) data = self.sample.copy() result0 = mapper(data) @@ -55,14 +55,14 @@ class TestBase(unittest.TestCase): self.assertEqual(len(result0['image'].shape), 3) # RandFlipImage ops_conf = [{'op': 'RandomFlipImage'}] - mapper = tf.build(ops_conf) + mapper = tf.build_mapper(ops_conf) self.assertTrue(mapper is not None) result1 = mapper(result0) self.assertEqual(result1['image'].shape, result0['image'].shape) self.assertEqual(result1['gt_bbox'].shape, result0['gt_bbox'].shape) # NormalizeImage ops_conf = [{'op': 'NormalizeImage', 'is_channel_first': False}] - mapper = tf.build(ops_conf) + mapper = tf.build_mapper(ops_conf) self.assertTrue(mapper is not None) result2 = mapper(result1) im1 = result1['image'] @@ -71,7 +71,7 @@ class TestBase(unittest.TestCase): self.assertEqual(count, im1.shape[0] * im1.shape[1], im1.shape[2]) # ArrangeSample ops_conf = [{'op': 'ArrangeRCNN'}] - mapper = tf.build(ops_conf) + mapper = tf.build_mapper(ops_conf) self.assertTrue(mapper is not None) result3 = mapper(result2) self.assertEqual(type(result3), tuple) @@ -93,7 +93,7 @@ class TestBase(unittest.TestCase): [1, 50, 0.3, 1.0, 0.5, 2.0, 0.9, 0.0], [1, 50, 0.3, 1.0, 0.5, 2.0, 0.0, 1.0]] }] - mapper = tf.build(ops_conf) + mapper = tf.build_mapper(ops_conf) self.assertTrue(mapper is not None) data = self.sample.copy() result = mapper(data) @@ -111,7 +111,7 @@ class TestBase(unittest.TestCase): 'max_ratio': 1.5, 'prob': 1 }] - mapper = tf.build(ops_conf) + mapper = tf.build_mapper(ops_conf) self.assertTrue(mapper is not None) data = self.sample.copy() result = mapper(data) @@ -130,7 +130,7 @@ class TestBase(unittest.TestCase): 'op': 'RandomInterpImage', 'target_size': 608 }] - mapper = tf.build(ops_conf) + mapper = tf.build_mapper(ops_conf) self.assertTrue(mapper is not None) data = self.sample.copy() result = mapper(data) diff --git a/ppdet/data/tests/test_reader.py b/ppdet/data/tests/test_reader.py index c04d24bdc..bd6db801d 100644 --- a/ppdet/data/tests/test_reader.py +++ b/ppdet/data/tests/test_reader.py @@ -7,7 +7,7 @@ import numpy as np import yaml import set_env -from data import Reader +from ppdet.data.reader import Reader class TestReader(unittest.TestCase): diff --git a/ppdet/data/tests/test_roidb_source.py b/ppdet/data/tests/test_roidb_source.py index e0f050e46..ad197ce2a 100644 --- a/ppdet/data/tests/test_roidb_source.py +++ b/ppdet/data/tests/test_roidb_source.py @@ -5,7 +5,7 @@ import sys import logging import set_env -from data import build_source +from ppdet.data.source import build_source class TestRoiDbSource(unittest.TestCase): diff --git a/ppdet/data/tests/test_transformer.py b/ppdet/data/tests/test_transformer.py index 5dc2b757e..47c177e44 100644 --- a/ppdet/data/tests/test_transformer.py +++ b/ppdet/data/tests/test_transformer.py @@ -6,8 +6,8 @@ import logging import numpy as np import set_env -from data import build_source -from data import transform as tf +import ppdet.data.transform as tf +from ppdet.data.source import build_source logger = logging.getLogger(__name__) @@ -53,7 +53,7 @@ class TestTransformer(unittest.TestCase): def test_map(self): """ test transformer.map """ - mapper = tf.build(self.ops) + mapper = tf.build_mapper(self.ops) ds = build_source(self.sc_config) mapped_ds = tf.map(ds, mapper) ct = 0 @@ -66,7 +66,7 @@ class TestTransformer(unittest.TestCase): def test_parallel_map(self): """ test transformer.map with concurrent workers """ - mapper = tf.build(self.ops) + mapper = tf.build_mapper(self.ops) ds = build_source(self.sc_config) worker_conf = {'WORKER_NUM': 2, 'use_process': True} mapped_ds = tf.map(ds, mapper, worker_conf) @@ -91,7 +91,7 @@ class TestTransformer(unittest.TestCase): """ test batched dataset """ batchsize = 2 - mapper = tf.build(self.ops) + mapper = tf.build_mapper(self.ops) ds = build_source(self.sc_config) mapped_ds = tf.map(ds, mapper) batched_ds = tf.batch(mapped_ds, batchsize, True) diff --git a/ppdet/data/transform/__init__.py b/ppdet/data/transform/__init__.py index 3f202487c..3517e48c3 100644 --- a/ppdet/data/transform/__init__.py +++ b/ppdet/data/transform/__init__.py @@ -17,6 +17,7 @@ from __future__ import print_function import copy import logging +import traceback from .transformer import MappedDataset, BatchedDataset from .post_map import build_post_map @@ -76,7 +77,10 @@ def build_mapper(ops, context=None): out = f(sample, ctx) sample = out except Exception as e: - logger.warn("fail to map op [{}] with error: {}".format(f, e)) + stack_info = traceback.format_exc() + logger.warn("fail to map op [{}] with error: {} and stack:\n{}".format(f, e, str(stack_info))) + raise e + return out _mapper.ops = op_repr diff --git a/ppdet/data/transform/operators.py b/ppdet/data/transform/operators.py index 7d86cf8c9..efaabcd21 100644 --- a/ppdet/data/transform/operators.py +++ b/ppdet/data/transform/operators.py @@ -540,7 +540,9 @@ class CropImage(BaseOperator): gt_class = sample['gt_class'] im_width = sample['w'] im_height = sample['h'] - gt_score = sample['gt_score'] + gt_score = None + if 'gt_score' in sample: + gt_score = sample['gt_score'] sampled_bbox = [] gt_bbox = gt_bbox.tolist() for sampler in self.batch_sampler: -- GitLab