Fix bug in unittests of ppdet.data.tests (#2894)

ppdet/data/source/coco_loader.py

@@ -13,6 +13,9 @@
 # limitations under the License.
 
 import numpy as np
+import matplotlib
+matplotlib.use('Agg')
+
 from pycocotools.coco import COCO
 
 import logging

ppdet/data/tests/coco.yml

@@ -23,7 +23,7 @@ TRANSFORM:
             - OP: ResizeImage
               TARGET_SIZE: 800
               MAX_SIZE: 1333
-            - OP: Rgb2Bgr
+            - OP: Permute
               TO_BGR: False
             - OP: ArrangeRCNN
         BATCH_SIZE: 1

ppdet/data/tests/rcnn_dataset.yml

@@ -19,7 +19,7 @@ TRANSFORM:
             - OP: ResizeImage
               TARGET_SIZE: 800
               MAX_SIZE: 1333
-            - OP: Rgb2Bgr
+            - OP: Permute
               TO_BGR: False
             - OP: ArrangeRCNN
         BATCH_SIZE: 1

ppdet/data/tests/set_env.py

@@ -3,10 +3,6 @@ import os
 import six
 import logging
 
-path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '../../')
-if path not in sys.path:
-    sys.path.insert(0, path)
-
 prefix = os.path.dirname(os.path.abspath(__file__))
 
 #coco data for testing

ppdet/data/tests/test_loader.py

@@ -51,7 +51,7 @@ class TestLoader(unittest.TestCase):
     def test_load_coco_in_json(self):
         """ test loading COCO data in json file
         """
-        from data.source.coco_loader import load
+        from ppdet.data.source.coco_loader import load
         if not os.path.exists(self.anno_path):
             logging.warn('not found %s, so skip this test' % (self.anno_path))
             return
@@ -71,7 +71,7 @@ class TestLoader(unittest.TestCase):
             return
 
         samples = 10
-        from data.source.loader import load_roidb
+        from ppdet.data.source.loader import load_roidb
         records, cname2cid = load_roidb(anno_path, samples)
         self.assertEqual(len(records), samples)
         self.assertGreater(len(cname2cid), 0)
@@ -79,7 +79,7 @@ class TestLoader(unittest.TestCase):
     def test_load_voc_in_xml(self):
         """ test loading VOC data in xml files
         """
-        from data.source.voc_loader import load
+        from ppdet.data.source.voc_loader import load
         if not os.path.exists(self.anno_path1):
             logging.warn('not found %s, so skip this test' % (self.anno_path1))
             return
@@ -98,7 +98,7 @@ class TestLoader(unittest.TestCase):
             return
 
         samples = 3
-        from loader import load_roidb
+        from ppdet.data.source.loader import load_roidb
         records, cname2cid = load_roidb(anno_path, samples)
         self.assertEqual(len(records), samples)
         self.assertGreater(len(cname2cid), 0)

ppdet/data/tests/test_operator.py

@@ -3,7 +3,7 @@ import unittest
 import logging
 import numpy as np
 import set_env
-from data import transform as tf
+import ppdet.data.transform as tf
 logging.basicConfig(level=logging.INFO)
 
 
@@ -47,7 +47,7 @@ class TestBase(unittest.TestCase):
             'target_size': 300,
             'max_size': 1333
         }]
-        mapper = tf.build(ops_conf)
+        mapper = tf.build_mapper(ops_conf)
         self.assertTrue(mapper is not None)
         data = self.sample.copy()
         result0 = mapper(data)
@@ -55,14 +55,14 @@ class TestBase(unittest.TestCase):
         self.assertEqual(len(result0['image'].shape), 3)
         # RandFlipImage
         ops_conf = [{'op': 'RandomFlipImage'}]
-        mapper = tf.build(ops_conf)
+        mapper = tf.build_mapper(ops_conf)
         self.assertTrue(mapper is not None)
         result1 = mapper(result0)
         self.assertEqual(result1['image'].shape, result0['image'].shape)
         self.assertEqual(result1['gt_bbox'].shape, result0['gt_bbox'].shape)
         # NormalizeImage
         ops_conf = [{'op': 'NormalizeImage', 'is_channel_first': False}]
-        mapper = tf.build(ops_conf)
+        mapper = tf.build_mapper(ops_conf)
         self.assertTrue(mapper is not None)
         result2 = mapper(result1)
         im1 = result1['image']
@@ -71,7 +71,7 @@ class TestBase(unittest.TestCase):
             self.assertEqual(count, im1.shape[0] * im1.shape[1], im1.shape[2])
         # ArrangeSample
         ops_conf = [{'op': 'ArrangeRCNN'}]
-        mapper = tf.build(ops_conf)
+        mapper = tf.build_mapper(ops_conf)
         self.assertTrue(mapper is not None)
         result3 = mapper(result2)
         self.assertEqual(type(result3), tuple)
@@ -93,7 +93,7 @@ class TestBase(unittest.TestCase):
                               [1, 50, 0.3, 1.0, 0.5, 2.0, 0.9, 0.0],
                               [1, 50, 0.3, 1.0, 0.5, 2.0, 0.0, 1.0]]
         }]
-        mapper = tf.build(ops_conf)
+        mapper = tf.build_mapper(ops_conf)
         self.assertTrue(mapper is not None)
         data = self.sample.copy()
         result = mapper(data)
@@ -111,7 +111,7 @@ class TestBase(unittest.TestCase):
             'max_ratio': 1.5,
             'prob': 1
         }]
-        mapper = tf.build(ops_conf)
+        mapper = tf.build_mapper(ops_conf)
         self.assertTrue(mapper is not None)
         data = self.sample.copy()
         result = mapper(data)
@@ -130,7 +130,7 @@ class TestBase(unittest.TestCase):
             'op': 'RandomInterpImage',
             'target_size': 608
         }]
-        mapper = tf.build(ops_conf)
+        mapper = tf.build_mapper(ops_conf)
         self.assertTrue(mapper is not None)
         data = self.sample.copy()
         result = mapper(data)

ppdet/data/tests/test_reader.py

@@ -7,7 +7,7 @@ import numpy as np
 import yaml
 
 import set_env
-from data import Reader
+from ppdet.data.reader import Reader
 
 
 class TestReader(unittest.TestCase):

ppdet/data/tests/test_roidb_source.py

@@ -5,7 +5,7 @@ import sys
 import logging
 
 import set_env
-from data import build_source
+from ppdet.data.source import build_source
 
 
 class TestRoiDbSource(unittest.TestCase):

ppdet/data/tests/test_transformer.py

@@ -6,8 +6,8 @@ import logging
 import numpy as np
 
 import set_env
-from data import build_source
-from data import transform as tf
+import ppdet.data.transform as tf
+from ppdet.data.source import build_source
 
 logger = logging.getLogger(__name__)
 
@@ -53,7 +53,7 @@ class TestTransformer(unittest.TestCase):
     def test_map(self):
         """ test transformer.map
         """
-        mapper = tf.build(self.ops)
+        mapper = tf.build_mapper(self.ops)
         ds = build_source(self.sc_config)
         mapped_ds = tf.map(ds, mapper)
         ct = 0
@@ -66,7 +66,7 @@ class TestTransformer(unittest.TestCase):
     def test_parallel_map(self):
         """ test transformer.map with concurrent workers
         """
-        mapper = tf.build(self.ops)
+        mapper = tf.build_mapper(self.ops)
         ds = build_source(self.sc_config)
         worker_conf = {'WORKER_NUM': 2, 'use_process': True}
         mapped_ds = tf.map(ds, mapper, worker_conf)
@@ -91,7 +91,7 @@ class TestTransformer(unittest.TestCase):
         """ test batched dataset
         """
         batchsize = 2
-        mapper = tf.build(self.ops)
+        mapper = tf.build_mapper(self.ops)
         ds = build_source(self.sc_config)
         mapped_ds = tf.map(ds, mapper)
         batched_ds = tf.batch(mapped_ds, batchsize, True)

ppdet/data/transform/__init__.py

@@ -17,6 +17,7 @@ from __future__ import print_function
 
 import copy
 import logging
+import traceback
 
 from .transformer import MappedDataset, BatchedDataset
 from .post_map import build_post_map
@@ -76,7 +77,10 @@ def build_mapper(ops, context=None):
                 out = f(sample, ctx)
                 sample = out
             except Exception as e:
-                logger.warn("fail to map op [{}] with error: {}".format(f, e))
+                stack_info = traceback.format_exc()
+                logger.warn("fail to map op [{}] with error: {} and stack:\n{}".format(f, e, str(stack_info)))
+                raise e
+
         return out
 
     _mapper.ops = op_repr

ppdet/data/transform/operators.py

@@ -540,7 +540,9 @@ class CropImage(BaseOperator):
         gt_class = sample['gt_class']
         im_width = sample['w']
         im_height = sample['h']
-        gt_score = sample['gt_score']
+        gt_score = None
+        if 'gt_score' in sample:
+            gt_score = sample['gt_score']
         sampled_bbox = []
         gt_bbox = gt_bbox.tolist()
         for sampler in self.batch_sampler: