updated test util file + new BoundingBoxCheck + fixed VOCDataset annotations

Style Error fix

fixed PyLint problem

reverting testVOC2012 due to CI break for existing test

reverting testVOC2012 due to CI break for existing test-2

updated old error messages to confirm with global standard

addressing PR 2355 Comments - 1

addressing PR 2355 Comments - 2

addressing PR 2355 Comments - 3

mindspore/ccsrc/dataset/kernels/image/random_vertical_flip_with_bbox_op.cc

@@ -41,7 +41,7 @@ Status RandomVerticalFlipWithBBoxOp::Compute(const TensorRow &input, TensorRow *
       RETURN_IF_NOT_OK(input[1]->GetUnsignedIntAt(&boxHeight, {i, 3}));    // get height of bbox
 
       // subtract (curCorner + height) from (max) for new Corner position
-      newBoxCorner_y = (imHeight - 1) - (boxCorner_y + boxHeight);
+      newBoxCorner_y = (imHeight - 1) - ((boxCorner_y + boxHeight) - 1);
       RETURN_IF_NOT_OK(input[1]->SetItemAt({i, 1}, newBoxCorner_y));
     }
 

mindspore/ccsrc/dataset/kernels/tensor_op.h

@@ -45,14 +45,18 @@
 
 #define BOUNDING_BOX_CHECK(input)                                                           \
   do {                                                                                      \
+    if (input.size() != 2) {                                                                \
+      return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__,               \
+                    "Requires Image and Bounding Boxes, likely missed bounding boxes.");    \
+    }                                                                                       \
     if (input[1]->shape().Size() < 2) {                                                     \
       return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__,               \
-                    "Bounding boxes shape should have at least two dims");                  \
+                    "Bounding boxes shape should have at least two dimensions.");           \
     }                                                                                       \
     uint32_t num_of_features = input[1]->shape()[1];                                        \
     if (num_of_features < 4) {                                                              \
       return Status(StatusCode::kBoundingBoxInvalidShape, __LINE__, __FILE__,               \
-                    "Bounding boxes should be have at least 4 features");                   \
+                    "Bounding boxes should be have at least 4 features.");                  \
     }                                                                                       \
     uint32_t num_of_boxes = input[1]->shape()[0];                                           \
     uint32_t img_h = input[0]->shape()[0];                                                  \

tests/ut/data/dataset/testVOC2012/Annotations/129.xml

+<annotation>
+	<folder>VOC2012</folder>
+	<filename>129.jpg</filename>
+	<source>
+		<database>simulate VOC2007 Database</database>
+		<annotation>simulate VOC2007</annotation>
+		<image>flickr</image>
+	</source>
+	<size>
+		<width>500</width>
+		<height>375</height>
+		<depth>3</depth>
+	</size>
+	<segmented>1</segmented>
+	<object>
+		<name>dog</name>
+		<pose>Frontal</pose>
+		<truncated>0</truncated>
+		<difficult>0</difficult>
+		<bndbox>
+			<xmin>1124</xmin>
+			<ymin>437</ymin>
+			<xmax>1684</xmax>
+			<ymax>2669</ymax>
+		</bndbox>
+	</object>
+</annotation>

tests/ut/data/dataset/testVOC2012_2/README.txt

+Custom VOC2012-like dataset with valid annotations for images.
+Created to test BoundingBox Augmentation Ops - June 2020.
\ No newline at end of file

tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py

@@ -305,8 +305,8 @@ def test_c_random_resized_crop_with_bbox_op_bad():
 
 
 if __name__ == "__main__":
-    test_c_random_resized_crop_with_bbox_op(False)
-    test_c_random_resized_crop_with_bbox_op_edge(False)
+    test_c_random_resized_crop_with_bbox_op(plot_vis=True)
+    test_c_random_resized_crop_with_bbox_op_edge(plot_vis=True)
     test_c_random_resized_crop_with_bbox_op_invalid()
     test_c_random_resized_crop_with_bbox_op_invalid2()
     test_c_random_resized_crop_with_bbox_op_bad()

tests/ut/python/dataset/test_random_crop_with_bbox.py

@@ -142,7 +142,7 @@ def gen_bbox_edge(im, bbox):
     return im, bbox
 
 
-def c_random_crop_with_bbox_op(plot_vis=False):
+def test_random_crop_with_bbox_op_c(plot_vis=False):
     """
      Prints images side by side with and without Aug applied + bboxes
     """
@@ -176,7 +176,7 @@ def c_random_crop_with_bbox_op(plot_vis=False):
         visualize(unaugSamp, augSamp)
 
 
-def c_random_crop_with_bbox_op2(plot_vis=False):
+def test_random_crop_with_bbox_op2_c(plot_vis=False):
     """
      Prints images side by side with and without Aug applied + bboxes
      With Fill Value
@@ -212,7 +212,7 @@ def c_random_crop_with_bbox_op2(plot_vis=False):
         visualize(unaugSamp, augSamp)
 
 
-def c_random_crop_with_bbox_op3(plot_vis=False):
+def test_random_crop_with_bbox_op3_c(plot_vis=False):
     """
      Prints images side by side with and without Aug applied + bboxes
      With Padding Mode passed
@@ -247,7 +247,7 @@ def c_random_crop_with_bbox_op3(plot_vis=False):
         visualize(unaugSamp, augSamp)
 
 
-def c_random_crop_with_bbox_op_edge(plot_vis=False):
+def test_random_crop_with_bbox_op_edge_c(plot_vis=False):
     """
      Prints images side by side with and without Aug applied + bboxes
      Testing for an Edge case
@@ -289,7 +289,7 @@ def c_random_crop_with_bbox_op_edge(plot_vis=False):
         visualize(unaugSamp, augSamp)
 
 
-def c_random_crop_with_bbox_op_invalid():
+def test_random_crop_with_bbox_op_invalid_c():
     """
     Checking for invalid params passed to Aug Constructor
     """
@@ -319,7 +319,7 @@ def c_random_crop_with_bbox_op_invalid():
         assert "Size should be a single integer" in str(err)
 
 
-def c_random_crop_with_bbox_op_bad():
+def test_random_crop_with_bbox_op_bad_c():
     # Should Fail - Errors logged to logger
     for ix, badFunc in enumerate(badGenFuncs):
         try:
@@ -352,9 +352,9 @@ def c_random_crop_with_bbox_op_bad():
 
 
 if __name__ == "__main__":
-    c_random_crop_with_bbox_op(False)
-    c_random_crop_with_bbox_op2(False)
-    c_random_crop_with_bbox_op3(False)
-    c_random_crop_with_bbox_op_edge(False)
-    c_random_crop_with_bbox_op_invalid()
-    c_random_crop_with_bbox_op_bad()
+    test_random_crop_with_bbox_op_c(plot_vis=True)
+    test_random_crop_with_bbox_op2_c(plot_vis=True)
+    test_random_crop_with_bbox_op3_c(plot_vis=True)
+    test_random_crop_with_bbox_op_edge_c(plot_vis=True)
+    test_random_crop_with_bbox_op_invalid_c()
+    test_random_crop_with_bbox_op_bad_c()

tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py

@@ -15,15 +15,12 @@
 """
 Testing RandomVerticalFlipWithBBox op
 """
-import numpy as np
-
-import matplotlib.pyplot as plt
-import matplotlib.patches as patches
 
 import mindspore.dataset as ds
 import mindspore.dataset.transforms.vision.c_transforms as c_vision
 
 from mindspore import log as logger
+from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox
 
 # updated VOC dataset with correct annotations
 DATA_DIR = "../data/dataset/testVOC2012_2"
@@ -46,106 +43,11 @@ def fix_annotate(bboxes):
     return bboxes
 
 
-def add_bounding_boxes(ax, bboxes):
-    for bbox in bboxes:
-        rect = patches.Rectangle((bbox[0], bbox[1]),
-                                 bbox[2], bbox[3],
-                                 linewidth=1, edgecolor='r', facecolor='none')
-        # Add the patch to the Axes
-        ax.add_patch(rect)
-
-
-def vis_check(orig, aug):
-    if not isinstance(orig, list) or not isinstance(aug, list):
-        return False
-    if len(orig) != len(aug):
-        return False
-    return True
-
-
-def visualize(orig, aug):
-
-    if not vis_check(orig, aug):
-        return
-
-    plotrows = 3
-    compset = int(len(orig)/plotrows)
-
-    orig, aug = np.array(orig), np.array(aug)
-
-    orig = np.split(orig[:compset*plotrows], compset) + [orig[compset*plotrows:]]
-    aug = np.split(aug[:compset*plotrows], compset) + [aug[compset*plotrows:]]
-
-    for ix, allData in enumerate(zip(orig, aug)):
-        base_ix = ix * plotrows  # will signal what base level we're on
-        fig, axs = plt.subplots(len(allData[0]), 2)
-        fig.tight_layout(pad=1.5)
-
-        for x, (dataA, dataB) in enumerate(zip(allData[0], allData[1])):
-            cur_ix = base_ix + x
-
-            axs[x, 0].imshow(dataA["image"])
-            add_bounding_boxes(axs[x, 0], dataA["annotation"])
-            axs[x, 0].title.set_text("Original" + str(cur_ix+1))
-            print("Original **\n ", str(cur_ix+1), " :", dataA["annotation"])
-
-            axs[x, 1].imshow(dataB["image"])
-            add_bounding_boxes(axs[x, 1], dataB["annotation"])
-            axs[x, 1].title.set_text("Augmented" + str(cur_ix+1))
-            print("Augmented **\n", str(cur_ix+1), " ", dataB["annotation"], "\n")
-
-        plt.show()
-
-# Functions to pass to Gen for creating invalid bounding boxes
-
-
-def gen_bad_bbox_neg_xy(im, bbox):
-    im_h, im_w = im.shape[0], im.shape[1]
-    bbox[0][:4] = [-50, -50, im_w - 10, im_h - 10]
-    return im, bbox
-
-
-def gen_bad_bbox_overflow_width(im, bbox):
-    im_h, im_w = im.shape[0], im.shape[1]
-    bbox[0][:4] = [0, 0, im_w + 10, im_h - 10]
-    return im, bbox
-
-
-def gen_bad_bbox_overflow_height(im, bbox):
-    im_h, im_w = im.shape[0], im.shape[1]
-    bbox[0][:4] = [0, 0, im_w - 10, im_h + 10]
-    return im, bbox
-
-
-def gen_bad_bbox_wrong_shape(im, bbox):
-    bbox = np.array([[0, 0, 0]]).astype(bbox.dtype)
-    return im, bbox
-
-
-badGenFuncs = [gen_bad_bbox_neg_xy,
-               gen_bad_bbox_overflow_width,
-               gen_bad_bbox_overflow_height,
-               gen_bad_bbox_wrong_shape]
-
-assertVal = ["min_x",
-             "is out of bounds of the image",
-             "is out of bounds of the image",
-             "4 features"]
-
-
-# Gen Edge case BBox
-def gen_bbox_edge(im, bbox):
-    im_h, im_w = im.shape[0], im.shape[1]
-    bbox[0][:4] = [0, 0, im_w, im_h]
-    return im, bbox
-
-
-def c_random_vertical_flip_with_bbox_op(plot_vis=False):
+def test_random_vertical_flip_with_bbox_op_c(plot_vis=False):
     """
      Prints images side by side with and without Aug applied + bboxes to
      compare and test
     """
-
     # Load dataset
     dataVoc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train",
                              decode=True, shuffle=False)
@@ -175,10 +77,10 @@ def c_random_vertical_flip_with_bbox_op(plot_vis=False):
         augSamp.append(Aug)
 
     if plot_vis:
-        visualize(unaugSamp, augSamp)
+        visualize_with_bounding_boxes(unaugSamp, augSamp)
 
 
-def c_random_vertical_flip_with_bbox_op_rand(plot_vis=False):
+def test_random_vertical_flip_with_bbox_op_rand_c(plot_vis=False):
     """
      Prints images side by side with and without Aug applied + bboxes to
      compare and test
@@ -213,54 +115,12 @@ def c_random_vertical_flip_with_bbox_op_rand(plot_vis=False):
         augSamp.append(Aug)
 
     if plot_vis:
-        visualize(unaugSamp, augSamp)
-
+        visualize_with_bounding_boxes(unaugSamp, augSamp)
 
-def c_random_vertical_flip_with_bbox_op_edge(plot_vis=False):
-    # Should Pass
-    # Load dataset
-    dataVoc1 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train",
-                             decode=True, shuffle=False)
-
-    dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train",
-                             decode=True, shuffle=False)
 
-    test_op = c_vision.RandomVerticalFlipWithBBox(0.6)
-
-    # maps to fix annotations to HQ standard
-    dataVoc1 = dataVoc1.map(input_columns=["annotation"],
-                            output_columns=["annotation"],
-                            operations=fix_annotate)
-    dataVoc2 = dataVoc2.map(input_columns=["annotation"],
-                            output_columns=["annotation"],
-                            operations=fix_annotate)
-
-    # Modify BBoxes to serve as valid edge cases
-    dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"],
-                            output_columns=["image", "annotation"],
-                            columns_order=["image", "annotation"],
-                            operations=[gen_bbox_edge])
-
-    # map to apply ops
-    dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"],
-                            output_columns=["image", "annotation"],
-                            columns_order=["image", "annotation"],
-                            operations=[test_op])
-
-    unaugSamp, augSamp = [], []
-
-    for unAug, Aug in zip(dataVoc1.create_dict_iterator(), dataVoc2.create_dict_iterator()):
-        unaugSamp.append(unAug)
-        augSamp.append(Aug)
-
-    if plot_vis:
-        visualize(unaugSamp, augSamp)
-
-
-def c_random_vertical_flip_with_bbox_op_invalid():
+def test_random_vertical_flip_with_bbox_op_invalid_c():
     # Should Fail
     # Load dataset
-
     dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train",
                              decode=True, shuffle=False)
 
@@ -286,41 +146,25 @@ def c_random_vertical_flip_with_bbox_op_invalid():
         assert "Input is not" in str(err)
 
 
-def c_random_vertical_flip_with_bbox_op_bad():
-    # Should Fail - Errors logged to logger
-    for ix, badFunc in enumerate(badGenFuncs):
-        try:
-            dataVoc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train",
-                                     decode=True, shuffle=False)
-
-            test_op = c_vision.RandomVerticalFlipWithBBox(1)
-
-            dataVoc2 = dataVoc2.map(input_columns=["annotation"],
-                                    output_columns=["annotation"],
-                                    operations=fix_annotate)
-
-            dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"],
-                                    output_columns=["image", "annotation"],
-                                    columns_order=["image", "annotation"],
-                                    operations=[badFunc])
-
-            # map to apply ops
-            dataVoc2 = dataVoc2.map(input_columns=["image", "annotation"],
-                                    output_columns=["image", "annotation"],
-                                    columns_order=["image", "annotation"],
-                                    operations=[test_op])
-
-            for _ in dataVoc2.create_dict_iterator():
-                break  # first sample will cause exception
+def test_random_vertical_flip_with_bbox_op_bad_c():
+    """
+    Test RandomHorizontalFlipWithBBox op with invalid bounding boxes
+    """
+    logger.info("test_random_horizontal_bbox_invalid_bounds_c")
+    test_op = c_vision.RandomVerticalFlipWithBBox(1)
 
-        except RuntimeError as err:
-            logger.info("Got an exception in DE: {}".format(str(err)))
-            assert assertVal[ix] in str(err)
+    data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False)
+    check_bad_bbox(data_voc2, test_op, InvalidBBoxType.WidthOverflow, "bounding boxes is out of bounds of the image")
+    data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False)
+    check_bad_bbox(data_voc2, test_op, InvalidBBoxType.HeightOverflow, "bounding boxes is out of bounds of the image")
+    data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False)
+    check_bad_bbox(data_voc2, test_op, InvalidBBoxType.NegativeXY, "min_x")
+    data_voc2 = ds.VOCDataset(DATA_DIR, task="Detection", mode="train", decode=True, shuffle=False)
+    check_bad_bbox(data_voc2, test_op, InvalidBBoxType.WrongShape, "4 features")
 
 
 if __name__ == "__main__":
-    c_random_vertical_flip_with_bbox_op(False)
-    c_random_vertical_flip_with_bbox_op_rand(False)
-    c_random_vertical_flip_with_bbox_op_edge(False)
-    c_random_vertical_flip_with_bbox_op_invalid()
-    c_random_vertical_flip_with_bbox_op_bad()
+    test_random_vertical_flip_with_bbox_op_c(plot_vis=True)
+    test_random_vertical_flip_with_bbox_op_rand_c(plot_vis=True)
+    test_random_vertical_flip_with_bbox_op_invalid_c()
+    test_random_vertical_flip_with_bbox_op_bad_c()

tests/ut/python/dataset/util.py

@@ -16,7 +16,9 @@
 import hashlib
 import json
 import os
+from enum import Enum
 import matplotlib.pyplot as plt
+import matplotlib.patches as patches
 import numpy as np
 # import jsbeautifier
 import mindspore.dataset as ds
@@ -284,3 +286,124 @@ def config_get_set_num_parallel_workers(num_parallel_workers_new):
     logger.info("num_parallel_workers: original = {}  new = {} ".format(num_parallel_workers_original,
                                                                         num_parallel_workers_new))
     return num_parallel_workers_original
+
+
+def visualize_with_bounding_boxes(orig, aug, plot_rows=3):
+    """
+    Take a list of un-augmented and augmented images with "annotation" bounding boxes
+    Plot images to compare test correct BBox augment functionality
+    :param orig: list of original images and bboxes (without aug)
+    :param aug: list of augmented images and bboxes
+    :param plot_rows: number of rows on plot (rows = samples on one plot)
+    :return: None
+    """
+
+    def add_bounding_boxes(ax, bboxes):
+        for bbox in bboxes:
+            rect = patches.Rectangle((bbox[0], bbox[1]),
+                                     bbox[2], bbox[3],
+                                     linewidth=1, edgecolor='r', facecolor='none')
+            # Add the patch to the Axes
+            ax.add_patch(rect)
+
+    # Quick check to confirm correct input parameters
+    if not isinstance(orig, list) or not isinstance(aug, list):
+        return
+    if len(orig) != len(aug) or not orig:
+        return
+
+    comp_set = int(len(orig)/plot_rows)
+    orig, aug = np.array(orig), np.array(aug)
+
+    if len(orig) > plot_rows:
+        orig = np.split(orig[:comp_set*plot_rows], comp_set) + [orig[comp_set*plot_rows:]]
+        aug = np.split(aug[:comp_set*plot_rows], comp_set) + [aug[comp_set*plot_rows:]]
+    else:
+        orig = [orig]
+        aug = [aug]
+
+    for ix, allData in enumerate(zip(orig, aug)):
+        base_ix = ix * plot_rows  # will signal what base level we're on
+
+        sub_plot_count = 2 if (len(allData[0]) < 2) else len(allData[0])  # if 1 image remains, create subplot for 2 to simplify axis selection
+        fig, axs = plt.subplots(sub_plot_count, 2)
+        fig.tight_layout(pad=1.5)
+
+        for x, (dataA, dataB) in enumerate(zip(allData[0], allData[1])):
+            cur_ix = base_ix + x
+
+            axs[x, 0].imshow(dataA["image"])
+            add_bounding_boxes(axs[x, 0], dataA["annotation"])
+            axs[x, 0].title.set_text("Original" + str(cur_ix+1))
+            logger.info("Original **\n{} : {}".format(str(cur_ix+1), dataA["annotation"]))
+
+            axs[x, 1].imshow(dataB["image"])
+            add_bounding_boxes(axs[x, 1], dataB["annotation"])
+            axs[x, 1].title.set_text("Augmented" + str(cur_ix+1))
+            logger.info("Augmented **\n{} : {}\n".format(str(cur_ix+1), dataB["annotation"]))
+
+        plt.show()
+
+
+class InvalidBBoxType(Enum):
+    """
+    Defines Invalid Bounding Bbox types for test cases
+    """
+    WidthOverflow = 1
+    HeightOverflow = 2
+    NegativeXY = 3
+    WrongShape = 4
+
+
+def check_bad_bbox(data, test_op, invalid_bbox_type, expected_error):
+    """
+    :param data: de object detection pipeline
+    :param test_op: Augmentation Op to test on image
+    :param invalid_bbox_type: type of bad box
+    :param expected_error: error expected to get due to bad box
+    :return: None
+    """
+
+    def add_bad_annotation(img, bboxes, invalid_bbox_type_):
+        """
+        Used to generate erroneous bounding box examples on given img.
+        :param img: image where the bounding boxes are.
+        :param bboxes: in [x_min, y_min, w, h, label, truncate, difficult] format
+        :param box_type_: type of bad box
+        :return: bboxes with bad examples added
+        """
+        height = img.shape[0]
+        width = img.shape[1]
+        if invalid_bbox_type_ == InvalidBBoxType.WidthOverflow:
+            # use box that overflows on width
+            return img, np.array([[0, 0, width + 1, height, 0, 0, 0]]).astype(np.uint32)
+
+        if invalid_bbox_type_ == InvalidBBoxType.HeightOverflow:
+            # use box that overflows on height
+            return img, np.array([[0, 0, width, height + 1, 0, 0, 0]]).astype(np.uint32)
+
+        if invalid_bbox_type_ == InvalidBBoxType.NegativeXY:
+            # use box with negative xy
+            return img, np.array([[-10, -10, width, height, 0, 0, 0]]).astype(np.uint32)
+
+        if invalid_bbox_type_ == InvalidBBoxType.WrongShape:
+            # use box that has incorrect shape
+            return img, np.array([[0, 0, width - 1]]).astype(np.uint32)
+        return img, bboxes
+
+    try:
+        # map to use selected invalid bounding box type
+        data = data.map(input_columns=["image", "annotation"],
+                        output_columns=["image", "annotation"],
+                        columns_order=["image", "annotation"],
+                        operations=lambda img, bboxes: add_bad_annotation(img, bboxes, invalid_bbox_type))
+        # map to apply ops
+        data = data.map(input_columns=["image", "annotation"],
+                        output_columns=["image", "annotation"],
+                        columns_order=["image", "annotation"],
+                        operations=[test_op])  # Add column for "annotation"
+        for _, _ in enumerate(data.create_dict_iterator()):
+            break
+    except RuntimeError as error:
+        logger.info("Got an exception in DE: {}".format(str(error)))
+        assert expected_error in str(error)