Update Python UA op, AutoContrast Op and UT

Updated Validators.py

Fixed UA UT

Fixed UA UT

Fixed PyLint Errors

Added aditional changes

remove old golden AutoContrast UT file

updated Autocontrast python UT

Added golden files

Fixed issue

fixed UT

mindspore/ccsrc/minddata/dataset/api/python_bindings.cc

@@ -425,7 +425,7 @@ void bindTensorOps1(py::module *m) {
 
   (void)py::class_<UniformAugOp, TensorOp, std::shared_ptr<UniformAugOp>>(
     *m, "UniformAugOp", "Tensor operation to apply random augmentation(s).")
-    .def(py::init<std::vector<std::shared_ptr<TensorOp>>, int32_t>(), py::arg("operations"),
+    .def(py::init<std::vector<std::shared_ptr<TensorOp>>, int32_t>(), py::arg("transforms"),
          py::arg("NumOps") = UniformAugOp::kDefNumOps);
 
   (void)py::class_<BoundingBoxAugmentOp, TensorOp, std::shared_ptr<BoundingBoxAugmentOp>>(

mindspore/ccsrc/minddata/dataset/api/transforms.cc

@@ -90,9 +90,9 @@ std::shared_ptr<CenterCropOperation> CenterCrop(std::vector<int32_t> size) {
 }
 
 // Function to create UniformAugOperation.
-std::shared_ptr<UniformAugOperation> UniformAugment(std::vector<std::shared_ptr<TensorOperation>> operations,
+std::shared_ptr<UniformAugOperation> UniformAugment(std::vector<std::shared_ptr<TensorOperation>> transforms,
                                                     int32_t num_ops) {
-  auto op = std::make_shared<UniformAugOperation>(operations, num_ops);
+  auto op = std::make_shared<UniformAugOperation>(transforms, num_ops);
   // Input validation
   if (!op->ValidateParams()) {
     return nullptr;
@@ -290,14 +290,14 @@ std::shared_ptr<TensorOp> CenterCropOperation::Build() {
 }
 
 // UniformAugOperation
-UniformAugOperation::UniformAugOperation(std::vector<std::shared_ptr<TensorOperation>> operations, int32_t num_ops)
-    : operations_(operations), num_ops_(num_ops) {}
+UniformAugOperation::UniformAugOperation(std::vector<std::shared_ptr<TensorOperation>> transforms, int32_t num_ops)
+    : transforms_(transforms), num_ops_(num_ops) {}
 
 bool UniformAugOperation::ValidateParams() { return true; }
 
 std::shared_ptr<TensorOp> UniformAugOperation::Build() {
   std::vector<std::shared_ptr<TensorOp>> tensor_ops;
-  (void)std::transform(operations_.begin(), operations_.end(), std::back_inserter(tensor_ops),
+  (void)std::transform(transforms_.begin(), transforms_.end(), std::back_inserter(tensor_ops),
                        [](std::shared_ptr<TensorOperation> op) -> std::shared_ptr<TensorOp> { return op->Build(); });
   std::shared_ptr<UniformAugOp> tensor_op = std::make_shared<UniformAugOp>(tensor_ops, num_ops_);
   return tensor_op;

mindspore/ccsrc/minddata/dataset/include/transforms.h

@@ -108,10 +108,10 @@ std::shared_ptr<CenterCropOperation> CenterCrop(std::vector<int32_t> size);
 
 /// \brief Function to create a UniformAugment TensorOperation.
 /// \notes Tensor operation to perform randomly selected augmentation.
-/// \param[in] operations - a vector of TensorOperation operations.
+/// \param[in] transforms - a vector of TensorOperation transforms.
 /// \param[in] num_ops - integer representing the number of OPs to be selected and applied.
 /// \return Shared pointer to the current TensorOperation.
-std::shared_ptr<UniformAugOperation> UniformAugment(std::vector<std::shared_ptr<TensorOperation>> operations,
+std::shared_ptr<UniformAugOperation> UniformAugment(std::vector<std::shared_ptr<TensorOperation>> transforms,
                                                     int32_t num_ops = 2);
 
 /// \brief Function to create a RandomHorizontalFlip TensorOperation.
@@ -264,7 +264,7 @@ class CenterCropOperation : public TensorOperation {
 
 class UniformAugOperation : public TensorOperation {
  public:
-  explicit UniformAugOperation(std::vector<std::shared_ptr<TensorOperation>> operations, int32_t num_ops = 2);
+  explicit UniformAugOperation(std::vector<std::shared_ptr<TensorOperation>> transforms, int32_t num_ops = 2);
 
   ~UniformAugOperation() = default;
 
@@ -273,7 +273,7 @@ class UniformAugOperation : public TensorOperation {
   bool ValidateParams() override;
 
  private:
-  std::vector<std::shared_ptr<TensorOperation>> operations_;
+  std::vector<std::shared_ptr<TensorOperation>> transforms_;
   int32_t num_ops_;
 };
 

mindspore/dataset/transforms/vision/c_transforms.py

@@ -722,7 +722,7 @@ class UniformAugment(cde.UniformAugOp):
     Tensor operation to perform randomly selected augmentation.
 
     Args:
-        operations: list of C++ operations (python OPs are not accepted).
+        transforms: list of C++ operations (python OPs are not accepted).
         num_ops (int, optional): number of OPs to be selected and applied (default=2).
 
     Examples:
@@ -730,7 +730,7 @@ class UniformAugment(cde.UniformAugOp):
         >>>                    c_transforms.RandomVerticalFlip(),
         >>>                    c_transforms.RandomColorAdjust(),
         >>>                    c_transforms.RandomRotation(degrees=45)]
-        >>> uni_aug = c_transforms.UniformAugment(operations=transforms_list, num_ops=2)
+        >>> uni_aug = c_transforms.UniformAugment(transforms=transforms_list, num_ops=2)
         >>> transforms_all = [c_transforms.Decode(), c_transforms.Resize(size=[224, 224]),
         >>>                   uni_aug, F.ToTensor()]
         >>> ds_ua = ds.map(input_columns="image",
@@ -738,10 +738,10 @@ class UniformAugment(cde.UniformAugOp):
     """
 
     @check_uniform_augment_cpp
-    def __init__(self, operations, num_ops=2):
-        self.operations = operations
+    def __init__(self, transforms, num_ops=2):
+        self.transforms = transforms
         self.num_ops = num_ops
-        super().__init__(operations, num_ops)
+        super().__init__(transforms, num_ops)
 
 
 class RandomSelectSubpolicy(cde.RandomSelectSubpolicyOp):

mindspore/dataset/transforms/vision/py_transforms.py

@@ -33,7 +33,7 @@ from .validators import check_prob, check_crop, check_resize_interpolation, chec
     check_normalize_py, check_random_crop, check_random_color_adjust, check_random_rotation, \
     check_transforms_list, check_random_apply, check_ten_crop, check_num_channels, check_pad, \
     check_random_perspective, check_random_erasing, check_cutout, check_linear_transform, check_random_affine, \
-    check_mix_up, check_positive_degrees, check_uniform_augment_py, check_compose_list
+    check_mix_up, check_positive_degrees, check_uniform_augment_py, check_compose_list, check_auto_contrast
 from .utils import Inter, Border
 
 DE_PY_INTER_MODE = {Inter.NEAREST: Image.NEAREST,
@@ -1361,6 +1361,10 @@ class AutoContrast:
     """
     Automatically maximize the contrast of the input PIL image.
 
+    Args:
+        cutoff (float, optional): Percent of pixels to cut off from the histogram (default=0.0).
+        ignore (int or sequence, optional): Pixel values to ignore (default=None).
+
     Examples:
         >>> py_transforms.ComposeOp([py_transforms.Decode(),
         >>>                          py_transforms.AutoContrast(),
@@ -1368,6 +1372,11 @@ class AutoContrast:
 
     """
 
+    @check_auto_contrast
+    def __init__(self, cutoff=0.0, ignore=None):
+        self.cutoff = cutoff
+        self.ignore = ignore
+
     def __call__(self, img):
         """
         Call method.
@@ -1379,7 +1388,7 @@ class AutoContrast:
             img (PIL Image), Augmented image.
         """
 
-        return util.auto_contrast(img)
+        return util.auto_contrast(img, self.cutoff, self.ignore)
 
 
 class Invert:

mindspore/dataset/transforms/vision/py_transforms_util.py

@@ -1457,13 +1457,15 @@ def random_sharpness(img, degrees):
     return ImageEnhance.Sharpness(img).enhance(v)
 
 
-def auto_contrast(img):
+def auto_contrast(img, cutoff, ignore):
 
     """
     Automatically maximize the contrast of the input PIL image.
 
     Args:
         img (PIL Image): Image to be augmented with AutoContrast.
+        cutoff (float, optional): Percent of pixels to cut off from the histogram (default=0.0).
+        ignore (int or sequence, optional): Pixel values to ignore (default=None).
 
     Returns:
         img (PIL Image), Augmented image.
@@ -1473,7 +1475,7 @@ def auto_contrast(img):
     if not is_pil(img):
         raise TypeError('img should be PIL Image. Got {}'.format(type(img)))
 
-    return ImageOps.autocontrast(img)
+    return ImageOps.autocontrast(img, cutoff, ignore)
 
 
 def invert_color(img):

mindspore/dataset/transforms/vision/validators.py

@@ -506,13 +506,13 @@ def check_uniform_augment_cpp(method):
 
     @wraps(method)
     def new_method(self, *args, **kwargs):
-        [operations, num_ops], _ = parse_user_args(method, *args, **kwargs)
+        [transforms, num_ops], _ = parse_user_args(method, *args, **kwargs)
         type_check(num_ops, (int,), "num_ops")
         check_positive(num_ops, "num_ops")
 
-        if num_ops > len(operations):
-            raise ValueError("num_ops is greater than operations list size")
-        type_check_list(operations, (TensorOp,), "tensor_ops")
+        if num_ops > len(transforms):
+            raise ValueError("num_ops is greater than transforms list size")
+        type_check_list(transforms, (TensorOp,), "tensor_ops")
 
         return method(self, *args, **kwargs)
 

tests/ut/python/dataset/test_autocontrast.py

@@ -58,7 +58,7 @@ def test_auto_contrast_py(plot=False):
 
     transforms_auto_contrast = F.ComposeOp([F.Decode(),
                                             F.Resize((224, 224)),
-                                            F.AutoContrast(),
+                                            F.AutoContrast(cutoff=10.0, ignore=[10, 20]),
                                             F.ToTensor()])
 
     ds_auto_contrast = ds.map(input_columns="image",
@@ -99,8 +99,8 @@ def test_auto_contrast_c(plot=False):
     ds = ds.map(input_columns=["image"],
                 operations=[C.Decode(),
                             C.Resize((224, 224))])
-    python_op = F.AutoContrast()
-    c_op = C.AutoContrast()
+    python_op = F.AutoContrast(cutoff=10.0, ignore=[10, 20])
+    c_op = C.AutoContrast(cutoff=10.0, ignore=[10, 20])
     transforms_op = F.ComposeOp([lambda img: F.ToPIL()(img.astype(np.uint8)),
                                  python_op,
                                  np.array])()
@@ -143,6 +143,10 @@ def test_auto_contrast_c(plot=False):
     logger.info("MSE= {}".format(str(np.mean(mse))))
     np.testing.assert_equal(np.mean(mse), 0.0)
 
+    # Compare with expected md5 from images
+    filename = "autcontrast_01_result_c.npz"
+    save_and_check_md5(ds_auto_contrast_c, filename, generate_golden=GENERATE_GOLDEN)
+
     if plot:
         visualize_list(images_auto_contrast_c, images_auto_contrast_py, visualize_mode=2)
 
@@ -209,11 +213,11 @@ def test_auto_contrast_one_channel_c(plot=False):
         visualize_list(images_auto_contrast_c, images_auto_contrast_py, visualize_mode=2)
 
 
-def test_auto_contrast_invalid_input_c():
+def test_auto_contrast_invalid_ignore_param_c():
     """
-    Test AutoContrast C Op with invalid params
+    Test AutoContrast C Op with invalid ignore parameter
     """
-    logger.info("Test AutoContrast C Op with invalid params")
+    logger.info("Test AutoContrast C Op with invalid ignore parameter")
     try:
         ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
         ds = ds.map(input_columns=["image"],
@@ -226,10 +230,110 @@ def test_auto_contrast_invalid_input_c():
     except TypeError as error:
         logger.info("Got an exception in DE: {}".format(str(error)))
         assert "Argument ignore with value 255.5 is not of type" in str(error)
+    try:
+        ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
+        ds = ds.map(input_columns=["image"],
+                    operations=[C.Decode(),
+                                C.Resize((224, 224)),
+                                lambda img: np.array(img[:, :, 0])])
+        # invalid ignore
+        ds = ds.map(input_columns="image",
+                    operations=C.AutoContrast(ignore=(10, 100)))
+    except TypeError as error:
+        logger.info("Got an exception in DE: {}".format(str(error)))
+        assert "Argument ignore with value (10,100) is not of type" in str(error)
+
+
+def test_auto_contrast_invalid_cutoff_param_c():
+    """
+    Test AutoContrast C Op with invalid cutoff parameter
+    """
+    logger.info("Test AutoContrast C Op with invalid cutoff parameter")
+    try:
+        ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
+        ds = ds.map(input_columns=["image"],
+                    operations=[C.Decode(),
+                                C.Resize((224, 224)),
+                                lambda img: np.array(img[:, :, 0])])
+        # invalid ignore
+        ds = ds.map(input_columns="image",
+                    operations=C.AutoContrast(cutoff=-10.0))
+    except ValueError as error:
+        logger.info("Got an exception in DE: {}".format(str(error)))
+        assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
+    try:
+        ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
+        ds = ds.map(input_columns=["image"],
+                    operations=[C.Decode(),
+                                C.Resize((224, 224)),
+                                lambda img: np.array(img[:, :, 0])])
+        # invalid ignore
+        ds = ds.map(input_columns="image",
+                    operations=C.AutoContrast(cutoff=120.0))
+    except ValueError as error:
+        logger.info("Got an exception in DE: {}".format(str(error)))
+        assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
+
+
+def test_auto_contrast_invalid_ignore_param_py():
+    """
+    Test AutoContrast python Op with invalid ignore parameter
+    """
+    logger.info("Test AutoContrast python Op with invalid ignore parameter")
+    try:
+        ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
+        ds = ds.map(input_columns=["image"],
+                    operations=[F.ComposeOp([F.Decode(),
+                                             F.Resize((224, 224)),
+                                             F.AutoContrast(ignore=255.5),
+                                             F.ToTensor()])])
+    except TypeError as error:
+        logger.info("Got an exception in DE: {}".format(str(error)))
+        assert "Argument ignore with value 255.5 is not of type" in str(error)
+    try:
+        ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
+        ds = ds.map(input_columns=["image"],
+                    operations=[F.ComposeOp([F.Decode(),
+                                             F.Resize((224, 224)),
+                                             F.AutoContrast(ignore=(10, 100)),
+                                             F.ToTensor()])])
+    except TypeError as error:
+        logger.info("Got an exception in DE: {}".format(str(error)))
+        assert "Argument ignore with value (10,100) is not of type" in str(error)
+
+
+def test_auto_contrast_invalid_cutoff_param_py():
+    """
+    Test AutoContrast python Op with invalid cutoff parameter
+    """
+    logger.info("Test AutoContrast python Op with invalid cutoff parameter")
+    try:
+        ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
+        ds = ds.map(input_columns=["image"],
+                    operations=[F.ComposeOp([F.Decode(),
+                                             F.Resize((224, 224)),
+                                             F.AutoContrast(cutoff=-10.0),
+                                             F.ToTensor()])])
+    except ValueError as error:
+        logger.info("Got an exception in DE: {}".format(str(error)))
+        assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
+    try:
+        ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
+        ds = ds.map(input_columns=["image"],
+                    operations=[F.ComposeOp([F.Decode(),
+                                             F.Resize((224, 224)),
+                                             F.AutoContrast(cutoff=120.0),
+                                             F.ToTensor()])])
+    except ValueError as error:
+        logger.info("Got an exception in DE: {}".format(str(error)))
+        assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
 
 
 if __name__ == "__main__":
     test_auto_contrast_py(plot=True)
     test_auto_contrast_c(plot=True)
     test_auto_contrast_one_channel_c(plot=True)
-    test_auto_contrast_invalid_input_c()
+    test_auto_contrast_invalid_ignore_param_c()
+    test_auto_contrast_invalid_ignore_param_py()
+    test_auto_contrast_invalid_cutoff_param_c()
+    test_auto_contrast_invalid_cutoff_param_py()

tests/ut/python/dataset/test_uniform_augment.py

@@ -124,7 +124,7 @@ def test_cpp_uniform_augment(plot=False, num_ops=2):
                      C.RandomColorAdjust(),
                      C.RandomRotation(degrees=45)]
 
-    uni_aug = C.UniformAugment(operations=transforms_ua, num_ops=num_ops)
+    uni_aug = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
 
     transforms_all = [C.Decode(), C.Resize(size=[224, 224]),
                       uni_aug,
@@ -166,7 +166,7 @@ def test_cpp_uniform_augment_exception_pyops(num_ops=2):
                      F.Invert()]
 
     with pytest.raises(TypeError) as e:
-        C.UniformAugment(operations=transforms_ua, num_ops=num_ops)
+        C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
 
     logger.info("Got an exception in DE: {}".format(str(e)))
     assert "Argument tensor_ops[5] with value" \
@@ -187,7 +187,7 @@ def test_cpp_uniform_augment_exception_large_numops(num_ops=6):
                      C.RandomRotation(degrees=45)]
 
     try:
-        _ = C.UniformAugment(operations=transforms_ua, num_ops=num_ops)
+        _ = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
 
     except Exception as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
@@ -207,7 +207,7 @@ def test_cpp_uniform_augment_exception_nonpositive_numops(num_ops=0):
                      C.RandomRotation(degrees=45)]
 
     try:
-        _ = C.UniformAugment(operations=transforms_ua, num_ops=num_ops)
+        _ = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
 
     except Exception as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
@@ -227,7 +227,7 @@ def test_cpp_uniform_augment_exception_float_numops(num_ops=2.5):
                      C.RandomRotation(degrees=45)]
 
     try:
-        _ = C.UniformAugment(operations=transforms_ua, num_ops=num_ops)
+        _ = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
 
     except Exception as e:
         logger.info("Got an exception in DE: {}".format(str(e)))
@@ -248,7 +248,7 @@ def test_cpp_uniform_augment_random_crop_badinput(num_ops=1):
         C.RandomCrop(size=[224, 224]),
         C.RandomHorizontalFlip()
     ]
-    uni_aug = C.UniformAugment(operations=transforms_ua, num_ops=num_ops)
+    uni_aug = C.UniformAugment(transforms=transforms_ua, num_ops=num_ops)
     ds1 = ds1.map(input_columns="image", operations=uni_aug)
 
     # apply DatasetOps