Skip to content

M
mindspore

magicwindyyd / mindspore
与 Fork 源项目一致

Fork自 MindSpore / mindspore

通知 1
Star 1
Fork 0
M
mindspore
提交 3bac9d37 编写于 作者: N nhussain
浏览文件
操作

switch input columns and operation 

change ImagefolderDV2 name

change ds.transforms.vision to ds.vision

change batch api to match map api more closely

compose op changes

test_pylint

remove compose op from vision, move to transform module, refactor map and batch to use column_order
上级 75045e3e
隐藏空白更改
内联 并排
Showing with 1339 addition and 1107 deletion
mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc
浏览文件 @ 3bac9d37
......@@ -733,7 +733,7 @@ Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr<DatasetOp> *
(void)map_builder.SetInColNames(in_col_names);
} else if (key == "output_columns") {
(void)map_builder.SetOutColNames(ToStringVector(value));
} else if (key == "columns_order") {
} else if (key == "column_order") {
project_columns = ToStringVector(value);
} else if (key == "num_parallel_workers") {
num_workers = ToInt(value);
......
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc
浏览文件 @ 3bac9d37
......@@ -113,7 +113,7 @@ Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) {
num_rows_ = image_label_pairs_.size();
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API ImageFolderDatasetV2.Please check file path or dataset "
"There is no valid data matching the dataset API ImageFolderDataset. Please check file path or dataset "
"API validation first.");
}
// free memory of two queues used for pre-scan
......
mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h
浏览文件 @ 3bac9d37
......@@ -111,7 +111,7 @@ constexpr char kWhitespaceTokenizerOp[] = "WhitespaceTokenizerOp";
constexpr char kWordpieceTokenizerOp[] = "WordpieceTokenizerOp";
constexpr char kRandomChoiceOp[] = "RandomChoiceOp";
constexpr char kRandomApplyOp[] = "RandomApplyOp";
constexpr char kComposeOp[] = "ComposeOp";
constexpr char kComposeOp[] = "Compose";
constexpr char kRandomSelectSubpolicyOp[] = "RandomSelectSubpolicyOp";
constexpr char kSentencepieceTokenizerOp[] = "SentencepieceTokenizerOp";
......
mindspore/dataset/__init__.py
浏览文件 @ 3bac9d37
......@@ -19,7 +19,7 @@ can also create samplers with this module to sample data.
"""
from .core import config
from .engine.datasets import TFRecordDataset, ImageFolderDatasetV2, MnistDataset, MindDataset, NumpySlicesDataset, \
from .engine.datasets import TFRecordDataset, ImageFolderDataset, MnistDataset, MindDataset, NumpySlicesDataset, \
GeneratorDataset, ManifestDataset, Cifar10Dataset, Cifar100Dataset, VOCDataset, CocoDataset, CelebADataset, \
TextFileDataset, CLUEDataset, CSVDataset, Schema, Shuffle, zip, RandomDataset, PaddedDataset
from .engine.samplers import DistributedSampler, PKSampler, RandomSampler, SequentialSampler, SubsetRandomSampler, \
......@@ -28,7 +28,7 @@ from .engine.cache_client import DatasetCache
from .engine.serializer_deserializer import serialize, deserialize, show
from .engine.graphdata import GraphData
__all__ = ["config", "ImageFolderDatasetV2", "MnistDataset", "PaddedDataset",
__all__ = ["config", "ImageFolderDataset", "MnistDataset", "PaddedDataset",
"MindDataset", "GeneratorDataset", "TFRecordDataset",
"ManifestDataset", "Cifar10Dataset", "Cifar100Dataset", "CelebADataset", "NumpySlicesDataset", "VOCDataset",
"CocoDataset", "TextFileDataset", "CLUEDataset", "CSVDataset", "Schema", "DistributedSampler", "PKSampler",
......
mindspore/dataset/core/py_util_helpers.py 0 → 100644
浏览文件 @ 3bac9d37
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
General py_transforms_utils functions.
"""
import numpy as np
def is_numpy(img):
"""
Check if the input image is Numpy format.
Args:
img: Image to be checked.
Returns:
Bool, True if input is Numpy image.
"""
return isinstance(img, np.ndarray)
mindspore/dataset/engine/__init__.py
浏览文件 @ 3bac9d37
......@@ -28,7 +28,7 @@ from .serializer_deserializer import serialize, deserialize, show, compare
from .samplers import *
from ..core import config
__all__ = ["config", "zip", "ImageFolderDatasetV2", "MnistDataset",
__all__ = ["config", "zip", "ImageFolderDataset", "MnistDataset",
"MindDataset", "GeneratorDataset", "TFRecordDataset", "CLUEDataset", "CSVDataset",
"ManifestDataset", "Cifar10Dataset", "Cifar100Dataset", "CelebADataset",
"VOCDataset", "CocoDataset", "TextFileDataset", "Schema", "DistributedSampler",
......
mindspore/dataset/engine/datasets.py
浏览文件 @ 3bac9d37
......@@ -41,7 +41,7 @@ from . import samplers
from .iterators import DictIterator, TupleIterator, DummyIterator, SaveOp, Iterator
from .validators import check_batch, check_shuffle, check_map, check_filter, check_repeat, check_skip, check_zip, \
check_rename, check_numpyslicesdataset, check_device_send, \
check_take, check_project, check_imagefolderdatasetv2, check_mnist_cifar_dataset, check_manifestdataset, \
check_take, check_project, check_imagefolderdataset, check_mnist_cifar_dataset, check_manifestdataset, \
check_tfrecorddataset, check_vocdataset, check_cocodataset, check_celebadataset, check_minddataset, \
check_generatordataset, check_sync_wait, check_zip_dataset, check_add_column, check_textfiledataset, check_concat, \
check_random_dataset, check_split, check_bucket_batch_by_length, check_cluedataset, check_save, check_csvdataset, \
......@@ -81,8 +81,8 @@ def zip(datasets):
>>>
>>> dataset_dir1 = "path/to/imagefolder_directory1"
>>> dataset_dir2 = "path/to/imagefolder_directory2"
>>> ds1 = ds.ImageFolderDatasetV2(dataset_dir1, num_parallel_workers=8)
>>> ds2 = ds.ImageFolderDatasetV2(dataset_dir2, num_parallel_workers=8)
>>> ds1 = ds.ImageFolderDataset(dataset_dir1, num_parallel_workers=8)
>>> ds2 = ds.ImageFolderDataset(dataset_dir2, num_parallel_workers=8)
>>>
>>> # creates a dataset which is the combination of ds1 and ds2
>>> data = ds.zip((ds1, ds2))
......@@ -246,7 +246,7 @@ class Dataset:
@check_batch
def batch(self, batch_size, drop_remainder=False, num_parallel_workers=None, per_batch_map=None,
input_columns=None, pad_info=None):
input_columns=None, output_columns=None, column_order=None, pad_info=None):
"""
Combine batch_size number of consecutive rows into batches.
......@@ -272,6 +272,18 @@ class Dataset:
The last parameter of the callable should always be a BatchInfo object.
input_columns (list[str], optional): List of names of the input columns. The size of the list should
match with signature of per_batch_map callable.
output_columns (list[str], optional): [Not currently implmented] List of names assigned to the columns
outputted by the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. (default=None, output columns will have the same
name as the input columns, i.e., the columns will be replaced).
column_order (list[str], optional): [Not currently implmented] list of all the desired columns to
propagate to the child node. This list must be a subset of all the columns in the dataset after
all operations are applied. The order of the columns in each row propagated to the
child node follow the order they appear in this list. The parameter is mandatory
if the len(input_columns) != len(output_columns). (default=None, all columns
will be propagated to the child node, the order of the columns will remain the
same).
pad_info (dict, optional): Whether to perform padding on selected columns. pad_info={"col1":([224,224],0)}
would pad column with name "col1" to a tensor of size [224,224] and fill the missing with 0.
......@@ -286,7 +298,7 @@ class Dataset:
>>> data = data.batch(100, True)
"""
return BatchDataset(self, batch_size, drop_remainder, num_parallel_workers, per_batch_map, input_columns,
pad_info)
output_columns, column_order, pad_info)
@check_sync_wait
def sync_wait(self, condition_name, num_batch=1, callback=None):
......@@ -367,7 +379,7 @@ class Dataset:
>>> # declare a function which returns a Dataset object
>>> def flat_map_func(x):
>>> data_dir = text.to_str(x[0])
>>> d = ds.ImageFolderDatasetV2(data_dir)
>>> d = ds.ImageFolderDataset(data_dir)
>>> return d
>>> # data is a Dataset object
>>> data = ds.TextFileDataset(DATA_FILE)
......@@ -394,7 +406,7 @@ class Dataset:
return dataset
@check_map
def map(self, input_columns=None, operations=None, output_columns=None, columns_order=None,
def map(self, operations=None, input_columns=None, output_columns=None, column_order=None,
num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None):
"""
Apply each operation in operations to this dataset.
......@@ -409,23 +421,23 @@ class Dataset:
The columns outputted by the very last operation will be assigned names specified by
output_columns.
Only the columns specified in columns_order will be propagated to the child node. These
columns will be in the same order as specified in columns_order.
Only the columns specified in column_order will be propagated to the child node. These
columns will be in the same order as specified in column_order.
Args:
operations (Union[list[TensorOp], list[functions]]): List of operations to be
applied on the dataset. Operations are applied in the order they appear in this list.
input_columns (list[str]): List of the names of the columns that will be passed to
the first operation as input. The size of this list must match the number of
input columns expected by the first operator. (default=None, the first
operation will be passed however many columns that is required, starting from
the first column).
operations (Union[list[TensorOp], list[functions]]): List of operations to be
applied on the dataset. Operations are applied in the order they appear in this list.
output_columns (list[str], optional): List of names assigned to the columns outputted by
the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. (default=None, output columns will have the same
name as the input columns, i.e., the columns will be replaced).
columns_order (list[str], optional): list of all the desired columns to propagate to the
column_order (list[str], optional): list of all the desired columns to propagate to the
child node. This list must be a subset of all the columns in the dataset after
all operations are applied. The order of the columns in each row propagated to the
child node follow the order they appear in this list. The parameter is mandatory
......@@ -446,7 +458,7 @@ class Dataset:
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.transforms.vision.c_transforms as c_transforms
>>> import mindspore.dataset.vision.c_transforms as c_transforms
>>>
>>> # data is an instance of Dataset which has 2 columns, "image" and "label".
>>> # ds_pyfunc is an instance of Dataset which has 3 columns, "col0", "col1", and "col2". Each column is
......@@ -468,33 +480,33 @@ class Dataset:
>>> input_columns = ["image"]
>>>
>>> # Applies decode_op on column "image". This column will be replaced by the outputed
>>> # column of decode_op. Since columns_order is not provided, both columns "image"
>>> # column of decode_op. Since column_order is not provided, both columns "image"
>>> # and "label" will be propagated to the child node in their original order.
>>> ds_decoded = data.map(input_columns, operations)
>>> ds_decoded = data.map(operations, input_columns)
>>>
>>> # Rename column "image" to "decoded_image"
>>> output_columns = ["decoded_image"]
>>> ds_decoded = data.map(input_columns, operations, output_columns)
>>> ds_decoded = data.map(operations, input_columns, output_columns)
>>>
>>> # Specify the order of the columns.
>>> columns_order ["label", "image"]
>>> ds_decoded = data.map(input_columns, operations, None, columns_order)
>>> column_order ["label", "image"]
>>> ds_decoded = data.map(operations, input_columns, None, column_order)
>>>
>>> # Rename column "image" to "decoded_image" and also specify the order of the columns.
>>> columns_order ["label", "decoded_image"]
>>> column_order ["label", "decoded_image"]
>>> output_columns = ["decoded_image"]
>>> ds_decoded = data.map(input_columns, operations, output_columns, columns_order)
>>> ds_decoded = data.map(operations, input_columns, output_columns, column_order)
>>>
>>> # Rename column "image" to "decoded_image" and keep only this column.
>>> columns_order ["decoded_image"]
>>> column_order ["decoded_image"]
>>> output_columns = ["decoded_image"]
>>> ds_decoded = data.map(input_columns, operations, output_columns, columns_order)
>>> ds_decoded = data.map(operations, input_columns, output_columns, column_order)
>>>
>>> # Simple example using pyfunc. Renaming columns and specifying column order
>>> # work in the same way as the previous examples.
>>> input_columns = ["col0"]
>>> operations = [(lambda x: x + 1)]
>>> ds_mapped = ds_pyfunc.map(input_columns, operations)
>>> ds_mapped = ds_pyfunc.map(operations, input_columns)
>>>
>>> # 2) Map example with more than one operation
>>>
......@@ -509,22 +521,22 @@ class Dataset:
>>> # outputted by decode_op is passed as input to random_jitter_op.
>>> # random_jitter_op will output one column. Column "image" will be replaced by
>>> # the column outputted by random_jitter_op (the very last operation). All other
>>> # columns are unchanged. Since columns_order is not specified, the order of the
>>> # columns are unchanged. Since column_order is not specified, the order of the
>>> # columns will remain the same.
>>> ds_mapped = data.map(input_columns, operations)
>>> ds_mapped = data.map(operations, input_columns)
>>>
>>> # Creates a dataset that is identical to ds_mapped, except the column "image"
>>> # that is outputted by random_jitter_op is renamed to "image_transformed".
>>> # Specifying column order works in the same way as examples in 1).
>>> output_columns = ["image_transformed"]
>>> ds_mapped_and_renamed = data.map(input_columns, operation, output_columns)
>>> ds_mapped_and_renamed = data.map(operation, input_columns, output_columns)
>>>
>>> # Multiple operations using pyfunc. Renaming columns and specifying column order
>>> # work in the same way as examples in 1).
>>> input_columns = ["col0"]
>>> operations = [(lambda x: x + x), (lambda x: x - 1)]
>>> output_columns = ["col0_mapped"]
>>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns)
>>> ds_mapped = ds_pyfunc.map(operations, input_columns, output_columns)
>>>
>>> # 3) Example where number of input columns is not equal to number of output columns
>>>
......@@ -540,20 +552,21 @@ class Dataset:
>>> (lambda x: (x % 2, x % 3, x % 5, x % 7))]
>>>
>>> # Note: because the number of input columns is not the same as the number of
>>> # output columns, the output_columns and columns_order parameter must be
>>> # output columns, the output_columns and column_order parameter must be
>>> # specified. Otherwise, this map call will also result in an error.
>>> input_columns = ["col2", "col0"]
>>> output_columns = ["mod2", "mod3", "mod5", "mod7"]
>>>
>>> # Propagate all columns to the child node in this order:
>>> columns_order = ["col0", "col2", "mod2", "mod3", "mod5", "mod7", "col1"]
>>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns, columns_order)
>>> column_order = ["col0", "col2", "mod2", "mod3", "mod5", "mod7", "col1"]
>>> ds_mapped = ds_pyfunc.map(operations, input_columns, output_columns, column_order)
>>>
>>> # Propagate some columns to the child node in this order:
>>> columns_order = ["mod7", "mod3", "col1"]
>>> ds_mapped = ds_pyfunc.map(input_columns, operations, output_columns, columns_order)
>>> column_order = ["mod7", "mod3", "col1"]
>>> ds_mapped = ds_pyfunc.map(operations, input_columns, output_columns, column_order)
"""
return MapDataset(self, input_columns, operations, output_columns, columns_order, num_parallel_workers,
return MapDataset(self, operations, input_columns, output_columns, column_order, num_parallel_workers,
python_multiprocessing, cache, callbacks)
@check_filter
......@@ -1012,7 +1025,7 @@ class Dataset:
def get_distribution(output_dataset):
dev_id = 0
if isinstance(output_dataset, (Cifar10Dataset, Cifar100Dataset, GeneratorDataset, ImageFolderDatasetV2,
if isinstance(output_dataset, (Cifar10Dataset, Cifar100Dataset, GeneratorDataset, ImageFolderDataset,
ManifestDataset, MnistDataset, VOCDataset, CocoDataset, CelebADataset,
MindDataset)):
sampler = output_dataset.sampler
......@@ -1412,7 +1425,7 @@ class MappableDataset(SourceDataset):
>>>
>>> dataset_dir = "/path/to/imagefolder_directory"
>>> # a SequentialSampler is created by default
>>> data = ds.ImageFolderDatasetV2(dataset_dir)
>>> data = ds.ImageFolderDataset(dataset_dir)
>>>
>>> # use a DistributedSampler instead of the SequentialSampler
>>> new_sampler = ds.DistributedSampler(10, 2)
......@@ -1501,7 +1514,7 @@ class MappableDataset(SourceDataset):
>>> dataset_dir = "/path/to/imagefolder_directory"
>>>
>>> # many datasets have shuffle on by default, set shuffle to False if split will be called!
>>> data = ds.ImageFolderDatasetV2(dataset_dir, shuffle=False)
>>> data = ds.ImageFolderDataset(dataset_dir, shuffle=False)
>>>
>>> # sets the seed, and tells split to use this seed when randomizing. This
>>> # is needed because we are sharding later
......@@ -1629,13 +1642,25 @@ class BatchDataset(DatasetOp):
last parameter of the callable should always be a BatchInfo object.
input_columns (list[str], optional): List of names of the input columns. The size of the list should
match with signature of per_batch_map callable.
output_columns (list[str], optional): List of names assigned to the columns outputted by
the last operation. This parameter is mandatory if len(input_columns) !=
len(output_columns). The size of this list must match the number of output
columns of the last operation. (default=None, output columns will have the same
name as the input columns, i.e., the columns will be replaced).
column_order (list[str], optional): list of all the desired columns to propagate to the
child node. This list must be a subset of all the columns in the dataset after
all operations are applied. The order of the columns in each row propagated to the
child node follow the order they appear in this list. The parameter is mandatory
if the len(input_columns) != len(output_columns). (default=None, all columns
will be propagated to the child node, the order of the columns will remain the
same).
pad_info (dict, optional): Whether to perform padding on selected columns. pad_info={"col1":([224,224],0)}
would pad column with name "col1" to a tensor of size [224,224] and fill the missing with 0.
"""
def __init__(self, input_dataset, batch_size, drop_remainder=False, num_parallel_workers=None,
per_batch_map=None, input_columns=None, pad_info=None):
per_batch_map=None, input_columns=None, output_columns=None, column_order=None, pad_info=None):
super().__init__(num_parallel_workers)
if BatchDataset._is_ancestor_of_repeat(input_dataset):
......@@ -1647,6 +1672,8 @@ class BatchDataset(DatasetOp):
self.drop_remainder = drop_remainder
self.per_batch_map = per_batch_map
self.input_columns = input_columns
self.output_columns = output_columns
self.column_order = column_order
self.pad_info = pad_info
self.children.append(input_dataset)
input_dataset.parent.append(self)
......@@ -1962,16 +1989,16 @@ class MapDataset(DatasetOp):
Args:
input_dataset (Dataset): Input Dataset to be mapped.
operations (TensorOp): A function mapping a nested structure of tensors
to another nested structure of tensor (default=None).
input_columns (list[str]): List of names of the input columns
(default=None, the operations will be applied on the first columns in the dataset).
The size of the list should match the number of inputs of the first operator.
operations (TensorOp): A function mapping a nested structure of tensors
to another nested structure of tensor (default=None).
output_columns (list[str], optional): list of names of the output columns.
The size of the list should match the number of outputs of the last operator
(default=None, output columns will be the input columns, i.e., the columns will
be replaced).
columns_order (list[str], optional): list of all the desired columns of the dataset (default=None).
column_order (list[str], optional): list of all the desired columns of the dataset (default=None).
The argument is mandatory if len(input_columns) != len(output_columns).
num_parallel_workers (int, optional): Number of workers to process the Dataset
in parallel (default=None).
......@@ -1982,29 +2009,29 @@ class MapDataset(DatasetOp):
callbacks: (DSCallback, list[DSCallback], optional): list of Dataset callbacks to be called (Default=None)
Raises:
ValueError: If len(input_columns) != len(output_columns) and columns_order is not specified.
ValueError: If len(input_columns) != len(output_columns) and column_order is not specified.
"""
def __init__(self, input_dataset, input_columns=None, operations=None, output_columns=None, columns_order=None,
def __init__(self, input_dataset, operations=None, input_columns=None, output_columns=None, column_order=None,
num_parallel_workers=None, python_multiprocessing=False, cache=None, callbacks=None):
super().__init__(num_parallel_workers)
self.children.append(input_dataset)
if input_columns is not None and not isinstance(input_columns, list):
input_columns = [input_columns]
self.input_columns = input_columns
if operations is not None and not isinstance(operations, list):
operations = [operations]
self.operations = operations
if input_columns is not None and not isinstance(input_columns, list):
input_columns = [input_columns]
self.input_columns = input_columns
if output_columns is not None and not isinstance(output_columns, list):
output_columns = [output_columns]
self.output_columns = output_columns
self.cache = cache
self.columns_order = columns_order
self.column_order = column_order
if self.input_columns and self.output_columns \
and len(self.input_columns) != len(self.output_columns) \
and self.columns_order is None:
raise ValueError("When (len(input_columns) != len(output_columns)), columns_order must be specified.")
and self.column_order is None:
raise ValueError("When (len(input_columns) != len(output_columns)), column_order must be specified.")
input_dataset.parent.append(self)
self._input_indexs = input_dataset.input_indexs
......@@ -2021,7 +2048,7 @@ class MapDataset(DatasetOp):
args["input_columns"] = self.input_columns
args["operations"] = self.operations
args["output_columns"] = self.output_columns
args["columns_order"] = self.columns_order
args["column_order"] = self.column_order
args["cache"] = self.cache.cache_client if self.cache is not None else None
if self.callbacks is not None:
......@@ -2048,7 +2075,7 @@ class MapDataset(DatasetOp):
new_op.children = copy.deepcopy(self.children, memodict)
new_op.input_columns = copy.deepcopy(self.input_columns, memodict)
new_op.output_columns = copy.deepcopy(self.output_columns, memodict)
new_op.columns_order = copy.deepcopy(self.columns_order, memodict)
new_op.column_order = copy.deepcopy(self.column_order, memodict)
new_op.num_parallel_workers = copy.deepcopy(self.num_parallel_workers, memodict)
new_op.parent = copy.deepcopy(self.parent, memodict)
new_op.ms_role = copy.deepcopy(self.ms_role, memodict)
......@@ -2646,7 +2673,7 @@ def _select_sampler(num_samples, input_sampler, shuffle, num_shards, shard_id, n
return samplers.SequentialSampler(num_samples=num_samples)
class ImageFolderDatasetV2(MappableDataset):
class ImageFolderDataset(MappableDataset):
"""
A source dataset that reads images from a tree of directories.
......@@ -2722,14 +2749,14 @@ class ImageFolderDatasetV2(MappableDataset):
>>> # path to imagefolder directory. This directory needs to contain sub-directories which contain the images
>>> dataset_dir = "/path/to/imagefolder_directory"
>>> # 1) read all samples (image files) in dataset_dir with 8 threads
>>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8)
>>> imagefolder_dataset = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8)
>>> # 2) read all samples (image files) from folder cat and folder dog with label 0 and 1
>>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir,class_indexing={"cat":0,"dog":1})
>>> imagefolder_dataset = ds.ImageFolderDataset(dataset_dir,class_indexing={"cat":0,"dog":1})
>>> # 3) read all samples (image files) in dataset_dir with extensions .JPEG and .png (case sensitive)
>>> imagefolder_dataset = ds.ImageFolderDatasetV2(dataset_dir, extensions=[".JPEG",".png"])
>>> imagefolder_dataset = ds.ImageFolderDataset(dataset_dir, extensions=[".JPEG",".png"])
"""
@check_imagefolderdatasetv2
@check_imagefolderdataset
def __init__(self, dataset_dir, num_samples=None, num_parallel_workers=None,
shuffle=None, sampler=None, extensions=None, class_indexing=None,
decode=False, num_shards=None, shard_id=None, cache=None):
......@@ -3168,6 +3195,7 @@ class SamplerFn:
"""
Multiprocessing or multithread generator function wrapper master process.
"""
def __init__(self, dataset, num_worker, multi_process):
self.workers = []
self.num_worker = num_worker
......
mindspore/dataset/engine/iterators.py
浏览文件 @ 3bac9d37
......@@ -150,7 +150,7 @@ class Iterator:
op_type = OpName.SKIP
elif isinstance(dataset, de.TakeDataset):
op_type = OpName.TAKE
elif isinstance(dataset, de.ImageFolderDatasetV2):
elif isinstance(dataset, de.ImageFolderDataset):
op_type = OpName.IMAGEFOLDER
elif isinstance(dataset, de.GeneratorDataset):
op_type = OpName.GENERATOR
......
mindspore/dataset/engine/samplers.py
浏览文件 @ 3bac9d37
......@@ -41,7 +41,7 @@ class Sampler:
>>> for i in range(self.dataset_size - 1, -1, -1):
>>> yield i
>>>
>>> ds = ds.ImageFolderDatasetV2(path, sampler=ReverseSampler())
>>> ds = ds.ImageFolderDataset(path, sampler=ReverseSampler())
"""
def __init__(self, num_samples=None):
......@@ -232,7 +232,7 @@ class DistributedSampler(BuiltinSampler):
>>>
>>> # creates a distributed sampler with 10 shards total. This shard is shard 5
>>> sampler = ds.DistributedSampler(10, 5)
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If num_shards is not positive.
......@@ -315,7 +315,7 @@ class PKSampler(BuiltinSampler):
>>>
>>> # creates a PKSampler that will get 3 samples from every class.
>>> sampler = ds.PKSampler(3)
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If num_val is not positive.
......@@ -387,7 +387,7 @@ class RandomSampler(BuiltinSampler):
>>>
>>> # creates a RandomSampler
>>> sampler = ds.RandomSampler()
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If replacement is not boolean.
......@@ -447,7 +447,7 @@ class SequentialSampler(BuiltinSampler):
>>>
>>> # creates a SequentialSampler
>>> sampler = ds.SequentialSampler()
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
"""
def __init__(self, start_index=None, num_samples=None):
......@@ -510,7 +510,7 @@ class SubsetRandomSampler(BuiltinSampler):
>>>
>>> # creates a SubsetRandomSampler, will sample from the provided indices
>>> sampler = ds.SubsetRandomSampler()
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
"""
def __init__(self, indices, num_samples=None):
......@@ -573,7 +573,7 @@ class WeightedRandomSampler(BuiltinSampler):
>>>
>>> # creates a WeightedRandomSampler that will sample 4 elements without replacement
>>> sampler = ds.WeightedRandomSampler(weights, 4)
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If num_samples is not positive.
......
mindspore/dataset/engine/serializer_deserializer.py
浏览文件 @ 3bac9d37
......@@ -21,9 +21,10 @@ import sys
from mindspore import log as logger
from . import datasets as de
from ..transforms.vision.utils import Inter, Border
from ..vision.utils import Inter, Border
from ..core import config
def serialize(dataset, json_filepath=None):
"""
Serialize dataset pipeline into a json file.
......@@ -44,7 +45,7 @@ def serialize(dataset, json_filepath=None):
>>> DATA_DIR = "../../data/testMnistData"
>>> data = ds.MnistDataset(DATA_DIR, 100)
>>> one_hot_encode = C.OneHot(10) # num_classes is input argument
>>> data = data.map(input_column_names="label", operation=one_hot_encode)
>>> data = data.map(operation=one_hot_encode, input_column_names="label")
>>> data = data.batch(batch_size=10, drop_remainder=True)
>>>
>>> ds.engine.serialize(data, json_filepath="mnist_dataset_pipeline.json") # serialize it to json file
......@@ -77,7 +78,7 @@ def deserialize(input_dict=None, json_filepath=None):
>>> DATA_DIR = "../../data/testMnistData"
>>> data = ds.MnistDataset(DATA_DIR, 100)
>>> one_hot_encode = C.OneHot(10) # num_classes is input argument
>>> data = data.map(input_column_names="label", operation=one_hot_encode)
>>> data = data.map(operation=one_hot_encode, input_column_names="label")
>>> data = data.batch(batch_size=10, drop_remainder=True)
>>>
>>> # Use case 1: to/from json file
......@@ -254,7 +255,7 @@ def create_node(node):
pyobj = None
# Find a matching Dataset class and call the constructor with the corresponding args.
# When a new Dataset class is introduced, another if clause and parsing code needs to be added.
if dataset_op == 'ImageFolderDatasetV2':
if dataset_op == 'ImageFolderDataset':
sampler = construct_sampler(node.get('sampler'))
pyobj = pyclass(node['dataset_dir'], node.get('num_samples'), node.get('num_parallel_workers'),
node.get('shuffle'), sampler, node.get('extensions'),
......@@ -336,8 +337,8 @@ def create_node(node):
elif dataset_op == 'MapDataset':
tensor_ops = construct_tensor_ops(node.get('operations'))
pyobj = de.Dataset().map(node.get('input_columns'), tensor_ops, node.get('output_columns'),
node.get('columns_order'), node.get('num_parallel_workers'))
pyobj = de.Dataset().map(tensor_ops, node.get('input_columns'), node.get('output_columns'),
node.get('column_order'), node.get('num_parallel_workers'))
elif dataset_op == 'ShuffleDataset':
pyobj = de.Dataset().shuffle(node.get('buffer_size'))
......
mindspore/dataset/engine/validators.py
浏览文件 @ 3bac9d37
......@@ -35,8 +35,8 @@ from . import cache_client
from .. import callback
def check_imagefolderdatasetv2(method):
"""A wrapper that wraps a parameter checker around the original Dataset(ImageFolderDatasetV2)."""
def check_imagefolderdataset(method):
"""A wrapper that wraps a parameter checker around the original Dataset(ImageFolderDataset)."""
@wraps(method)
def new_method(self, *args, **kwargs):
......@@ -474,8 +474,8 @@ def check_batch(method):
@wraps(method)
def new_method(self, *args, **kwargs):
[batch_size, drop_remainder, num_parallel_workers, per_batch_map,
input_columns, pad_info], param_dict = parse_user_args(method, *args, **kwargs)
[batch_size, drop_remainder, num_parallel_workers, per_batch_map, input_columns, output_columns,
column_order, pad_info], param_dict = parse_user_args(method, *args, **kwargs)
if not (isinstance(batch_size, int) or (callable(batch_size))):
raise TypeError("batch_size should either be an int or a callable.")
......@@ -510,6 +510,12 @@ def check_batch(method):
if len(input_columns) != (len(ins.signature(per_batch_map).parameters) - 1):
raise ValueError("the signature of per_batch_map should match with input columns")
if output_columns is not None:
raise ValueError("output_columns is currently not implemented.")
if column_order is not None:
raise ValueError("column_order is currently not implemented.")
return method(self, *args, **kwargs)
return new_method
......@@ -551,14 +557,14 @@ def check_map(method):
@wraps(method)
def new_method(self, *args, **kwargs):
[input_columns, _, output_columns, columns_order, num_parallel_workers, python_multiprocessing, cache,
[_, input_columns, output_columns, column_order, num_parallel_workers, python_multiprocessing, cache,
callbacks], _ = \
parse_user_args(method, *args, **kwargs)
nreq_param_columns = ['input_columns', 'output_columns', 'columns_order']
nreq_param_columns = ['input_columns', 'output_columns', 'column_order']
if columns_order is not None:
type_check(columns_order, (list,), "columns_order")
if column_order is not None:
type_check(column_order, (list,), "column_order")
if num_parallel_workers is not None:
check_num_parallel_workers(num_parallel_workers)
type_check(python_multiprocessing, (bool,), "python_multiprocessing")
......@@ -571,7 +577,7 @@ def check_map(method):
else:
type_check(callbacks, (callback.DSCallback,), "callbacks")
for param_name, param in zip(nreq_param_columns, [input_columns, output_columns, columns_order]):
for param_name, param in zip(nreq_param_columns, [input_columns, output_columns, column_order]):
if param is not None:
check_columns(param, param_name)
if callbacks is not None:
......
mindspore/dataset/text/transforms.py
浏览文件 @ 3bac9d37
......@@ -103,7 +103,6 @@ class SlidingWindow(cde.SlidingWindowOp):
super().__init__(width, axis)
class Ngram(cde.NgramOp):
"""
TensorOp to generate n-gram from a 1-D string Tensor.
......@@ -161,8 +160,9 @@ class JiebaTokenizer(cde.JiebaTokenizerOp):
>>> # If with_offsets=False, then output three columns {["token", dtype=str], ["offsets_start", dtype=uint32],
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> data = data.map(operations=tokenizer_op, input_columns=["text"],
>>> output_columns=["token", "offsets_start", "offsets_limit"],
>>> column_order=["token", "offsets_start", "offsets_limit"])
"""
@check_jieba_init
......@@ -281,7 +281,7 @@ class UnicodeCharTokenizer(cde.UnicodeCharTokenizerOp):
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.UnicodeCharTokenizer(True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_with_offsets
......@@ -312,7 +312,7 @@ class WordpieceTokenizer(cde.WordpieceTokenizerOp):
>>> tokenizer_op = text.WordpieceTokenizer(vocab=vocab, unknown_token=['UNK'],
>>> max_bytes_per_token=100, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_wordpiece_tokenizer
......@@ -377,7 +377,7 @@ if platform.system().lower() != 'windows':
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.WhitespaceTokenizer(True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_with_offsets
......@@ -403,7 +403,7 @@ if platform.system().lower() != 'windows':
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.UnicodeScriptTokenizerOp(keep_whitespace=True, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_unicode_script_tokenizer
......@@ -496,7 +496,7 @@ if platform.system().lower() != 'windows':
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.RegexTokenizer(delim_pattern, keep_delim_pattern, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_regex_tokenizer
......@@ -539,7 +539,7 @@ if platform.system().lower() != 'windows':
>>> preserve_unused_token=True,
>>> with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_basic_tokenizer
......@@ -592,7 +592,7 @@ if platform.system().lower() != 'windows':
>>> normalization_form=NormalizeForm.NONE, preserve_unused_token=True,
>>> with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_bert_tokenizer
......
mindspore/dataset/transforms/__init__.py
浏览文件 @ 3bac9d37
......@@ -16,6 +16,6 @@ This module is to support common augmentations. C_transforms is a high performan
image augmentation module which is developed with C++ OpenCV. Py_transforms
provide more kinds of image augmentations which is developed with Python PIL.
"""
from . import vision
from .. import vision
from . import c_transforms
from . import py_transforms
mindspore/dataset/transforms/c_transforms.py
浏览文件 @ 3bac9d37
......@@ -229,8 +229,8 @@ class Duplicate(cde.DuplicateOp):
>>> # +---------+
>>> # | [1,2,3] |
>>> # +---------+
>>> data = data.map(input_columns=["x"], operations=Duplicate(),
>>> output_columns=["x", "y"], columns_order=["x", "y"])
>>> data = data.map(operations=Duplicate(), input_columns=["x"],
>>> output_columns=["x", "y"], column_order=["x", "y"])
>>> # Data after
>>> # | x | y |
>>> # +---------+---------+
......
mindspore/dataset/transforms/py_transforms.py
浏览文件 @ 3bac9d37
......@@ -17,9 +17,8 @@
This module py_transforms is implemented basing on Python. It provides common
operations including OneHotOp.
"""
from .validators import check_one_hot_op
from .vision import py_transforms_util as util
from .validators import check_one_hot_op, check_compose_list
from . import py_transforms_util as util
class OneHotOp:
......@@ -48,3 +47,48 @@ class OneHotOp:
label (numpy.ndarray), label after being Smoothed.
"""
return util.one_hot_encoding(label, self.num_classes, self.smoothing_rate)
class Compose:
"""
Compose a list of transforms.
.. Note::
Compose takes a list of transformations either provided in py_transforms or from user-defined implementation;
each can be an initialized transformation class or a lambda function, as long as the output from the last
transformation is a single tensor of type numpy.ndarray. See below for an example of how to use Compose
with py_transforms classes and check out FiveCrop or TenCrop for the use of them in conjunction with lambda
functions.
Args:
transforms (list): List of transformations to be applied.
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>> dataset_dir = "path/to/imagefolder_directory"
>>> # create a dataset that reads all files in dataset_dir with 8 threads
>>> dataset = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8)
>>> # create a list of transformations to be applied to the image data
>>> transform = Compose([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)),
>>> py_transforms.RandomErasing()])
>>> # apply the transform to the dataset through dataset.map()
>>> dataset = dataset.map(operations=transform, input_columns="image")
"""
@check_compose_list
def __init__(self, transforms):
self.transforms = transforms
def __call__(self, img):
"""
Call method.
Returns:
lambda function, Lambda function that takes in an img to apply transformations on.
"""
return util.compose(img, self.transforms)
mindspore/dataset/transforms/py_transforms_util.py 0 → 100644
浏览文件 @ 3bac9d37
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Built-in py_transforms_utils functions.
"""
import numpy as np
from ..core.py_util_helpers import is_numpy
def compose(img, transforms):
"""
Compose a list of transforms and apply on the image.
Args:
img (numpy.ndarray): An image in Numpy ndarray.
transforms (list): A list of transform Class objects to be composed.
Returns:
img (numpy.ndarray), An augmented image in Numpy ndarray.
"""
if is_numpy(img):
for transform in transforms:
img = transform(img)
if is_numpy(img):
return img
raise TypeError('img should be Numpy ndarray. Got {}. Append ToTensor() to transforms'.format(type(img)))
raise TypeError('img should be Numpy ndarray. Got {}.'.format(type(img)))
def one_hot_encoding(label, num_classes, epsilon):
"""
Apply label smoothing transformation to the input label, and make label be more smoothing and continuous.
Args:
label (numpy.ndarray): label to be applied label smoothing.
num_classes (int): Num class of object in dataset, value should over 0.
epsilon (float): The adjustable Hyper parameter. Default is 0.0.
Returns:
img (numpy.ndarray), label after being one hot encoded and done label smoothed.
"""
if label > num_classes:
raise ValueError('the num_classes is smaller than the category number.')
num_elements = label.size
one_hot_label = np.zeros((num_elements, num_classes), dtype=int)
if isinstance(label, list) is False:
label = [label]
for index in range(num_elements):
one_hot_label[index, label[index]] = 1
return (1 - epsilon) * one_hot_label + epsilon / num_classes
mindspore/dataset/transforms/validators.py
浏览文件 @ 3bac9d37
......@@ -200,3 +200,19 @@ def check_random_transform_ops(method):
return method(self, *args, **kwargs)
return new_method
def check_compose_list(method):
"""Wrapper method to check the transform list of Compose."""
@wraps(method)
def new_method(self, *args, **kwargs):
[transforms], _ = parse_user_args(method, *args, **kwargs)
type_check(transforms, (list,), transforms)
if not transforms:
raise ValueError("transforms list is empty.")
return method(self, *args, **kwargs)
return new_method
mindspore/dataset/transforms/vision/c_transforms.py → mindspore/dataset/vision/c_transforms.py
浏览文件 @ 3bac9d37
......@@ -25,11 +25,12 @@ to improve their training models.
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.transforms.c_transforms as c_transforms
>>> import mindspore.dataset.transforms.vision.c_transforms as c_vision
>>> import mindspore.dataset.vision.c_transforms as c_vision
>>> from mindspore.dataset.transforms.vision.utils import Border, ImageBatchFormat, Inter
>>> dataset_dir = "path/to/imagefolder_directory"
>>> # create a dataset that reads all files in dataset_dir with 8 threads
>>> data1 = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8)
>>> data1 = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8)
>>> # create a list of transformations to be applied to the image data
>>> transforms_list = [c_vision.Decode(),
>>> c_vision.Resize((256, 256)),
......@@ -1095,7 +1096,7 @@ class UniformAugment(cde.UniformAugOp):
num_ops (int, optional): Number of operations to be selected and applied (default=2).
Examples:
>>> import mindspore.dataset.transforms.vision.py_transforms as py_vision
>>> import mindspore.dataset.vision.py_transforms as py_vision
>>> transforms_list = [c_vision.RandomHorizontalFlip(),
>>> c_vision.RandomVerticalFlip(),
>>> c_vision.RandomColorAdjust(),
......
mindspore/dataset/transforms/vision/py_transforms.py → mindspore/dataset/vision/py_transforms.py
浏览文件 @ 3bac9d37
......@@ -33,7 +33,7 @@ from .validators import check_prob, check_crop, check_resize_interpolation, chec
check_normalize_py, check_random_crop, check_random_color_adjust, check_random_rotation, \
check_transforms_list, check_random_apply, check_ten_crop, check_num_channels, check_pad, \
check_random_perspective, check_random_erasing, check_cutout, check_linear_transform, check_random_affine, \
check_mix_up, check_positive_degrees, check_uniform_augment_py, check_compose_list, check_auto_contrast
check_mix_up, check_positive_degrees, check_uniform_augment_py, check_auto_contrast
from .utils import Inter, Border
DE_PY_INTER_MODE = {Inter.NEAREST: Image.NEAREST,
......@@ -46,50 +46,6 @@ DE_PY_BORDER_TYPE = {Border.CONSTANT: 'constant',
Border.SYMMETRIC: 'symmetric'}
class ComposeOp:
"""
Compose a list of transforms.
.. Note::
ComposeOp takes a list of transformations either provided in py_transforms or from user-defined implementation;
each can be an initialized transformation class or a lambda function, as long as the output from the last
transformation is a single tensor of type numpy.ndarray. See below for an example of how to use ComposeOp
with py_transforms classes and check out FiveCrop or TenCrop for the use of them in conjunction with lambda
functions.
Args:
transforms (list): List of transformations to be applied.
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.transforms.vision.py_transforms as py_transforms
>>> dataset_dir = "path/to/imagefolder_directory"
>>> # create a dataset that reads all files in dataset_dir with 8 threads
>>> dataset = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8)
>>> # create a list of transformations to be applied to the image data
>>> transform = py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)),
>>> py_transforms.RandomErasing()])
>>> # apply the transform to the dataset through dataset.map()
>>> dataset = dataset.map(input_columns="image", operations=transform())
"""
@check_compose_list
def __init__(self, transforms):
self.transforms = transforms
def __call__(self):
"""
Call method.
Returns:
lambda function, Lambda function that takes in an image to apply transformations on.
"""
return lambda img: util.compose(img, self.transforms)
class ToTensor:
"""
Convert the input NumPy image array or PIL image of shape (H,W,C) to a NumPy ndarray of shape (C,H,W).
......@@ -103,9 +59,11 @@ class ToTensor:
output_type (Numpy datatype, optional): The datatype of the NumPy output (default=np.float32).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(), py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
"""
def __init__(self, output_type=np.float32):
......@@ -132,11 +90,13 @@ class ToType:
output_type (Numpy datatype): The datatype of the NumPy output, e.g. numpy.float32.
Examples:
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>> import numpy as np
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.ToType(np.float32)])
>>>
>>> Compose([py_transforms.Decode(), py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.ToType(np.float32)])
"""
def __init__(self, output_type):
......@@ -179,9 +139,11 @@ class ToPIL:
Examples:
>>> # data is already decoded, but not in PIL image format
>>> py_transforms.ComposeOp([py_transforms.ToPIL(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.ToPIL(), py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
"""
def __call__(self, img):
......@@ -202,7 +164,10 @@ class Decode:
Decode the input image to PIL image format in RGB mode.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
"""
......@@ -233,10 +198,13 @@ class Normalize:
The standard deviation values must be in range (0.0, 1.0].
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262))])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262))])
"""
@check_normalize_py
......@@ -291,9 +259,12 @@ class RandomCrop:
value of edge.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomCrop(224),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomCrop(224),
>>> py_transforms.ToTensor()])
"""
@check_random_crop
......@@ -330,9 +301,12 @@ class RandomHorizontalFlip:
prob (float, optional): Probability of the image being flipped (default=0.5).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor()])
"""
@check_prob
......@@ -360,9 +334,12 @@ class RandomVerticalFlip:
prob (float, optional): Probability of the image being flipped (default=0.5).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomVerticalFlip(0.5),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomVerticalFlip(0.5),
>>> py_transforms.ToTensor()])
"""
@check_prob
......@@ -401,9 +378,12 @@ class Resize:
- Inter.BICUBIC, means interpolation method is bicubic interpolation.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.Resize(256),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.Resize(256),
>>> py_transforms.ToTensor()])
"""
@check_resize_interpolation
......@@ -448,9 +428,12 @@ class RandomResizedCrop:
crop area (default=10). If exceeded, fall back to use center crop instead.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomResizedCrop(224),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomResizedCrop(224),
>>> py_transforms.ToTensor()])
"""
@check_random_resize_crop
......@@ -486,9 +469,12 @@ class CenterCrop:
If size is a sequence of length 2, it should be (height, width).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.CenterCrop(64),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.CenterCrop(64),
>>> py_transforms.ToTensor()])
"""
@check_crop
......@@ -527,9 +513,12 @@ class RandomColorAdjust:
If it is a sequence, it should be [min, max] where -0.5 <= min <= max <= 0.5.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomColorAdjust(0.4, 0.4, 0.4, 0.1),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomColorAdjust(0.4, 0.4, 0.4, 0.1),
>>> py_transforms.ToTensor()])
"""
@check_random_color_adjust
......@@ -585,9 +574,12 @@ class RandomRotation:
If it is an int, it is used for all RGB channels. Default is 0.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomRotation(30),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomRotation(30),
>>> py_transforms.ToTensor()])
"""
@check_random_rotation
......@@ -619,10 +611,12 @@ class RandomOrder:
transforms (list): List of the transformations to be applied.
Examples:
>>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)]
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomOrder(transforms_list),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomOrder(transforms_list),
>>> py_transforms.ToTensor()])
"""
@check_transforms_list
......@@ -651,10 +645,12 @@ class RandomApply:
prob (float, optional): The probability to apply the transformation list (default=0.5).
Examples:
>>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)]
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomApply(transforms_list, prob=0.6),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomApply(transforms_list, prob=0.6),
>>> py_transforms.ToTensor()])
"""
@check_random_apply
......@@ -683,10 +679,12 @@ class RandomChoice:
transforms (list): List of transformations to be chosen from to apply.
Examples:
>>> transforms_list = [py_transforms.CenterCrop(64), py_transforms.RandomRotation(30)]
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomChoice(transforms_list),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomChoice(transforms_list),
>>> py_transforms.ToTensor()])
"""
@check_transforms_list
......@@ -716,10 +714,13 @@ class FiveCrop:
If size is a sequence of length 2, it should be (height, width).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.FiveCrop(size),
>>> # 4D stack of 5 images
>>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.FiveCrop(size),
>>> # 4D stack of 5 images
>>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])])
"""
@check_crop
......@@ -752,10 +753,13 @@ class TenCrop:
if set to True (default=False).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.TenCrop(size),
>>> # 4D stack of 10 images
>>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.TenCrop(size),
>>> # 4D stack of 10 images
>>> lambda images: numpy.stack([py_transforms.ToTensor()(image) for image in images])])
"""
@check_ten_crop
......@@ -789,9 +793,12 @@ class Grayscale:
Default is 1. If set to 3, the returned image has 3 identical RGB channels.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.Grayscale(3),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.Grayscale(3),
>>> py_transforms.ToTensor()])
"""
@check_num_channels
......@@ -819,9 +826,12 @@ class RandomGrayscale:
prob (float, optional): Probability of the image being converted to grayscale (default=0.1).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomGrayscale(0.3),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomGrayscale(0.3),
>>> py_transforms.ToTensor()])
"""
@check_prob
......@@ -878,10 +888,13 @@ class Pad:
value of edge.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> # adds 10 pixels (default black) to each side of the border of the image
>>> py_transforms.Pad(padding=10),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> # adds 10 pixels (default black) to each side of the border of the image
>>> py_transforms.Pad(padding=10),
>>> py_transforms.ToTensor()])
"""
@check_pad
......@@ -922,9 +935,12 @@ class RandomPerspective:
- Inter.BICUBIC, means interpolation method is bicubic interpolation.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomPerspective(prob=0.1),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomPerspective(prob=0.1),
>>> py_transforms.ToTensor()])
"""
@check_random_perspective
......@@ -972,9 +988,12 @@ class RandomErasing:
erase_area (default=10). If exceeded, return the original image.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.ToTensor(),
>>> py_transforms.RandomErasing(value='random')])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.ToTensor(),
>>> py_transforms.RandomErasing(value='random')])
"""
@check_random_erasing
......@@ -1016,9 +1035,12 @@ class Cutout:
num_patches (int, optional): Number of patches to be cut out of an image (default=1).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.ToTensor(),
>>> py_transforms.Cutout(80)])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.ToTensor(),
>>> py_transforms.Cutout(80)])
"""
@check_cutout
......@@ -1043,7 +1065,8 @@ class Cutout:
bounded = False
for _ in range(self.num_patches):
i, j, erase_h, erase_w, erase_value = util.get_erase_params(np_img, (scale, scale), (1, 1), 0, bounded, 1)
i, j, erase_h, erase_w, erase_value = util.get_erase_params(np_img, (scale, scale), (1, 1), 0, bounded,
1)
np_img = util.erase(np_img, i, j, erase_h, erase_w, erase_value)
return np_img
......@@ -1061,10 +1084,13 @@ class LinearTransformation:
mean_vector (numpy.ndarray): a NumPy ndarray of shape (D,) where D = C x H x W.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.Resize(256),
>>> py_transforms.ToTensor(),
>>> py_transforms.LinearTransformation(transformation_matrix, mean_vector)])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.Resize(256),
>>> py_transforms.ToTensor(),
>>> py_transforms.LinearTransformation(transformation_matrix, mean_vector)])
"""
@check_linear_transform
......@@ -1133,9 +1159,12 @@ class RandomAffine:
TypeError: If fill_value is not a single integer or a 3-tuple.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)),
>>> py_transforms.ToTensor()])
"""
@check_random_affine
......@@ -1278,9 +1307,12 @@ class RandomColor:
It should be in (min, max) format (default=(0.1,1.9)).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomColor((0.5,1.5)),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomColor((0.5,1.5)),
>>> py_transforms.ToTensor()])
"""
@check_positive_degrees
......@@ -1310,9 +1342,12 @@ class RandomSharpness:
It should be in (min, max) format (default=(0.1,1.9)).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.RandomSharpness((0.5,1.5)),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.RandomSharpness((0.5,1.5)),
>>> py_transforms.ToTensor()])
"""
......@@ -1343,9 +1378,12 @@ class AutoContrast:
ignore (Union[int, sequence], optional): Pixel values to ignore (default=None).
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.AutoContrast(),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.AutoContrast(),
>>> py_transforms.ToTensor()])
"""
......@@ -1373,9 +1411,12 @@ class Invert:
Invert colors of input PIL image.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.Invert(),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.Invert(),
>>> py_transforms.ToTensor()])
"""
......@@ -1398,9 +1439,12 @@ class Equalize:
Equalize the histogram of input PIL image.
Examples:
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.Equalize(),
>>> py_transforms.ToTensor()])
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> Compose([py_transforms.Decode(),
>>> py_transforms.Equalize(),
>>> py_transforms.ToTensor()])
"""
......@@ -1430,13 +1474,16 @@ class UniformAugment:
num_ops (int, optional): number of transforms to sequentially apply (default=2).
Examples:
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>>
>>> transforms_list = [py_transforms.CenterCrop(64),
>>> py_transforms.RandomColor(),
>>> py_transforms.RandomSharpness(),
>>> py_transforms.RandomRotation(30)]
>>> py_transforms.ComposeOp([py_transforms.Decode(),
>>> py_transforms.UniformAugment(transforms_list),
>>> py_transforms.ToTensor()])
>>> Compose([py_transforms.Decode(),
>>> py_transforms.UniformAugment(transforms_list),
>>> py_transforms.ToTensor()])
"""
@check_uniform_augment_py
......
mindspore/dataset/transforms/vision/py_transforms_util.py → mindspore/dataset/vision/py_transforms_util.py
浏览文件 @ 3bac9d37
......@@ -24,6 +24,7 @@ import numpy as np
from PIL import Image, ImageOps, ImageEnhance, __version__
from .utils import Inter
from ..core.py_util_helpers import is_numpy
augment_error_message = 'img should be PIL image. Got {}. Use Decode() for encoded data or ToPIL() for decoded data.'
......@@ -41,39 +42,6 @@ def is_pil(img):
return isinstance(img, Image.Image)
def is_numpy(img):
"""
Check if the input image is NumPy format.
Args:
img: Image to be checked.
Returns:
Bool, True if input is NumPy image.
"""
return isinstance(img, np.ndarray)
def compose(img, transforms):
"""
Compose a list of transforms and apply on the image.
Args:
img (numpy.ndarray): An image in NumPy ndarray.
transforms (list): A list of transform Class objects to be composed.
Returns:
img (numpy.ndarray), An augmented image in NumPy ndarray.
"""
if is_numpy(img):
for transform in transforms:
img = transform(img)
if is_numpy(img):
return img
raise TypeError('img should be NumPy ndarray. Got {}. Append ToTensor() to transforms'.format(type(img)))
raise TypeError('img should be NumPy ndarray. Got {}.'.format(type(img)))
def normalize(img, mean, std):
"""
Normalize the image between [0, 1] with respect to mean and standard deviation.
......@@ -1221,32 +1189,6 @@ def random_affine(img, angle, translations, scale, shear, resample, fill_value=0
return img.transform(output_size, Image.AFFINE, matrix, resample, **kwargs)
def one_hot_encoding(label, num_classes, epsilon):
"""
Apply label smoothing transformation to the input label, and make label be more smoothing and continuous.
Args:
label (numpy.ndarray): label to be applied label smoothing.
num_classes (int): Num class of object in dataset, value should over 0.
epsilon (float): The adjustable Hyper parameter. Default is 0.0.
Returns:
img (numpy.ndarray), label after being one hot encoded and done label smoothed.
"""
if label > num_classes:
raise ValueError('the num_classes is smaller than the category number.')
num_elements = label.size
one_hot_label = np.zeros((num_elements, num_classes), dtype=int)
if isinstance(label, list) is False:
label = [label]
for index in range(num_elements):
one_hot_label[index, label[index]] = 1
return (1 - epsilon) * one_hot_label + epsilon / num_classes
def mix_up_single(batch_size, img, label, alpha=0.2):
"""
Apply mix up transformation to image and label in single batch internal, One hot encoding should done before this.
......
mindspore/dataset/transforms/vision/validators.py → mindspore/dataset/vision/validators.py
浏览文件 @ 3bac9d37
......@@ -19,10 +19,10 @@ from functools import wraps
import numpy as np
from mindspore._c_dataengine import TensorOp
from .utils import Inter, Border, ImageBatchFormat
from ...core.validator_helpers import check_value, check_uint8, FLOAT_MAX_INTEGER, check_pos_float32, \
from mindspore.dataset.core.validator_helpers import check_value, check_uint8, FLOAT_MAX_INTEGER, check_pos_float32, \
check_2tuple, check_range, check_positive, INT32_MAX, parse_user_args, type_check, type_check_list, \
check_tensor_op, UINT8_MAX, check_value_normalize_std
from .utils import Inter, Border, ImageBatchFormat
def check_crop_size(size):
......@@ -678,21 +678,6 @@ def check_positive_degrees(method):
return new_method
def check_compose_list(method):
"""Wrapper method to check the transform list of ComposeOp."""
@wraps(method)
def new_method(self, *args, **kwargs):
[transforms], _ = parse_user_args(method, *args, **kwargs)
type_check(transforms, (list,), transforms)
if not transforms:
raise ValueError("transforms list is empty.")
return method(self, *args, **kwargs)
return new_method
def check_random_select_subpolicy_op(method):
"""Wrapper method to check the parameters of RandomSelectSubpolicyOp."""
......
mindspore/train/callback/_summary_collector.py
浏览文件 @ 3bac9d37
......@@ -727,7 +727,7 @@ class SummaryCollector(Callback):
Get dataset path of MindDataset object.
Args:
output_dataset (Union[Dataset, ImageFolderDatasetV2, MnistDataset, Cifar10Dataset, Cifar100Dataset,
output_dataset (Union[Dataset, ImageFolderDataset, MnistDataset, Cifar10Dataset, Cifar100Dataset,
VOCDataset, CelebADataset, MindDataset, ManifestDataset, TFRecordDataset, TextFileDataset]):
Refer to mindspore.dataset.Dataset.
......@@ -738,7 +738,7 @@ class SummaryCollector(Callback):
IndexError: it means get dataset path failed.
"""
dataset_package = import_module('mindspore.dataset')
dataset_dir_set = (dataset_package.ImageFolderDatasetV2, dataset_package.MnistDataset,
dataset_dir_set = (dataset_package.ImageFolderDataset, dataset_package.MnistDataset,
dataset_package.Cifar10Dataset, dataset_package.Cifar100Dataset,
dataset_package.VOCDataset, dataset_package.CelebADataset)
dataset_file_set = (dataset_package.MindDataset, dataset_package.ManifestDataset)
......
model_zoo/official/cv/faster_rcnn/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -449,7 +449,7 @@ def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, devi
if is_training:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "box", "label", "valid_num"],
columns_order=["image", "image_shape", "box", "label", "valid_num"],
column_order=["image", "image_shape", "box", "label", "valid_num"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
flip = (np.random.rand() < config.flip_ratio)
......@@ -467,7 +467,7 @@ def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, devi
else:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "box", "label", "valid_num"],
columns_order=["image", "image_shape", "box", "label", "valid_num"],
column_order=["image", "image_shape", "box", "label", "valid_num"],
operations=compose_map_func,
num_parallel_workers=num_parallel_workers)
......
model_zoo/official/cv/inceptionv3/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -37,10 +37,10 @@ def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
dataset
"""
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
# define map operations
if do_train:
trans = [
......
model_zoo/official/cv/maskrcnn/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -505,7 +505,7 @@ def create_maskrcnn_dataset(mindrecord_file, batch_size=2, device_num=1, rank_id
if is_training:
ds = ds.map(input_columns=["image", "annotation", "mask", "mask_shape"],
output_columns=["image", "image_shape", "box", "label", "valid_num", "mask"],
columns_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
column_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
operations=compose_map_func,
python_multiprocessing=False,
num_parallel_workers=num_parallel_workers)
......@@ -514,7 +514,7 @@ def create_maskrcnn_dataset(mindrecord_file, batch_size=2, device_num=1, rank_id
else:
ds = ds.map(input_columns=["image", "annotation", "mask", "mask_shape"],
output_columns=["image", "image_shape", "box", "label", "valid_num", "mask"],
columns_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
column_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
operations=compose_map_func,
num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
......
model_zoo/official/cv/mobilenetv2/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -26,6 +26,7 @@ import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
def create_dataset(dataset_path, do_train, config, repeat_num=1):
"""
create a train or eval dataset
......@@ -44,20 +45,19 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1):
rank_size = int(os.getenv("RANK_SIZE", '1'))
rank_id = int(os.getenv("RANK_ID", '0'))
if rank_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
elif config.platform == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
elif config.platform == "CPU":
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
resize_height = config.image_height
resize_width = config.image_width
......@@ -71,7 +71,8 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1):
resize_op = C.Resize((256, 256))
center_crop = C.CenterCrop(resize_width)
rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255])
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
change_swap_op = C.HWC2CHW()
if do_train:
......@@ -95,6 +96,7 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1):
return ds
def extract_features(net, dataset_path, config):
features_folder = dataset_path + '_features'
if not os.path.exists(features_folder):
......@@ -110,13 +112,13 @@ def extract_features(net, dataset_path, config):
for data in pbar:
features_path = os.path.join(features_folder, f"feature_{i}.npy")
label_path = os.path.join(features_folder, f"label_{i}.npy")
if not(os.path.exists(features_path) and os.path.exists(label_path)):
if not (os.path.exists(features_path) and os.path.exists(label_path)):
image = data["image"]
label = data["label"]
features = model.predict(Tensor(image))
np.save(features_path, features.asnumpy())
np.save(label_path, label)
pbar.set_description("Process dataset batch: %d"%(i+1))
pbar.set_description("Process dataset batch: %d" % (i + 1))
i += 1
return step_size
model_zoo/official/cv/mobilenetv2_quant/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,8 @@ import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.transforms.vision.py_transforms as P
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as P
def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32):
......@@ -44,7 +45,7 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1,
if config.data_load_mode == "mindrecord":
load_func = partial(de.MindDataset, dataset_path, columns_list)
else:
load_func = partial(de.ImageFolderDatasetV2, dataset_path)
load_func = partial(de.ImageFolderDataset, dataset_path)
if do_train:
if rank_size == 1:
ds = load_func(num_parallel_workers=8, shuffle=True)
......@@ -56,10 +57,10 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1,
elif device_target == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
raise ValueError("Unsupported device_target.")
......@@ -118,12 +119,12 @@ def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num=
rank_id = int(os.getenv("RANK_ID"))
if do_train:
if rank_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=False)
else:
raise ValueError("Unsupported device target.")
......@@ -149,9 +150,9 @@ def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num=
else:
trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op]
compose = P.ComposeOp(trans)
compose = mindspore.dataset.transforms.py_transforms.Compose(trans)
ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True)
ds = ds.map(input_columns="image", operations=compose, num_parallel_workers=8, python_multiprocessing=True)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
......
model_zoo/official/cv/mobilenetv3/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -37,10 +37,10 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1,
if device_target == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
raise ValueError("Unsupported device_target.")
......
model_zoo/official/cv/nasnet/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -37,24 +37,24 @@ def create_dataset(dataset_path, config, do_train, repeat_num=1):
rank = config.rank
group_size = config.group_size
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=config.work_nums, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=config.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(config.image_size),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode
#C.RandomColorAdjust(brightness=0.4, contrast=0.5, saturation=0.5, hue=0.2)
]
C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode
# C.RandomColorAdjust(brightness=0.4, contrast=0.5, saturation=0.5, hue=0.2)
]
else:
trans = [
C.Decode(),
C.Resize(int(config.image_size/0.875)),
C.Resize(int(config.image_size / 0.875)),
C.CenterCrop(config.image_size)
]
]
trans += [
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
......
model_zoo/official/cv/resnet/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -98,10 +98,10 @@ def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target=
device_num = get_group_size()
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
......@@ -153,10 +153,10 @@ def create_dataset3(dataset_path, do_train, repeat_num=1, batch_size=32, target=
device_num, rank_id = _get_rank_info()
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.475 * 255, 0.451 * 255, 0.392 * 255]
std = [0.275 * 255, 0.267 * 255, 0.278 * 255]
......@@ -207,10 +207,10 @@ def create_dataset4(dataset_path, do_train, repeat_num=1, batch_size=32, target=
if target == "Ascend":
device_num, rank_id = _get_rank_info()
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=12, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=12, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [123.68, 116.78, 103.94]
std = [1.0, 1.0, 1.0]
......
model_zoo/official/cv/resnet50_quant/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,8 @@ import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.transforms.vision.py_transforms as P
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as P
from mindspore.communication.management import init, get_rank, get_group_size
from src.config import config_quant
......@@ -54,7 +55,7 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="
if config.data_load_mode == "mindrecord":
load_func = partial(de.MindDataset, dataset_path, columns_list)
else:
load_func = partial(de.ImageFolderDatasetV2, dataset_path)
load_func = partial(de.ImageFolderDataset, dataset_path)
if device_num == 1:
ds = load_func(num_parallel_workers=8, shuffle=True)
else:
......@@ -120,12 +121,12 @@ def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, targe
if do_train:
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=False)
image_size = 224
......@@ -145,8 +146,8 @@ def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, targe
else:
trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op]
compose = P.ComposeOp(trans)
ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True)
compose = mindspore.dataset.transforms.py_transforms.Compose(trans)
ds = ds.map(input_columns="image", operations=compose, num_parallel_workers=8, python_multiprocessing=True)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
......
model_zoo/official/cv/resnet_thor/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -47,10 +47,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="
num_parallels = 4
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=num_parallels, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=num_parallels, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=num_parallels, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=num_parallels, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
......@@ -86,6 +86,7 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="
return ds
def _get_rank_info():
"""
get rank size and rank id
......
model_zoo/official/cv/resnext50/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -134,9 +134,9 @@ def classification_dataset(data_dir, image_size, per_batch_size, max_epoch, rank
transform_label = target_transform
if input_mode == 'folder':
de_dataset = de.ImageFolderDatasetV2(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
de_dataset = de.ImageFolderDataset(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
else:
dataset = TxtDataset(root, data_dir)
sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle)
......
model_zoo/official/cv/shufflenetv2/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -30,6 +30,7 @@ class toBGR():
img = np.ascontiguousarray(img)
return img
def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
"""
create a train or eval dataset
......@@ -45,23 +46,23 @@ def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
dataset
"""
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(224),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
]
]
else:
trans = [
C.Decode(),
C.Resize(256),
C.CenterCrop(224)
]
]
trans += [
toBGR(),
C.Rescale(1.0 / 255.0, 0.0),
......
model_zoo/official/cv/ssd/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -403,7 +403,7 @@ def create_ssd_dataset(mindrecord_file, batch_size=32, repeat_num=10, device_num
output_columns = ["img_id", "image", "image_shape"]
trans = [normalize_op, change_swap_op]
ds = ds.map(input_columns=["img_id", "image", "annotation"],
output_columns=output_columns, columns_order=output_columns,
output_columns=output_columns, column_order=output_columns,
operations=compose_map_func, python_multiprocessing=is_training,
num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=["image"], operations=trans, python_multiprocessing=is_training,
......
model_zoo/official/cv/vgg16/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -149,9 +149,9 @@ def classification_dataset(data_dir, image_size, per_batch_size, rank=0, group_s
transform_label = target_transform
if input_mode == 'folder':
de_dataset = de.ImageFolderDatasetV2(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
de_dataset = de.ImageFolderDataset(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
else:
dataset = TxtDataset(root, data_dir)
sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle)
......
model_zoo/official/cv/yolov3_darknet53/src/yolo_dataset.py
浏览文件 @ 3bac9d37
......@@ -178,7 +178,7 @@ def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num,
compose_map_func = (lambda image, img_id: reshape_fn(image, img_id, config))
ds = ds.map(input_columns=["image", "img_id"],
output_columns=["image", "image_shape", "img_id"],
columns_order=["image", "image_shape", "img_id"],
column_order=["image", "image_shape", "img_id"],
operations=compose_map_func, num_parallel_workers=8)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8)
ds = ds.batch(batch_size, drop_remainder=True)
......
model_zoo/official/cv/yolov3_darknet53_quant/src/yolo_dataset.py
浏览文件 @ 3bac9d37
......@@ -175,7 +175,7 @@ def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num,
compose_map_func = (lambda image, img_id: reshape_fn(image, img_id, config))
ds = ds.map(input_columns=["image", "img_id"],
output_columns=["image", "image_shape", "img_id"],
columns_order=["image", "image_shape", "img_id"],
column_order=["image", "image_shape", "img_id"],
operations=compose_map_func, num_parallel_workers=8)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8)
ds = ds.batch(batch_size, drop_remainder=True)
......
model_zoo/official/cv/yolov3_resnet18/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -303,7 +303,7 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=1, device_num=
hwc_to_chw = C.HWC2CHW()
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
columns_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
column_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
......@@ -311,6 +311,6 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=1, device_num=
else:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "annotation"],
columns_order=["image", "image_shape", "annotation"],
column_order=["image", "image_shape", "annotation"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
return ds
model_zoo/official/nlp/bert/src/clue_classification_dataset_process.py
浏览文件 @ 3bac9d37
......@@ -43,7 +43,7 @@ def process_tnews_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
### Processing label
if data_usage == 'test':
dataset = dataset.map(input_columns=["id"], output_columns=["id", "label_id"],
columns_order=["id", "label_id", "sentence"], operations=ops.Duplicate())
column_order=["id", "label_id", "sentence"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["label_id"], operations=ops.Fill(0))
else:
label_vocab = text.Vocab.from_list(label_list)
......@@ -61,10 +61,10 @@ def process_tnews_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
dataset = dataset.map(input_columns=["sentence"], output_columns=["text_ids"], operations=lookup)
dataset = dataset.map(input_columns=["text_ids"], operations=ops.PadEnd([max_seq_len], 0))
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "mask_ids"],
columns_order=["text_ids", "mask_ids", "label_id"], operations=ops.Duplicate())
column_order=["text_ids", "mask_ids", "label_id"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["mask_ids"], operations=ops.Mask(ops.Relational.NE, 0, mstype.int32))
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "segment_ids"],
columns_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
column_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["segment_ids"], operations=ops.Fill(0))
dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
return dataset
......@@ -87,7 +87,7 @@ def process_cmnli_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
### Processing label
if data_usage == 'test':
dataset = dataset.map(input_columns=["id"], output_columns=["id", "label_id"],
columns_order=["id", "label_id", "sentence1", "sentence2"], operations=ops.Duplicate())
column_order=["id", "label_id", "sentence1", "sentence2"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["label_id"], operations=ops.Fill(0))
else:
label_vocab = text.Vocab.from_list(label_list)
......@@ -110,26 +110,26 @@ def process_cmnli_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
operations=ops.Concatenate(append=np.array(["[SEP]"], dtype='S')))
### Generating segment_ids
dataset = dataset.map(input_columns=["sentence1"], output_columns=["sentence1", "type_sentence1"],
columns_order=["sentence1", "type_sentence1", "sentence2", "label_id"],
column_order=["sentence1", "type_sentence1", "sentence2", "label_id"],
operations=ops.Duplicate())
dataset = dataset.map(input_columns=["sentence2"], output_columns=["sentence2", "type_sentence2"],
columns_order=["sentence1", "type_sentence1", "sentence2", "type_sentence2", "label_id"],
column_order=["sentence1", "type_sentence1", "sentence2", "type_sentence2", "label_id"],
operations=ops.Duplicate())
dataset = dataset.map(input_columns=["type_sentence1"], operations=[lookup, ops.Fill(0)])
dataset = dataset.map(input_columns=["type_sentence2"], operations=[lookup, ops.Fill(1)])
dataset = dataset.map(input_columns=["type_sentence1", "type_sentence2"], output_columns=["segment_ids"],
columns_order=["sentence1", "sentence2", "segment_ids", "label_id"],
column_order=["sentence1", "sentence2", "segment_ids", "label_id"],
operations=ops.Concatenate())
dataset = dataset.map(input_columns=["segment_ids"], operations=ops.PadEnd([max_seq_len], 0))
### Generating text_ids
dataset = dataset.map(input_columns=["sentence1", "sentence2"], output_columns=["text_ids"],
columns_order=["text_ids", "segment_ids", "label_id"],
column_order=["text_ids", "segment_ids", "label_id"],
operations=ops.Concatenate())
dataset = dataset.map(input_columns=["text_ids"], operations=lookup)
dataset = dataset.map(input_columns=["text_ids"], operations=ops.PadEnd([max_seq_len], 0))
### Generating mask_ids
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "mask_ids"],
columns_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
column_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["mask_ids"], operations=ops.Mask(ops.Relational.NE, 0, mstype.int32))
dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
return dataset
model_zoo/official/recommend/deepfm/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -213,7 +213,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds
......@@ -261,7 +261,7 @@ def _get_tf_dataset(directory, train_mode=True, epochs=1, batch_size=1000,
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds
......
model_zoo/official/recommend/wide_and_deep/src/datasets.py
浏览文件 @ 3bac9d37
......@@ -230,7 +230,7 @@ def _get_tf_dataset(data_dir, train_mode=True, epochs=1, batch_size=1000,
ds = ds.map(operations=_padding_func(batch_size, manual_shape, target_column),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
# if train_mode:
ds = ds.repeat(epochs)
return ds
......@@ -270,7 +270,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100
ds = ds.batch(int(batch_size / line_per_sample), drop_remainder=True)
ds = ds.map(_padding_func(batch_size, manual_shape, target_column),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds
......
model_zoo/official/recommend/wide_and_deep_multitable/src/datasets.py
浏览文件 @ 3bac9d37
......@@ -263,7 +263,7 @@ def _get_tf_dataset(data_dir,
'multi_doc_ad_topic_id_mask', 'ad_id', 'display_ad_and_is_leak',
'display_id', 'is_leak'
],
columns_order=[
column_order=[
'label', 'continue_val', 'indicator_id', 'emb_128_id',
'emb_64_single_id', 'multi_doc_ad_category_id',
'multi_doc_ad_category_id_mask', 'multi_doc_event_entity_id',
......
tests/st/mem_reuse/resnet_cifar_memreuse.py
浏览文件 @ 3bac9d37
......@@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype
import mindspore.context as context
import mindspore.dataset as de
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.communication.management import init
......
tests/st/mem_reuse/resnet_cifar_normal.py
浏览文件 @ 3bac9d37
......@@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype
import mindspore.context as context
import mindspore.dataset as de
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.communication.management import init
......
tests/st/model_zoo_tests/wide_and_deep/python_file_for_ci/datasets.py
浏览文件 @ 3bac9d37
......@@ -57,7 +57,7 @@ def _get_tf_dataset(data_dir, train_mode=True, epochs=1, batch_size=1000,
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
#if train_mode:
ds = ds.repeat(epochs)
return ds
......@@ -97,7 +97,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds
......
tests/st/model_zoo_tests/yolov3/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -22,7 +22,7 @@ from matplotlib.colors import rgb_to_hsv, hsv_to_rgb
from PIL import Image
import mindspore.dataset as de
from mindspore.mindrecord import FileWriter
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
from src.config import ConfigYOLOV3ResNet18
iter_cnt = 0
......@@ -305,7 +305,7 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=10, device_num
hwc_to_chw = C.HWC2CHW()
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
columns_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
column_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
......@@ -313,6 +313,6 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=10, device_num
else:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "annotation"],
columns_order=["image", "image_shape", "annotation"],
column_order=["image", "image_shape", "annotation"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
return ds
tests/st/networks/models/deeplabv3/src/md_dataset.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
"""Dataset module."""
from PIL import Image
import mindspore.dataset as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
import numpy as np
from .ei_dataset import HwVocRawDataset
......
tests/st/networks/models/resnet50/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@
import os
import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
......@@ -39,10 +39,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
......
tests/st/networks/models/resnet50/src_thor/dataset.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,7 @@ import mindspore.common.dtype as mstype
import mindspore.dataset as dataset
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
dataset.config.set_seed(1)
......@@ -43,10 +43,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
......
tests/st/networks/test_gpu_lenet.py
浏览文件 @ 3bac9d37
......@@ -21,11 +21,11 @@ import pytest
import mindspore.context as context
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.common import dtype as mstype
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.nn import Dense, TrainOneStepCell, WithLossCell
from mindspore.nn.metrics import Accuracy
from mindspore.nn.optim import Momentum
......
tests/st/ops/ascend/test_tdt_data_ms.py
浏览文件 @ 3bac9d37
......@@ -17,11 +17,11 @@ import numpy as np
import mindspore.context as context
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore.common.api import _executor
from mindspore.common.tensor import Tensor
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.ops import operations as P
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
......@@ -83,8 +83,6 @@ if __name__ == '__main__':
class dataiter(nn.Cell):
def __init__(self):
super(dataiter, self).__init__()
def construct(self):
input_, _ = get_next()
......
tests/st/probability/dataset.py
浏览文件 @ 3bac9d37
......@@ -17,9 +17,9 @@ Produce the dataset
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.dataset.transforms.c_transforms as C
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.common import dtype as mstype
......
tests/st/probability/test_gpu_svi_cvae.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@ import os
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context, Tensor
from mindspore.ops import operations as P
......
tests/st/probability/test_gpu_svi_vae.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@ import os
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context, Tensor
from mindspore.ops import operations as P
......
tests/st/probability/test_gpu_vae_gan.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ The VAE interface can be called to construct VAE-GAN network.
import os
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context
from mindspore.ops import operations as P
......
tests/st/probability/test_uncertainty.py
浏览文件 @ 3bac9d37
......@@ -15,12 +15,12 @@
""" test uncertainty toolbox """
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context, Tensor
from mindspore.common import dtype as mstype
from mindspore.common.initializer import TruncatedNormal
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.nn.probability.toolbox.uncertainty_evaluation import UncertaintyEvaluation
from mindspore.train.serialization import load_checkpoint, load_param_into_net
......
tests/st/ps/full_ps/test_full_ps_lenet.py
浏览文件 @ 3bac9d37
......@@ -19,10 +19,10 @@ import argparse
import mindspore.context as context
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore.common import dtype as mstype
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.nn.metrics import Accuracy
from mindspore.train import Model
from mindspore.train.callback import LossMonitor
......
tests/st/pynative/test_pynative_resnet50.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,7 @@ import pytest
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
import mindspore.ops.functional as F
......
tests/st/quantization/lenet_quant/dataset.py
浏览文件 @ 3bac9d37
......@@ -17,9 +17,9 @@ Produce the dataset
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.dataset.transforms.c_transforms as C
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.common import dtype as mstype
......
tests/st/summary/test_summary.py
浏览文件 @ 3bac9d37
......@@ -25,8 +25,8 @@ from mindspore import nn, Tensor, context
from mindspore.nn.metrics import Accuracy
from mindspore.nn.optim import Momentum
from mindspore.dataset.transforms import c_transforms as C
from mindspore.dataset.transforms.vision import c_transforms as CV
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import c_transforms as CV
from mindspore.dataset.vision import Inter
from mindspore.common import dtype as mstype
from mindspore.common.initializer import TruncatedNormal
from mindspore.ops import operations as P
......
tests/st/tbe_networks/resnet_cifar.py
浏览文件 @ 3bac9d37
......@@ -24,7 +24,7 @@ from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMoni
from mindspore.train.serialization import load_checkpoint, load_param_into_net
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore import Tensor
from mindspore import context
......
tests/st/tbe_networks/test_resnet_cifar_1p.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,7 @@ from resnet import resnet50
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
import mindspore.ops.functional as F
from mindspore import Tensor
......
tests/st/tbe_networks/test_resnet_cifar_8p.py
浏览文件 @ 3bac9d37
......@@ -22,7 +22,7 @@ from resnet import resnet50
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
import mindspore.ops.functional as F
from mindspore import Tensor
......
tests/ut/python/dataset/test_HWC2CHW.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing HWC2CHW op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, visualize_list, save_and_check_md5
......@@ -99,8 +100,8 @@ def test_HWC2CHW_comp(plot=False):
py_vision.ToTensor(),
py_vision.HWC2CHW()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
image_c_transposed = []
image_py_transposed = []
......
tests/ut/python/dataset/test_apply.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = "../data/dataset/testPK/data"
......@@ -46,8 +46,8 @@ def test_apply_generator_case():
def test_apply_imagefolder_case():
# apply dataset map operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
data2 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
decode_op = vision.Decode()
normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0])
......
tests/ut/python/dataset/test_autocontrast.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing AutoContrast op in DE
"""
import numpy as np
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as F
import mindspore.dataset.vision.c_transforms as C
from mindspore import log as logger
from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5
......@@ -35,14 +36,14 @@ def test_auto_contrast_py(plot=False):
logger.info("Test AutoContrast Python Op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = ds.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
......@@ -55,15 +56,16 @@ def test_auto_contrast_py(plot=False):
axis=0)
# AutoContrast Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_auto_contrast = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=10.0, ignore=[10, 20]),
F.ToTensor()])
transforms_auto_contrast = \
mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=10.0, ignore=[10, 20]),
F.ToTensor()])
ds_auto_contrast = ds.map(input_columns="image",
operations=transforms_auto_contrast())
operations=transforms_auto_contrast)
ds_auto_contrast = ds_auto_contrast.batch(512)
......@@ -96,15 +98,15 @@ def test_auto_contrast_c(plot=False):
logger.info("Test AutoContrast C Op")
# AutoContrast Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224))])
python_op = F.AutoContrast(cutoff=10.0, ignore=[10, 20])
c_op = C.AutoContrast(cutoff=10.0, ignore=[10, 20])
transforms_op = F.ComposeOp([lambda img: F.ToPIL()(img.astype(np.uint8)),
python_op,
np.array])()
transforms_op = mindspore.dataset.transforms.py_transforms.Compose([lambda img: F.ToPIL()(img.astype(np.uint8)),
python_op,
np.array])
ds_auto_contrast_py = ds.map(input_columns="image",
operations=transforms_op)
......@@ -119,7 +121,7 @@ def test_auto_contrast_c(plot=False):
image,
axis=0)
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224))])
......@@ -159,17 +161,18 @@ def test_auto_contrast_one_channel_c(plot=False):
logger.info("Test AutoContrast C Op With One Channel Images")
# AutoContrast Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224))])
python_op = F.AutoContrast()
c_op = C.AutoContrast()
# not using F.ToTensor() since it converts to floats
transforms_op = F.ComposeOp([lambda img: (np.array(img)[:, :, 0]).astype(np.uint8),
F.ToPIL(),
python_op,
np.array])()
transforms_op = mindspore.dataset.transforms.py_transforms.Compose(
[lambda img: (np.array(img)[:, :, 0]).astype(np.uint8),
F.ToPIL(),
python_op,
np.array])
ds_auto_contrast_py = ds.map(input_columns="image",
operations=transforms_op)
......@@ -184,7 +187,7 @@ def test_auto_contrast_one_channel_c(plot=False):
image,
axis=0)
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -248,7 +251,7 @@ def test_auto_contrast_invalid_ignore_param_c():
"""
logger.info("Test AutoContrast C Op with invalid ignore parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -260,7 +263,7 @@ def test_auto_contrast_invalid_ignore_param_c():
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Argument ignore with value 255.5 is not of type" in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -279,7 +282,7 @@ def test_auto_contrast_invalid_cutoff_param_c():
"""
logger.info("Test AutoContrast C Op with invalid cutoff parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -291,7 +294,7 @@ def test_auto_contrast_invalid_cutoff_param_c():
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -310,22 +313,22 @@ def test_auto_contrast_invalid_ignore_param_py():
"""
logger.info("Test AutoContrast python Op with invalid ignore parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=255.5),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=255.5),
F.ToTensor()])])
except TypeError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Argument ignore with value 255.5 is not of type" in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=(10, 100)),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=(10, 100)),
F.ToTensor()])])
except TypeError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Argument ignore with value (10,100) is not of type" in str(error)
......@@ -337,22 +340,22 @@ def test_auto_contrast_invalid_cutoff_param_py():
"""
logger.info("Test AutoContrast python Op with invalid cutoff parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=-10.0),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=-10.0),
F.ToTensor()])])
except ValueError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=120.0),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=120.0),
F.ToTensor()])])
except ValueError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
......
tests/ut/python/dataset/test_batch.py
浏览文件 @ 3bac9d37
......@@ -449,6 +449,22 @@ def test_batch_exception_13():
logger.info("Got an exception in DE: {}".format(str(e)))
assert "shard_id" in str(e)
# test non-functional parameters
try:
data1 = data1.batch(batch_size, output_columns="3")
sum([1 for _ in data1])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "output_columns is currently not implemented." in str(e)
try:
data1 = data1.batch(batch_size, column_order="3")
sum([1 for _ in data1])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "column_order is currently not implemented." in str(e)
if __name__ == '__main__':
test_batch_01()
......
tests/ut/python/dataset/test_bounding_box_augment.py
浏览文件 @ 3bac9d37
......@@ -19,7 +19,7 @@ Testing the bounding box augment op in DE
import numpy as np
import mindspore.log as logger
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \
config_get_set_seed, config_get_set_num_parallel_workers, save_and_check_md5
......@@ -51,7 +51,7 @@ def test_bounding_box_augment_with_rotation_op(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "bounding_box_augment_rotation_c_result.npz"
......@@ -90,7 +90,7 @@ def test_bounding_box_augment_with_crop_op(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "bounding_box_augment_crop_c_result.npz"
......@@ -128,7 +128,7 @@ def test_bounding_box_augment_valid_ratio_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op]) # Add column for "bbox"
filename = "bounding_box_augment_valid_ratio_c_result.npz"
......@@ -165,7 +165,7 @@ def test_bounding_box_augment_op_coco_c(plot_vis=False):
dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
unaugSamp, augSamp = [], []
......@@ -197,17 +197,17 @@ def test_bounding_box_augment_valid_edge_c(plot_vis=False):
# Add column for "bbox"
dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=lambda img, bbox:
(img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32)))
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=lambda img, bbox:
(img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32)))
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "bounding_box_augment_valid_edge_c_result.npz"
save_and_check_md5(dataVoc2, filename, generate_golden=GENERATE_GOLDEN)
......@@ -240,7 +240,7 @@ def test_bounding_box_augment_invalid_ratio_c():
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op]) # Add column for "bbox"
except ValueError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
......
tests/ut/python/dataset/test_cache_map.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ Testing cache operator with mappable datasets
import os
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
from util import save_and_check_md5
......@@ -46,7 +46,7 @@ def test_cache_map_basic1():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.map(input_columns=["image"], operations=decode_op)
ds1 = ds1.repeat(4)
......@@ -75,7 +75,7 @@ def test_cache_map_basic2():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR)
decode_op = c_vision.Decode()
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
ds1 = ds1.repeat(4)
......@@ -104,7 +104,7 @@ def test_cache_map_basic3():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR)
decode_op = c_vision.Decode()
ds1 = ds1.repeat(4)
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
......@@ -128,7 +128,7 @@ def test_cache_map_basic4():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.repeat(4)
ds1 = ds1.map(input_columns=["image"], operations=decode_op)
......@@ -165,7 +165,7 @@ def test_cache_map_failure1():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
ds1 = ds1.repeat(4)
......
tests/ut/python/dataset/test_cache_nomap.py
浏览文件 @ 3bac9d37
......@@ -19,7 +19,7 @@ import os
import pytest
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
......
tests/ut/python/dataset/test_center_crop.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing CenterCrop op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, visualize_list, save_and_check_md5
......@@ -93,8 +94,8 @@ def test_center_crop_comp(height=375, width=375, plot=False):
py_vision.CenterCrop([height, width]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
image_c_cropped = []
image_py_cropped = []
......@@ -123,9 +124,9 @@ def test_crop_grayscale(height=375, width=375):
(lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8))
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=transform)
# If input is grayscale, the output dimensions should be single channel
crop_gray = vision.CenterCrop([height, width])
......
tests/ut/python/dataset/test_concat.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,8 @@ import numpy as np
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as F
from mindspore import log as logger
......@@ -317,15 +318,15 @@ def test_concat_14():
DATA_DIR = "../data/dataset/testPK/data"
DATA_DIR2 = "../data/dataset/testImageNetData/train/"
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=3)
data2 = ds.ImageFolderDatasetV2(DATA_DIR2, num_samples=2)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=3)
data2 = ds.ImageFolderDataset(DATA_DIR2, num_samples=2)
transforms1 = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms1 = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
data1 = data1.map(input_columns=["image"], operations=transforms1())
data2 = data2.map(input_columns=["image"], operations=transforms1())
data1 = data1.map(input_columns=["image"], operations=transforms1)
data2 = data2.map(input_columns=["image"], operations=transforms1)
data3 = data1 + data2
expected, output = [], []
......@@ -351,7 +352,7 @@ def test_concat_15():
DATA_DIR = "../data/dataset/testPK/data"
DATA_DIR2 = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
data1 = ds.ImageFolderDatasetV2(DATA_DIR)
data1 = ds.ImageFolderDataset(DATA_DIR)
data2 = ds.TFRecordDataset(DATA_DIR2, columns_list=["image"])
data1 = data1.project(["image"])
......
tests/ut/python/dataset/test_concatenate_op.py
浏览文件 @ 3bac9d37
......@@ -74,7 +74,7 @@ def test_concatenate_op_multi_input_string():
concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor, append=append_tensor)
data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"],
data = data.map(input_columns=["col1", "col2"], column_order=["out1"], output_columns=["out1"],
operations=concatenate_op)
expected = np.array(["dw", "df", "1", "2", "d", "3", "4", "e", "dwsdf", "df"], dtype='S')
for data_row in data:
......@@ -89,7 +89,7 @@ def test_concatenate_op_multi_input_numeric():
concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor)
data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"],
data = data.map(input_columns=["col1", "col2"], column_order=["out1"], output_columns=["out1"],
operations=concatenate_op)
expected = np.array([3, 5, 1, 2, 3, 4])
for data_row in data:
......
tests/ut/python/dataset/test_config.py
浏览文件 @ 3bac9d37
......@@ -21,8 +21,9 @@ import glob
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import dataset_equal
......@@ -283,8 +284,8 @@ def test_deterministic_python_seed():
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor(),
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
data1_output = []
# config.set_seed() calls random.seed()
for data_one in data1.create_dict_iterator(num_epochs=1):
......@@ -292,7 +293,7 @@ def test_deterministic_python_seed():
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
# config.set_seed() calls random.seed(), resets seed for next dataset iterator
ds.config.set_seed(0)
......@@ -326,8 +327,8 @@ def test_deterministic_python_seed_multi_thread():
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor(),
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform(), python_multiprocessing=True)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform, python_multiprocessing=True)
data1_output = []
# config.set_seed() calls random.seed()
for data_one in data1.create_dict_iterator(num_epochs=1):
......@@ -336,7 +337,7 @@ def test_deterministic_python_seed_multi_thread():
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
# If seed is set up on constructor
data2 = data2.map(input_columns=["image"], operations=transform(), python_multiprocessing=True)
data2 = data2.map(input_columns=["image"], operations=transform, python_multiprocessing=True)
# config.set_seed() calls random.seed()
ds.config.set_seed(0)
......
tests/ut/python/dataset/test_cut_out.py
浏览文件 @ 3bac9d37
......@@ -18,8 +18,9 @@ Testing CutOut op in DE
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c
import mindspore.dataset.transforms.vision.py_transforms as f
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c
import mindspore.dataset.vision.py_transforms as f
from mindspore import log as logger
from util import visualize_image, visualize_list, diff_mse, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -43,8 +44,8 @@ def test_cut_out_op(plot=False):
f.ToTensor(),
f.RandomErasing(value='random')
]
transform_1 = f.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -89,8 +90,8 @@ def test_cut_out_op_multicut(plot=False):
f.Decode(),
f.ToTensor(),
]
transform_1 = f.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -144,8 +145,8 @@ def test_cut_out_md5():
f.ToTensor(),
f.Cutout(100)
]
transform = f.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename1 = "cut_out_01_c_result.npz"
......@@ -172,8 +173,8 @@ def test_cut_out_comp(plot=False):
f.ToTensor(),
f.Cutout(200)
]
transform_1 = f.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......
tests/ut/python/dataset/test_cutmix_batch_op.py
浏览文件 @ 3bac9d37
......@@ -18,9 +18,9 @@ Testing the CutMixBatch op in DE
import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.dataset.transforms.c_transforms as data_trans
import mindspore.dataset.transforms.vision.utils as mode
import mindspore.dataset.vision.utils as mode
from mindspore import log as logger
from util import save_and_check_md5, diff_mse, visualize_list, config_get_set_seed, \
config_get_set_num_parallel_workers
......@@ -119,11 +119,11 @@ def test_cutmix_batch_success2(plot=False):
def test_cutmix_batch_success3(plot=False):
"""
Test CutMixBatch op with default values for alpha and prob on a batch of HWC images on ImageFolderDatasetV2
Test CutMixBatch op with default values for alpha and prob on a batch of HWC images on ImageFolderDataset
"""
logger.info("test_cutmix_batch_success3")
ds_original = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False)
ds_original = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False)
decode_op = vision.Decode()
ds_original = ds_original.map(input_columns=["image"], operations=[decode_op])
ds_original = ds_original.batch(4, pad_info={}, drop_remainder=True)
......@@ -136,7 +136,7 @@ def test_cutmix_batch_success3(plot=False):
images_original = np.append(images_original, image, axis=0)
# CutMix Images
data1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False)
data1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False)
decode_op = vision.Decode()
data1 = data1.map(input_columns=["image"], operations=[decode_op])
......
tests/ut/python/dataset/test_dataset_numpy_slices.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ import numpy as np
import pandas as pd
import mindspore.dataset as de
from mindspore import log as logger
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
def test_numpy_slices_list_1():
......
tests/ut/python/dataset/test_datasets_celeba.py
浏览文件 @ 3bac9d37
......@@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
DATA_DIR = "../data/dataset/testCelebAData/"
......
tests/ut/python/dataset/test_datasets_coco.py
浏览文件 @ 3bac9d37
......@@ -14,7 +14,7 @@
# ==============================================================================
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
DATA_DIR = "../data/dataset/testCOCO/train/"
DATA_DIR_2 = "../data/dataset/testCOCO/train"
......
tests/ut/python/dataset/test_datasets_generator.py
浏览文件 @ 3bac9d37
......@@ -244,7 +244,7 @@ def test_generator_8():
data1 = data1.map(input_columns="col0", output_columns="out0", operations=(lambda x: x * 3),
num_parallel_workers=2)
data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x * 7, x)),
num_parallel_workers=2, columns_order=["out0", "out1", "out2"])
num_parallel_workers=2, column_order=["out0", "out1", "out2"])
data1 = data1.map(input_columns="out2", output_columns="out2", operations=(lambda x: x + 1),
num_parallel_workers=2)
......@@ -299,7 +299,7 @@ def test_generator_10():
# apply dataset operations
data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"])
data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)),
columns_order=['col0', 'out1', 'out2'], num_parallel_workers=2)
column_order=['col0', 'out1', 'out2'], num_parallel_workers=2)
# Expected column order is |col0|out1|out2|
i = 0
......@@ -318,17 +318,17 @@ def test_generator_11():
Test map column order when len(input_columns) != len(output_columns).
"""
logger.info("Test map column order when len(input_columns) != len(output_columns), "
"and columns_order drops some columns.")
"and column_order drops some columns.")
# apply dataset operations
data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"])
data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)),
columns_order=['out1', 'out2'], num_parallel_workers=2)
column_order=['out1', 'out2'], num_parallel_workers=2)
# Expected column order is |out1|out2|
i = 0
for item in data1.create_tuple_iterator(num_epochs=1):
# len should be 2 because col0 is dropped (not included in columns_order)
# len should be 2 because col0 is dropped (not included in column_order)
assert len(item) == 2
golden = np.array([[i, i + 1], [i + 2, i + 3]])
np.testing.assert_array_equal(item[0], golden)
......@@ -358,7 +358,7 @@ def test_generator_12():
i = i + 1
data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"])
data1 = data1.map(operations=(lambda x: (x * 5)), columns_order=["col1", "col0"], num_parallel_workers=2)
data1 = data1.map(operations=(lambda x: (x * 5)), column_order=["col1", "col0"], num_parallel_workers=2)
# Expected column order is |col0|col1|
i = 0
......@@ -392,7 +392,7 @@ def test_generator_13():
i = i + 1
for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary
# len should be 2 because col0 is dropped (not included in columns_order)
# len should be 2 because col0 is dropped (not included in column_order)
assert len(item) == 2
golden = np.array([i * 5])
np.testing.assert_array_equal(item["out0"], golden)
......@@ -508,7 +508,7 @@ def test_generator_error_3():
for _ in data1:
pass
assert "When (len(input_columns) != len(output_columns)), columns_order must be specified." in str(info.value)
assert "When (len(input_columns) != len(output_columns)), column_order must be specified." in str(info.value)
def test_generator_error_4():
......
tests/ut/python/dataset/test_datasets_get_dataset_size.py
浏览文件 @ 3bac9d37
......@@ -27,16 +27,16 @@ CIFAR100_DATA_DIR = "../data/dataset/testCifar100Data"
def test_imagenet_rawdata_dataset_size():
ds_total = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR)
ds_total = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR)
assert ds_total.get_dataset_size() == 6
ds_shard_1_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=1, shard_id=0)
ds_shard_1_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=1, shard_id=0)
assert ds_shard_1_0.get_dataset_size() == 6
ds_shard_2_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=2, shard_id=0)
ds_shard_2_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=2, shard_id=0)
assert ds_shard_2_0.get_dataset_size() == 3
ds_shard_3_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=3, shard_id=0)
ds_shard_3_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=3, shard_id=0)
assert ds_shard_3_0.get_dataset_size() == 2
......
tests/ut/python/dataset/test_datasets_imagefolder.py
浏览文件 @ 3bac9d37
......@@ -24,7 +24,7 @@ def test_imagefolder_basic():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR)
data1 = ds.ImageFolderDataset(DATA_DIR)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -44,7 +44,7 @@ def test_imagefolder_numsamples():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10, num_parallel_workers=2)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10, num_parallel_workers=2)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -58,7 +58,7 @@ def test_imagefolder_numsamples():
assert num_iter == 10
random_sampler = ds.RandomSampler(num_samples=3, replacement=True)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
num_iter = 0
for item in data1.create_dict_iterator(num_epochs=1):
......@@ -67,7 +67,7 @@ def test_imagefolder_numsamples():
assert num_iter == 3
random_sampler = ds.RandomSampler(num_samples=3, replacement=False)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
num_iter = 0
for item in data1.create_dict_iterator(num_epochs=1):
......@@ -82,7 +82,7 @@ def test_imagefolder_numshards():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -102,7 +102,7 @@ def test_imagefolder_shardid():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=1)
data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=1)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -122,7 +122,7 @@ def test_imagefolder_noshuffle():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, shuffle=False)
data1 = ds.ImageFolderDataset(DATA_DIR, shuffle=False)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -142,7 +142,7 @@ def test_imagefolder_extrashuffle():
repeat_count = 2
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, shuffle=True)
data1 = ds.ImageFolderDataset(DATA_DIR, shuffle=True)
data1 = data1.shuffle(buffer_size=5)
data1 = data1.repeat(repeat_count)
......@@ -164,7 +164,7 @@ def test_imagefolder_classindex():
# apply dataset operations
class_index = {"class3": 333, "class1": 111}
data1 = ds.ImageFolderDatasetV2(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = ds.ImageFolderDataset(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = data1.repeat(repeat_count)
golden = [111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111,
......@@ -189,7 +189,7 @@ def test_imagefolder_negative_classindex():
# apply dataset operations
class_index = {"class3": -333, "class1": 111}
data1 = ds.ImageFolderDatasetV2(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = ds.ImageFolderDataset(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = data1.repeat(repeat_count)
golden = [111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111,
......@@ -214,7 +214,7 @@ def test_imagefolder_extensions():
# apply dataset operations
ext = [".jpg", ".JPEG"]
data1 = ds.ImageFolderDatasetV2(DATA_DIR, extensions=ext)
data1 = ds.ImageFolderDataset(DATA_DIR, extensions=ext)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -235,7 +235,7 @@ def test_imagefolder_decode():
# apply dataset operations
ext = [".jpg", ".JPEG"]
data1 = ds.ImageFolderDatasetV2(DATA_DIR, extensions=ext, decode=True)
data1 = ds.ImageFolderDataset(DATA_DIR, extensions=ext, decode=True)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -262,7 +262,7 @@ def test_sequential_sampler():
# apply dataset operations
sampler = ds.SequentialSampler()
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
result = []
......@@ -283,7 +283,7 @@ def test_random_sampler():
# apply dataset operations
sampler = ds.RandomSampler()
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -304,7 +304,7 @@ def test_distributed_sampler():
# apply dataset operations
sampler = ds.DistributedSampler(10, 1)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -325,7 +325,7 @@ def test_pk_sampler():
# apply dataset operations
sampler = ds.PKSampler(3)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -347,7 +347,7 @@ def test_subset_random_sampler():
# apply dataset operations
indices = [0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11]
sampler = ds.SubsetRandomSampler(indices)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -369,7 +369,7 @@ def test_weighted_random_sampler():
# apply dataset operations
weights = [1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1]
sampler = ds.WeightedRandomSampler(weights, 11)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -389,7 +389,7 @@ def test_imagefolder_rename():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -421,8 +421,8 @@ def test_imagefolder_zip():
repeat_count = 2
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10)
data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10)
data2 = ds.ImageFolderDataset(DATA_DIR, num_samples=10)
data1 = data1.repeat(repeat_count)
# rename dataset2 for no conflict
......
tests/ut/python/dataset/test_datasets_sharding.py
浏览文件 @ 3bac9d37
......@@ -20,9 +20,9 @@ def test_imagefolder_shardings(print_res=False):
image_folder_dir = "../data/dataset/testPK/data"
def sharding_config(num_shards, shard_id, num_samples, shuffle, class_index, repeat_cnt=1):
data1 = ds.ImageFolderDatasetV2(image_folder_dir, num_samples=num_samples, num_shards=num_shards,
shard_id=shard_id,
shuffle=shuffle, class_indexing=class_index, decode=True)
data1 = ds.ImageFolderDataset(image_folder_dir, num_samples=num_samples, num_shards=num_shards,
shard_id=shard_id,
shuffle=shuffle, class_indexing=class_index, decode=True)
data1 = data1.repeat(repeat_cnt)
res = []
for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary
......
tests/ut/python/dataset/test_datasets_voc.py
浏览文件 @ 3bac9d37
......@@ -13,7 +13,7 @@
# limitations under the License.
# ==============================================================================
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
DATA_DIR = "../data/dataset/testVOC2012"
IMAGE_SHAPE = [2268, 2268, 2268, 2268, 642, 607, 561, 596, 612, 2268]
......
tests/ut/python/dataset/test_decode.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ Testing Decode op in DE
import cv2
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from util import diff_mse
......
tests/ut/python/dataset/test_deviceop_cpu.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
import time
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
......
tests/ut/python/dataset/test_duplicate_op.py
浏览文件 @ 3bac9d37
......@@ -24,7 +24,7 @@ import mindspore.dataset.transforms.c_transforms as ops
def compare(array):
data = ds.NumpySlicesDataset([array], column_names="x")
array = np.array(array)
data = data.map(input_columns=["x"], output_columns=["x", "y"], columns_order=["x", "y"],
data = data.map(input_columns=["x"], output_columns=["x", "y"], column_order=["x", "y"],
operations=ops.Duplicate())
for d in data.create_dict_iterator(num_epochs=1):
np.testing.assert_array_equal(array, d["x"])
......
tests/ut/python/dataset/test_epoch_ctrl.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,7 @@ import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
......
tests/ut/python/dataset/test_equalize.py
浏览文件 @ 3bac9d37
......@@ -18,8 +18,9 @@ Testing Equalize op in DE
import numpy as np
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as C
import mindspore.dataset.vision.py_transforms as F
from mindspore import log as logger
from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5
......@@ -36,14 +37,14 @@ def test_equalize_py(plot=False):
logger.info("Test Equalize")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = ds.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
......@@ -56,15 +57,15 @@ def test_equalize_py(plot=False):
axis=0)
# Color Equalized Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_equalize = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.Equalize(),
F.ToTensor()])
transforms_equalize = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.Equalize(),
F.ToTensor()])
ds_equalize = ds.map(input_columns="image",
operations=transforms_equalize())
operations=transforms_equalize)
ds_equalize = ds_equalize.batch(512)
......@@ -93,7 +94,7 @@ def test_equalize_c(plot=False):
logger.info("Test Equalize cpp op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = [C.Decode(), C.Resize(size=[224, 224])]
......@@ -111,7 +112,7 @@ def test_equalize_c(plot=False):
axis=0)
# Equalize Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transform_equalize = [C.Decode(), C.Resize(size=[224, 224]),
C.Equalize()]
......@@ -145,7 +146,7 @@ def test_equalize_py_c(plot=False):
logger.info("Test Equalize cpp and python op")
# equalize Images in cpp
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
......@@ -163,17 +164,17 @@ def test_equalize_py_c(plot=False):
axis=0)
# Equalize images in python
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
transforms_p_equalize = F.ComposeOp([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Equalize(),
np.array])
transforms_p_equalize = mindspore.dataset.transforms.py_transforms.Compose([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Equalize(),
np.array])
ds_p_equalize = ds.map(input_columns="image",
operations=transforms_p_equalize())
operations=transforms_p_equalize)
ds_p_equalize = ds_p_equalize.batch(512)
......@@ -204,7 +205,7 @@ def test_equalize_one_channel():
c_op = C.Equalize()
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -253,12 +254,12 @@ def test_equalize_md5_py():
logger.info("Test Equalize")
# First dataset
data1 = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
transforms = F.ComposeOp([F.Decode(),
F.Equalize(),
F.ToTensor()])
data1 = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Equalize(),
F.ToTensor()])
data1 = data1.map(input_columns="image", operations=transforms())
data1 = data1.map(input_columns="image", operations=transforms)
# Compare with expected md5 from images
filename = "equalize_01_result.npz"
save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN)
......@@ -271,7 +272,7 @@ def test_equalize_md5_c():
logger.info("Test Equalize cpp op with md5 check")
# Generate dataset
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_equalize = [C.Decode(),
C.Resize(size=[224, 224]),
......
tests/ut/python/dataset/test_exceptions.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
......
tests/ut/python/dataset/test_filterop.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as cde
import mindspore.dataset.vision.c_transforms as cde
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
......
tests/ut/python/dataset/test_five_crop.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,8 @@ import pytest
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as vision
from mindspore import log as logger
from util import visualize_list, save_and_check_md5
......@@ -39,8 +40,8 @@ def test_five_crop_op(plot=False):
vision.Decode(),
vision.ToTensor(),
]
transform_1 = vision.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -49,8 +50,8 @@ def test_five_crop_op(plot=False):
vision.FiveCrop(200),
lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 5 images
]
transform_2 = vision.ComposeOp(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2())
transform_2 = mindspore.dataset.transforms.py_transforms.Compose(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
......@@ -83,8 +84,8 @@ def test_five_crop_error_msg():
vision.FiveCrop(200),
vision.ToTensor()
]
transform = vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
with pytest.raises(RuntimeError) as info:
for _ in data:
......@@ -108,8 +109,8 @@ def test_five_crop_md5():
vision.FiveCrop(100),
lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 5 images
]
transform = vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "five_crop_01_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......
tests/ut/python/dataset/test_flat_map.py
浏览文件 @ 3bac9d37
......@@ -27,7 +27,7 @@ def test_flat_map_1():
def flat_map_func(x):
data_dir = x[0].item().decode('utf8')
d = ds.ImageFolderDatasetV2(data_dir)
d = ds.ImageFolderDataset(data_dir)
return d
data = ds.TextFileDataset(DATA_FILE)
......@@ -47,7 +47,7 @@ def test_flat_map_2():
def flat_map_func_1(x):
data_dir = x[0].item().decode('utf8')
d = ds.ImageFolderDatasetV2(data_dir)
d = ds.ImageFolderDataset(data_dir)
return d
def flat_map_func_2(x):
......
tests/ut/python/dataset/test_get_col_names.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
CELEBA_DIR = "../data/dataset/testCelebAData"
CIFAR10_DIR = "../data/dataset/testCifar10Data"
......@@ -75,7 +75,7 @@ def test_get_column_name_generator():
def test_get_column_name_imagefolder():
data = ds.ImageFolderDatasetV2(IMAGE_FOLDER_DIR)
data = ds.ImageFolderDataset(IMAGE_FOLDER_DIR)
assert data.get_col_names() == ["image", "label"]
......@@ -105,7 +105,7 @@ def test_get_column_name_map():
assert data.get_col_names() == ["col1", "label"]
data = ds.Cifar10Dataset(CIFAR10_DIR)
data = data.map(input_columns=["image"], operations=center_crop_op, output_columns=["col1", "col2"],
columns_order=["col2", "col1"])
column_order=["col2", "col1"])
assert data.get_col_names() == ["col2", "col1"]
......
tests/ut/python/dataset/test_get_size.py
浏览文件 @ 3bac9d37
......@@ -150,13 +150,13 @@ def test_manifest():
def test_imagefolder():
data = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/")
data = ds.ImageFolderDataset("../data/dataset/testPK/data/")
assert data.get_dataset_size() == 44
assert data.num_classes() == 4
data = data.shuffle(100)
assert data.num_classes() == 4
data = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/", num_samples=10)
data = ds.ImageFolderDataset("../data/dataset/testPK/data/", num_samples=10)
assert data.get_dataset_size() == 10
assert data.num_classes() == 4
......
tests/ut/python/dataset/test_invert.py
浏览文件 @ 3bac9d37
......@@ -18,8 +18,9 @@ Testing Invert op in DE
import numpy as np
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as F
import mindspore.dataset.vision.c_transforms as C
from mindspore import log as logger
from util import visualize_list, save_and_check_md5, diff_mse
......@@ -35,14 +36,14 @@ def test_invert_py(plot=False):
logger.info("Test Invert Python op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = ds.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
......@@ -55,15 +56,15 @@ def test_invert_py(plot=False):
axis=0)
# Color Inverted Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_invert = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.Invert(),
F.ToTensor()])
transforms_invert = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.Invert(),
F.ToTensor()])
ds_invert = ds.map(input_columns="image",
operations=transforms_invert())
operations=transforms_invert)
ds_invert = ds_invert.batch(512)
......@@ -92,7 +93,7 @@ def test_invert_c(plot=False):
logger.info("Test Invert cpp op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = [C.Decode(), C.Resize(size=[224, 224])]
......@@ -110,7 +111,7 @@ def test_invert_c(plot=False):
axis=0)
# Invert Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transform_invert = [C.Decode(), C.Resize(size=[224, 224]),
C.Invert()]
......@@ -144,7 +145,7 @@ def test_invert_py_c(plot=False):
logger.info("Test Invert cpp and python op")
# Invert Images in cpp
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
......@@ -162,17 +163,17 @@ def test_invert_py_c(plot=False):
axis=0)
# invert images in python
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
transforms_p_invert = F.ComposeOp([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Invert(),
np.array])
transforms_p_invert = mindspore.dataset.transforms.py_transforms.Compose([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Invert(),
np.array])
ds_p_invert = ds.map(input_columns="image",
operations=transforms_p_invert())
operations=transforms_p_invert)
ds_p_invert = ds_p_invert.batch(512)
......@@ -203,7 +204,7 @@ def test_invert_one_channel():
c_op = C.Invert()
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -224,13 +225,13 @@ def test_invert_md5_py():
logger.info("Test Invert python op with md5 check")
# Generate dataset
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_invert = F.ComposeOp([F.Decode(),
F.Invert(),
F.ToTensor()])
transforms_invert = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Invert(),
F.ToTensor()])
data = ds.map(input_columns="image", operations=transforms_invert())
data = ds.map(input_columns="image", operations=transforms_invert)
# Compare with expected md5 from images
filename = "invert_01_result_py.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -243,7 +244,7 @@ def test_invert_md5_c():
logger.info("Test Invert cpp op with md5 check")
# Generate dataset
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_invert = [C.Decode(),
C.Resize(size=[224, 224]),
......
tests/ut/python/dataset/test_linear_transformation.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,8 @@ Testing LinearTransformation op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, visualize_list, save_and_check_md5
......@@ -46,11 +47,11 @@ def test_linear_transformation_op(plot=False):
py_vision.CenterCrop([height, weight]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=transform)
# Note: if transformation matrix is diagonal matrix with all 1 in diagonal,
# the output matrix in expected to be the same as the input matrix.
data1 = data1.map(input_columns=["image"],
......@@ -58,7 +59,7 @@ def test_linear_transformation_op(plot=False):
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
image_transformed = []
image = []
......@@ -96,8 +97,8 @@ def test_linear_transformation_md5():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "linear_transformation_01_result.npz"
......@@ -126,8 +127,8 @@ def test_linear_transformation_exception_01():
py_vision.ToTensor(),
py_vision.LinearTransformation(None, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except TypeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Argument transformation_matrix with value None is not of type (<class 'numpy.ndarray'>,)" in str(e)
......@@ -155,8 +156,8 @@ def test_linear_transformation_exception_02():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, None)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except TypeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Argument mean_vector with value None is not of type (<class 'numpy.ndarray'>,)" in str(e)
......@@ -185,8 +186,8 @@ def test_linear_transformation_exception_03():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "square matrix" in str(e)
......@@ -215,8 +216,8 @@ def test_linear_transformation_exception_04():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "should match" in str(e)
......
tests/ut/python/dataset/test_minddataset.py
浏览文件 @ 3bac9d37
......@@ -25,9 +25,9 @@ import pytest
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.mindrecord import FileWriter
FILES_NUM = 4
......
tests/ut/python/dataset/test_mixup_label_smoothing.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,8 @@ import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as c
import mindspore.dataset.transforms.py_transforms as f
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
DATA_DIR = "../data/dataset/testImageNetData/train"
......@@ -33,14 +33,13 @@ def test_one_hot_op():
# define map operations
# ds = de.ImageFolderDataset(DATA_DIR, schema=SCHEMA_DIR)
dataset = ds.ImageFolderDatasetV2(DATA_DIR)
dataset = ds.ImageFolderDataset(DATA_DIR)
num_classes = 2
epsilon_para = 0.1
transforms = [f.OneHotOp(num_classes=num_classes, smoothing_rate=epsilon_para),
]
transform_label = py_vision.ComposeOp(transforms)
dataset = dataset.map(input_columns=["label"], operations=transform_label())
transforms = [f.OneHotOp(num_classes=num_classes, smoothing_rate=epsilon_para),]
transform_label = f.Compose(transforms)
dataset = dataset.map(input_columns=["label"], operations=transform_label)
golden_label = np.ones(num_classes) * epsilon_para / num_classes
golden_label[1] = 1 - epsilon_para / num_classes
......@@ -63,7 +62,7 @@ def test_mix_up_single():
resize_width = 224
# Create dataset and define map operations
ds1 = ds.ImageFolderDatasetV2(DATA_DIR_2)
ds1 = ds.ImageFolderDataset(DATA_DIR_2)
num_classes = 10
decode_op = c_vision.Decode()
......@@ -112,7 +111,7 @@ def test_mix_up_multi():
resize_width = 224
# Create dataset and define map operations
ds1 = ds.ImageFolderDatasetV2(DATA_DIR_2)
ds1 = ds.ImageFolderDataset(DATA_DIR_2)
num_classes = 3
decode_op = c_vision.Decode()
......
tests/ut/python/dataset/test_mixup_op.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ Testing the MixUpBatch op in DE
import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.dataset.transforms.c_transforms as data_trans
from mindspore import log as logger
from util import save_and_check_md5, diff_mse, visualize_list, config_get_set_seed, \
......@@ -74,12 +74,12 @@ def test_mixup_batch_success1(plot=False):
def test_mixup_batch_success2(plot=False):
"""
Test MixUpBatch op with specified alpha parameter on ImageFolderDatasetV2
Test MixUpBatch op with specified alpha parameter on ImageFolderDataset
"""
logger.info("test_mixup_batch_success2")
# Original Images
ds_original = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False)
ds_original = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False)
decode_op = vision.Decode()
ds_original = ds_original.map(input_columns=["image"], operations=[decode_op])
ds_original = ds_original.batch(4, pad_info={}, drop_remainder=True)
......@@ -92,7 +92,7 @@ def test_mixup_batch_success2(plot=False):
images_original = np.append(images_original, image, axis=0)
# MixUp Images
data1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False)
data1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False)
decode_op = vision.Decode()
data1 = data1.map(input_columns=["image"], operations=[decode_op])
......
tests/ut/python/dataset/test_normalizeOp.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing Normalize op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, save_and_check_md5, visualize_image
......@@ -59,10 +60,10 @@ def util_test_normalize(mean, std, op_type):
py_vision.ToTensor(),
py_vision.Normalize(mean, std)
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
data = data.map(input_columns=["image"], operations=transform)
else:
raise ValueError("Wrong parameter value")
return data
......@@ -78,10 +79,10 @@ def util_test_normalize_grayscale(num_output_channels, mean, std):
py_vision.ToTensor(),
py_vision.Normalize(mean, std)
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
data = data.map(input_columns=["image"], operations=transform)
return data
......@@ -130,17 +131,17 @@ def test_normalize_op_py(plot=False):
py_vision.Decode(),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
normalize_op = py_vision.Normalize(mean, std)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=transform)
data1 = data1.map(input_columns=["image"], operations=normalize_op)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
......
tests/ut/python/dataset/test_onehot_op.py
浏览文件 @ 3bac9d37
......@@ -19,7 +19,7 @@ import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as data_trans
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
from util import dataset_equal_with_function
......@@ -47,7 +47,7 @@ def test_one_hot():
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
one_hot_op = data_trans.OneHot(num_classes=depth)
data1 = data1.map(input_columns=["label"], operations=one_hot_op, columns_order=["label"])
data1 = data1.map(input_columns=["label"], operations=one_hot_op, column_order=["label"])
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["label"], shuffle=False)
......
tests/ut/python/dataset/test_opt.py
浏览文件 @ 3bac9d37
......@@ -15,30 +15,36 @@
import numpy as np
import mindspore.dataset as ds
from util import config_get_set_seed, config_get_set_num_parallel_workers
# Generate 1d int numpy array from 0 - 63
def generator_1d():
for i in range(64):
for i in range(4):
yield (np.array([i]),)
def test_case_0():
"""
Test 1D Generator
Test 1D Generator.
Test without explicit kwargs for input args.
"""
original_seed = config_get_set_seed(55)
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# apply dataset operations
# apply dataset qoperations
data1 = ds.GeneratorDataset(generator_1d, ["data"])
data1 = data1.shuffle(2)
data1 = data1.map(["data"], operations=(lambda x: x))
data1 = data1.map((lambda x: x), ["data"])
data1 = data1.batch(2)
for _ in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary
pass
expected_data = np.array([[[1], [2]], [[3], [0]]])
for i, data_row in enumerate(data1):
np.testing.assert_array_equal(data_row[0], expected_data[i])
# Restore configuration
ds.config.set_seed(original_seed)
ds.config.set_num_parallel_workers((original_num_parallel_workers))
if __name__ == "__main__":
......
tests/ut/python/dataset/test_opt_pass.py
浏览文件 @ 3bac9d37
......@@ -27,7 +27,7 @@ def test_map_reorder0():
# Generator -> Map
data0 = ds.GeneratorDataset(generator_mc, ["col0", "col1"])
data0 = data0.map(input_columns="col0", output_columns="out", columns_order=["col1", "out"],
data0 = data0.map(input_columns="col0", output_columns="out", column_order=["col1", "out"],
operations=(lambda x: x))
for item in data0.create_tuple_iterator(num_epochs=1): # each data is a dictionary
......@@ -43,11 +43,11 @@ def test_map_reorder1():
# Three map and zip
data0 = ds.GeneratorDataset(generator_mc, ["a0", "a1", "a2"])
data0 = data0.map(input_columns="a0", columns_order=["a2", "a1", "a0"], operations=(lambda x: x))
data0 = data0.map(input_columns="a0", column_order=["a2", "a1", "a0"], operations=(lambda x: x))
data1 = ds.GeneratorDataset(generator_mc, ["b0", "b1", "b2"])
data1 = data1.map(input_columns="b0", columns_order=["b1", "b2", "b0"], operations=(lambda x: x))
data1 = data1.map(input_columns="b0", column_order=["b1", "b2", "b0"], operations=(lambda x: x))
data2 = ds.zip((data0, data1))
data2 = data2.map(input_columns="a0", columns_order=["b2", "a2", "b1", "a1", "b0", "a0"], operations=(lambda x: x))
data2 = data2.map(input_columns="a0", column_order=["b2", "a2", "b1", "a1", "b0", "a0"], operations=(lambda x: x))
for item in data2.create_tuple_iterator(num_epochs=1):
assert item == [np.array(2), np.array(2), np.array(1), np.array(1), np.array(0), np.array(0)]
......
tests/ut/python/dataset/test_pad.py
浏览文件 @ 3bac9d37
......@@ -18,8 +18,9 @@ Testing Pad op in DE
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, save_and_check_md5
......@@ -51,9 +52,9 @@ def test_pad_op():
py_vision.Pad(100),
py_vision.ToTensor(),
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
c_image = item1["image"]
......@@ -85,9 +86,9 @@ def test_pad_grayscale():
(lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8))
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=transform)
# if input is grayscale, the output dimensions should be single channel
pad_gray = c_vision.Pad(100, fill_value=(20, 20, 20))
......@@ -140,8 +141,8 @@ def test_pad_md5():
py_vision.Pad(150),
py_vision.ToTensor(),
]
transform = py_vision.ComposeOp(pytrans)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(pytrans)
data2 = data2.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename1 = "pad_01_c_result.npz"
save_and_check_md5(data1, filename1, generate_golden=GENERATE_GOLDEN)
......
tests/ut/python/dataset/test_paddeddataset.py
浏览文件 @ 3bac9d37
......@@ -5,7 +5,7 @@ import numpy as np
import pytest
import mindspore.dataset as ds
from mindspore.mindrecord import FileWriter
import mindspore.dataset.transforms.vision.c_transforms as V_C
import mindspore.dataset.vision.c_transforms as V_C
from PIL import Image
FILES_NUM = 4
......@@ -204,7 +204,7 @@ def test_raise_error():
def test_imagefolder_padded():
DATA_DIR = "../data/dataset/testPK/data"
data = ds.ImageFolderDatasetV2(DATA_DIR)
data = ds.ImageFolderDataset(DATA_DIR)
data1 = [{'image': np.zeros(1, np.uint8), 'label': np.array(0, np.int32)},
{'image': np.zeros(2, np.uint8), 'label': np.array(1, np.int32)},
......@@ -230,7 +230,7 @@ def test_imagefolder_padded_with_decode():
count = 0
for shard_id in range(num_shards):
DATA_DIR = "../data/dataset/testPK/data"
data = ds.ImageFolderDatasetV2(DATA_DIR)
data = ds.ImageFolderDataset(DATA_DIR)
white_io = BytesIO()
Image.new('RGB', (224, 224), (255, 255, 255)).save(white_io, 'JPEG')
......@@ -258,7 +258,7 @@ def test_imagefolder_padded_with_decode_and_get_dataset_size():
count = 0
for shard_id in range(num_shards):
DATA_DIR = "../data/dataset/testPK/data"
data = ds.ImageFolderDatasetV2(DATA_DIR)
data = ds.ImageFolderDataset(DATA_DIR)
white_io = BytesIO()
Image.new('RGB', (224, 224), (255, 255, 255)).save(white_io, 'JPEG')
......
tests/ut/python/dataset/test_profiling.py
浏览文件 @ 3bac9d37
......@@ -66,7 +66,7 @@ def test_profiling_complex_pipeline():
source = [(np.array([x]),) for x in range(1024)]
data1 = ds.GeneratorDataset(source, ["gen"])
data1 = data1.map("gen", operations=[(lambda x: x + 1)])
data1 = data1.map(operations=[(lambda x: x + 1)], input_columns=["gen"])
pattern = DATASET_ROOT + "/test.data"
data2 = ds.TFRecordDataset(pattern, SCHEMA_FILE, shuffle=ds.Shuffle.FILES)
......
tests/ut/python/dataset/test_pyfunc.py
浏览文件 @ 3bac9d37
......@@ -54,7 +54,7 @@ def test_case_1():
# apply dataset operations
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
data1 = data1.map(input_columns=col, output_columns=["out0", "out1"], operations=(lambda x: (x, x + x)),
columns_order=["out0", "out1"])
column_order=["out0", "out1"])
i = 0
for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary
......@@ -78,7 +78,7 @@ def test_case_2():
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
data1 = data1.map(input_columns=col, output_columns="out", operations=(lambda x, y: x + y),
columns_order=["out"])
column_order=["out"])
i = 0
for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary
......@@ -100,7 +100,7 @@ def test_case_3():
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
data1 = data1.map(input_columns=col, output_columns=["out0", "out1", "out2"],
operations=(lambda x, y: (x, x + y, x + y + 1)), columns_order=["out0", "out1", "out2"])
operations=(lambda x, y: (x, x + y, x + y + 1)), column_order=["out0", "out1", "out2"])
i = 0
for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary
......@@ -126,7 +126,7 @@ def test_case_4():
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
data1 = data1.map(input_columns=col, output_columns=["out0", "out1", "out2"], num_parallel_workers=4,
operations=(lambda x, y: (x, x + y, x + y + 1)), columns_order=["out0", "out1", "out2"])
operations=(lambda x, y: (x, x + y, x + y + 1)), column_order=["out0", "out1", "out2"])
i = 0
for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary
......@@ -166,7 +166,7 @@ def test_case_6():
"""
Test PyFunc
"""
logger.info("Test PyFunc ComposeOp : (lambda x : x + x), (lambda x : x + x)")
logger.info("Test PyFunc Compose : (lambda x : x + x), (lambda x : x + x)")
# apply dataset operations
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
......@@ -214,7 +214,7 @@ def test_case_8():
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
data1 = data1.map(input_columns=col, output_columns=["out0", "out1", "out2"], num_parallel_workers=4,
operations=(lambda x, y: (x, x + y, x + y + 1)), columns_order=["out0", "out1", "out2"],
operations=(lambda x, y: (x, x + y, x + y + 1)), column_order=["out0", "out1", "out2"],
python_multiprocessing=True)
i = 0
......
tests/ut/python/dataset/test_random_affine.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing RandomAffine op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as py_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
from util import visualize_list, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -41,20 +42,20 @@ def test_random_affine_op(plot=False):
py_vision.RandomAffine(degrees=15, translate=(0.1, 0.1), scale=(0.9, 1.1)),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
transforms2 = [
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform1())
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform2())
data2 = data2.map(input_columns=["image"], operations=transform2)
image_affine = []
image_original = []
......@@ -114,11 +115,11 @@ def test_random_affine_md5():
scale=(0.9, 1.1), shear=(-10, 10, -5, 5)),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
data = data.map(input_columns=["image"], operations=transform)
# check results with md5 comparison
filename = "random_affine_01_result.npz"
......@@ -189,9 +190,9 @@ def test_random_affine_py_exception_non_pil_images():
logger.info("test_random_affine_exception_negative_degrees")
dataset = ds.MnistDataset(MNIST_DATA_DIR, num_parallel_workers=3)
try:
transform = py_vision.ComposeOp([py_vision.ToTensor(),
py_vision.RandomAffine(degrees=(15, 15))])
dataset = dataset.map(input_columns=["image"], operations=transform(), num_parallel_workers=3,
transform = mindspore.dataset.transforms.py_transforms.Compose([py_vision.ToTensor(),
py_vision.RandomAffine(degrees=(15, 15))])
dataset = dataset.map(input_columns=["image"], operations=transform, num_parallel_workers=3,
python_multiprocessing=True)
for _ in dataset.create_dict_iterator(num_epochs=1):
break
......
tests/ut/python/dataset/test_random_apply.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,8 @@ Testing RandomApply op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import visualize_list, config_get_set_seed, \
config_get_set_num_parallel_workers, save_and_check_md5
......@@ -40,20 +41,20 @@ def test_random_apply_op(plot=False):
py_vision.RandomApply(transforms_list, prob=0.6),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
transforms2 = [
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform1())
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform2())
data2 = data2.map(input_columns=["image"], operations=transform2)
image_apply = []
image_original = []
......@@ -81,11 +82,11 @@ def test_random_apply_md5():
py_vision.RandomApply(transforms_list),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
data = data.map(input_columns=["image"], operations=transform)
# check results with md5 comparison
filename = "random_apply_01_result.npz"
......@@ -113,10 +114,10 @@ def test_random_apply_exception_random_crop_badinput():
py_vision.RandomApply(transforms_list, prob=0.6),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
data = data.map(input_columns=["image"], operations=transform)
try:
_ = data.create_dict_iterator(num_epochs=1).get_next()
except RuntimeError as e:
......
tests/ut/python/dataset/test_random_choice.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,8 @@ Testing RandomChoice op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import visualize_list, diff_mse
......@@ -37,20 +38,20 @@ def test_random_choice_op(plot=False):
py_vision.RandomChoice(transforms_list),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
transforms2 = [
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform1())
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform2())
data2 = data2.map(input_columns=["image"], operations=transform2)
image_choice = []
image_original = []
......@@ -75,21 +76,21 @@ def test_random_choice_comp(plot=False):
py_vision.RandomChoice(transforms_list),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
transforms2 = [
py_vision.Decode(),
py_vision.CenterCrop(64),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform1())
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform2())
data2 = data2.map(input_columns=["image"], operations=transform2)
image_choice = []
image_original = []
......@@ -119,10 +120,10 @@ def test_random_choice_exception_random_crop_badinput():
py_vision.RandomChoice(transforms_list),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
data = data.map(input_columns=["image"], operations=transform)
try:
_ = data.create_dict_iterator(num_epochs=1).get_next()
except RuntimeError as e:
......
tests/ut/python/dataset/test_random_color.py
浏览文件 @ 3bac9d37
......@@ -20,8 +20,9 @@ import pytest
import mindspore.dataset as ds
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as vision
import mindspore.dataset.vision.py_transforms as F
from mindspore import log as logger
from util import visualize_list, diff_mse, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -43,14 +44,14 @@ def test_random_color_py(degrees=(0.1, 1.9), plot=False):
logger.info("Test RandomColor")
# Original Images
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = data.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
......@@ -63,15 +64,15 @@ def test_random_color_py(degrees=(0.1, 1.9), plot=False):
axis=0)
# Random Color Adjusted Images
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_random_color = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.RandomColor(degrees=degrees),
F.ToTensor()])
transforms_random_color = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.RandomColor(degrees=degrees),
F.ToTensor()])
ds_random_color = data.map(input_columns="image",
operations=transforms_random_color())
operations=transforms_random_color)
ds_random_color = ds_random_color.batch(512)
......@@ -146,13 +147,13 @@ def test_random_color_py_md5():
original_num_parallel_workers = config_get_set_num_parallel_workers(1)
# Generate dataset
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms = F.ComposeOp([F.Decode(),
F.RandomColor((2.0, 2.5)),
F.ToTensor()])
transforms = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.RandomColor((2.0, 2.5)),
F.ToTensor()])
data = data.map(input_columns="image", operations=transforms())
data = data.map(input_columns="image", operations=transforms)
# Compare with expected md5 from images
filename = "random_color_01_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -183,12 +184,12 @@ def test_compare_random_color_op(degrees=None, plot=False):
c_op = vision.RandomColor(degrees)
p_op = F.RandomColor(degrees)
transforms_random_color_py = F.ComposeOp([lambda img: img.astype(np.uint8), F.ToPIL(),
p_op, np.array])
transforms_random_color_py = mindspore.dataset.transforms.py_transforms.Compose([lambda img: img.astype(np.uint8), F.ToPIL(),
p_op, np.array])
data1 = data1.map(input_columns=["image"], operations=[vision.Decode(), c_op])
data2 = data2.map(input_columns=["image"], operations=[vision.Decode()])
data2 = data2.map(input_columns=["image"], operations=transforms_random_color_py())
data2 = data2.map(input_columns=["image"], operations=transforms_random_color_py)
image_random_color_op = []
image = []
......@@ -249,7 +250,8 @@ if __name__ == "__main__":
test_random_color_c()
test_random_color_c(plot=True)
test_random_color_c(degrees=(2.0, 2.5), plot=True, run_golden=False) # Test with degree values that show more obvious transformation
test_random_color_c(degrees=(2.0, 2.5), plot=True,
run_golden=False) # Test with degree values that show more obvious transformation
test_random_color_c(degrees=(0.1, 0.1), plot=True, run_golden=False)
test_compare_random_color_op(plot=True)
test_random_color_c_errors()
tests/ut/python/dataset/test_random_color_adjust.py
浏览文件 @ 3bac9d37
......@@ -19,8 +19,9 @@ import pytest
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, visualize_image, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -42,9 +43,9 @@ def util_test_random_color_adjust_error(brightness=(1, 1), contrast=(1, 1), satu
(lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8))
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=transform)
# if input is grayscale, the output dimensions should be single channel, the following should fail
random_adjust_op = c_vision.RandomColorAdjust(brightness=brightness, contrast=contrast, saturation=saturation,
......@@ -86,9 +87,9 @@ def util_test_random_color_adjust_op(brightness=(1, 1), contrast=(1, 1), saturat
hue=hue),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
......@@ -211,9 +212,9 @@ def test_random_color_adjust_md5():
py_vision.RandomColorAdjust(0.4, 0.4, 0.4, 0.1),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "random_color_adjust_01_c_result.npz"
save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN)
......
tests/ut/python/dataset/test_random_crop.py
浏览文件 @ 3bac9d37
......@@ -16,9 +16,11 @@
Testing RandomCrop op in DE
"""
import numpy as np
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.vision.utils as mode
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
import mindspore.dataset.vision.utils as mode
import mindspore.dataset as ds
from mindspore import log as logger
from util import save_and_check_md5, visualize_list, config_get_set_seed, \
......@@ -71,8 +73,8 @@ def test_random_crop_op_py(plot=False):
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1())
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
# Second dataset for comparison
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -80,8 +82,8 @@ def test_random_crop_op_py(plot=False):
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2())
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2)
crop_images = []
original_images = []
......@@ -132,8 +134,8 @@ def test_random_crop_01_py():
py_vision.RandomCrop(512),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
filename = "random_crop_01_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -181,8 +183,8 @@ def test_random_crop_02_py():
py_vision.RandomCrop([512, 375]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
filename = "random_crop_02_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -230,8 +232,8 @@ def test_random_crop_03_py():
py_vision.RandomCrop([2268, 4032]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
filename = "random_crop_03_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -274,8 +276,8 @@ def test_random_crop_04_py():
py_vision.RandomCrop([2268, 4033]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
try:
data.create_dict_iterator(num_epochs=1).get_next()
except RuntimeError as e:
......@@ -325,8 +327,8 @@ def test_random_crop_05_py():
py_vision.RandomCrop([2268, 4033], [200, 200, 200, 200], pad_if_needed=True),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
filename = "random_crop_05_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -370,8 +372,8 @@ def test_random_crop_06_py():
py_vision.RandomCrop([512, 512, 375]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
except TypeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Size should be a single integer" in str(e)
......@@ -419,8 +421,8 @@ def test_random_crop_07_py():
py_vision.RandomCrop(512, [200, 200, 200, 200], fill_value=(255, 255, 255)),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
filename = "random_crop_07_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -470,8 +472,8 @@ def test_random_crop_08_py():
py_vision.RandomCrop(512, [200, 200, 200, 200], padding_mode=mode.Border.EDGE),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
filename = "random_crop_08_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -494,8 +496,8 @@ def test_random_crop_09():
# Note: if input is not PIL image, TypeError will raise
py_vision.RandomCrop(512)
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
try:
data.create_dict_iterator(num_epochs=1).get_next()
except RuntimeError as e:
......@@ -523,8 +525,8 @@ def test_random_crop_comp(plot=False):
py_vision.RandomCrop(cropped_size),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
image_c_cropped = []
image_py_cropped = []
......
tests/ut/python/dataset/test_random_crop_and_resize.py
浏览文件 @ 3bac9d37
......@@ -18,9 +18,10 @@ Testing RandomCropAndResize op in DE
import numpy as np
import cv2
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.vision.utils as mode
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
import mindspore.dataset.vision.utils as mode
import mindspore.dataset as ds
from mindspore import log as logger
from util import diff_mse, save_and_check_md5, visualize_list, \
......@@ -80,8 +81,8 @@ def test_random_crop_and_resize_op_py(plot=False):
py_vision.RandomResizedCrop((256, 512), (2, 2), (1, 3)),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1())
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
# Second dataset for comparison
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -89,8 +90,8 @@ def test_random_crop_and_resize_op_py(plot=False):
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2())
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2)
num_iter = 0
crop_and_resize_images = []
original_images = []
......@@ -131,8 +132,8 @@ def test_random_crop_and_resize_01():
py_vision.RandomResizedCrop((256, 512), (0.5, 0.5), (1, 1)),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
filename1 = "random_crop_and_resize_01_c_result.npz"
filename2 = "random_crop_and_resize_01_py_result.npz"
......@@ -167,8 +168,8 @@ def test_random_crop_and_resize_02():
py_vision.RandomResizedCrop((256, 512), interpolation=mode.Inter.NEAREST),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
filename1 = "random_crop_and_resize_02_c_result.npz"
filename2 = "random_crop_and_resize_02_py_result.npz"
......@@ -202,8 +203,8 @@ def test_random_crop_and_resize_03():
py_vision.RandomResizedCrop((256, 512), max_attempts=1),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
filename1 = "random_crop_and_resize_03_c_result.npz"
filename2 = "random_crop_and_resize_03_py_result.npz"
......@@ -251,8 +252,8 @@ def test_random_crop_and_resize_04_py():
py_vision.RandomResizedCrop((256, 512), (1, 0.5), (0.5, 0.5)),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input is not within the required interval of (0 to 16777216)." in str(e)
......@@ -294,8 +295,8 @@ def test_random_crop_and_resize_05_py():
py_vision.RandomResizedCrop((256, 512), (1, 1), (1, 0.5)),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input is not within the required interval of (0 to 16777216)." in str(e)
......@@ -321,8 +322,8 @@ def test_random_crop_and_resize_comp(plot=False):
py_vision.RandomResizedCrop(512, (1, 1), (0.5, 0.5)),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
image_c_cropped = []
image_py_cropped = []
......
tests/ut/python/dataset/test_random_crop_and_resize_with_bbox.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,7 @@ Testing RandomCropAndResizeWithBBox op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \
......@@ -50,7 +50,7 @@ def test_random_resized_crop_with_bbox_op_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "random_resized_crop_with_bbox_01_c_result.npz"
......@@ -87,7 +87,7 @@ def test_random_resized_crop_with_bbox_op_coco_c(plot_vis=False):
dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
unaugSamp, augSamp = [], []
......@@ -116,13 +116,13 @@ def test_random_resized_crop_with_bbox_op_edge_c(plot_vis=False):
# maps to convert data into valid edge case data
dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[lambda img, bboxes: (img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))])
# Test Op added to list of Operations here
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[lambda img, bboxes: (img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op])
unaugSamp, augSamp = [], []
......@@ -151,7 +151,7 @@ def test_random_resized_crop_with_bbox_op_invalid_c():
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
for _ in dataVoc2.create_dict_iterator(num_epochs=1):
......@@ -177,7 +177,7 @@ def test_random_resized_crop_with_bbox_op_invalid2_c():
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
for _ in dataVoc2.create_dict_iterator(num_epochs=1):
......
tests/ut/python/dataset/test_random_crop_decode_resize.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@
Testing RandomCropDecodeResize op in DE
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from util import diff_mse, visualize_image, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......
tests/ut/python/dataset/test_random_crop_with_bbox.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,8 @@ Testing RandomCropWithBBox op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.utils as mode
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.utils as mode
from mindspore import log as logger
from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \
......@@ -48,7 +48,7 @@ def test_random_crop_with_bbox_op_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op]) # Add column for "bbox"
unaugSamp, augSamp = [], []
......@@ -78,7 +78,7 @@ def test_random_crop_with_bbox_op_coco_c(plot_vis=False):
dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
unaugSamp, augSamp = [], []
......@@ -110,7 +110,7 @@ def test_random_crop_with_bbox_op2_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "random_crop_with_bbox_01_c_result.npz"
......@@ -147,7 +147,7 @@ def test_random_crop_with_bbox_op3_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
unaugSamp, augSamp = [], []
......@@ -177,14 +177,14 @@ def test_random_crop_with_bbox_op_edge_c(plot_vis=False):
# maps to convert data into valid edge case data
dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[lambda img, bboxes: (
img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))])
# Test Op added to list of Operations here
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[lambda img, bboxes: (
img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op])
......@@ -214,7 +214,7 @@ def test_random_crop_with_bbox_op_invalid_c():
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op]) # Add column for "bbox"
for _ in dataVoc2.create_dict_iterator(num_epochs=1):
......@@ -254,7 +254,7 @@ def test_random_crop_with_bbox_op_bad_padding():
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
for _ in dataVoc2.create_dict_iterator(num_epochs=1):
......@@ -268,7 +268,7 @@ def test_random_crop_with_bbox_op_bad_padding():
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
for _ in dataVoc2.create_dict_iterator(num_epochs=1):
......
tests/ut/python/dataset/test_random_erasing.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,8 @@ Testing RandomErasing op in DE
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as vision
from mindspore import log as logger
from util import diff_mse, visualize_image, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -41,8 +42,8 @@ def test_random_erasing_op(plot=False):
vision.ToTensor(),
vision.RandomErasing(value='random')
]
transform_1 = vision.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -51,8 +52,8 @@ def test_random_erasing_op(plot=False):
vision.ToTensor(),
vision.Cutout(80)
]
transform_2 = vision.ComposeOp(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2())
transform_2 = mindspore.dataset.transforms.py_transforms.Compose(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
......@@ -86,8 +87,8 @@ def test_random_erasing_md5():
vision.ToTensor(),
vision.RandomErasing(value='random')
]
transform_1 = vision.ComposeOp(transforms_1)
data = data.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data = data.map(input_columns=["image"], operations=transform_1)
# Compare with expected md5 from images
filename = "random_erasing_01_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......
tests/ut/python/dataset/test_random_grayscale.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,9 @@
Testing RandomGrayscale op in DE
"""
import numpy as np
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as py_vision
import mindspore.dataset as ds
from mindspore import log as logger
from util import save_and_check_md5, visualize_list, \
......@@ -41,8 +43,8 @@ def test_random_grayscale_valid_prob(plot=False):
py_vision.RandomGrayscale(1),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1())
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -50,8 +52,8 @@ def test_random_grayscale_valid_prob(plot=False):
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2())
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2)
image_gray = []
image = []
......@@ -80,8 +82,8 @@ def test_random_grayscale_input_grayscale_images():
py_vision.RandomGrayscale(0.5),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1())
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -89,8 +91,8 @@ def test_random_grayscale_input_grayscale_images():
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2())
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2)
image_gray = []
image = []
......@@ -124,8 +126,8 @@ def test_random_grayscale_md5_valid_input():
py_vision.RandomGrayscale(0.8),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
# Check output images with md5 comparison
filename = "random_grayscale_01_result.npz"
......@@ -150,8 +152,8 @@ def test_random_grayscale_md5_no_param():
py_vision.RandomGrayscale(),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
# Check output images with md5 comparison
filename = "random_grayscale_02_result.npz"
......@@ -175,8 +177,8 @@ def test_random_grayscale_invalid_param():
py_vision.RandomGrayscale(1.5),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input prob is not within the required interval of (0.0 to 1.0)." in str(e)
......
tests/ut/python/dataset/test_random_horizontal_flip.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing the random horizontal flip op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import save_and_check_md5, visualize_list, visualize_image, diff_mse, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -114,8 +115,8 @@ def test_random_horizontal_valid_prob_py():
py_vision.RandomHorizontalFlip(0.8),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
filename = "random_horizontal_01_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -160,8 +161,8 @@ def test_random_horizontal_invalid_prob_py():
py_vision.RandomHorizontalFlip(1.5),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Input prob is not within the required interval of (0.0 to 1.0)." in str(e)
......@@ -188,8 +189,8 @@ def test_random_horizontal_comp(plot=False):
py_vision.RandomHorizontalFlip(1),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
images_list_c = []
images_list_py = []
......
tests/ut/python/dataset/test_random_horizontal_flip_with_bbox.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ Testing the random horizontal flip with bounding boxes op in DE
import numpy as np
import mindspore.log as logger
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \
config_get_set_seed, config_get_set_num_parallel_workers, save_and_check_md5
......@@ -47,7 +47,7 @@ def test_random_horizontal_flip_with_bbox_op_c(plot_vis=False):
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
unaugSamp, augSamp = [], []
......@@ -77,7 +77,7 @@ def test_random_horizontal_flip_with_bbox_op_coco_c(plot_vis=False):
dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
unaugSamp, augSamp = [], []
......@@ -113,7 +113,7 @@ def test_random_horizontal_flip_with_bbox_valid_rand_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "random_horizontal_flip_with_bbox_01_c_result.npz"
......@@ -149,17 +149,17 @@ def test_random_horizontal_flip_with_bbox_valid_edge_c(plot_vis=False):
# Add column for "bbox"
dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=lambda img, bbox:
(img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32)))
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=lambda img, bbox:
(img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32)))
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
unaugSamp, augSamp = [], []
......@@ -186,7 +186,7 @@ def test_random_horizontal_flip_with_bbox_invalid_prob_c():
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op]) # Add column for "bbox"
except ValueError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
......
tests/ut/python/dataset/test_random_order.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,8 @@ Testing RandomOrder op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import visualize_list, config_get_set_seed, \
config_get_set_num_parallel_workers, save_and_check_md5
......@@ -40,20 +41,20 @@ def test_random_order_op(plot=False):
py_vision.RandomOrder(transforms_list),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
transforms2 = [
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform1())
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform2())
data2 = data2.map(input_columns=["image"], operations=transform2)
image_order = []
image_original = []
......@@ -80,11 +81,11 @@ def test_random_order_md5():
py_vision.RandomOrder(transforms_list),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
data = data.map(input_columns=["image"], operations=transform)
# check results with md5 comparison
filename = "random_order_01_result.npz"
......
tests/ut/python/dataset/test_random_perspective.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing RandomPerspective op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as py_vision
from mindspore.dataset.transforms.vision.utils import Inter
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore.dataset.vision.utils import Inter
from mindspore import log as logger
from util import visualize_list, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -41,20 +42,20 @@ def test_random_perspective_op(plot=False):
py_vision.RandomPerspective(),
py_vision.ToTensor()
]
transform1 = py_vision.ComposeOp(transforms1)
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
transforms2 = [
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform1())
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform2())
data2 = data2.map(input_columns=["image"], operations=transform2)
image_perspective = []
image_original = []
......@@ -83,11 +84,11 @@ def skip_test_random_perspective_md5():
py_vision.Resize(1450), # resize to a smaller size to prevent round-off error
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# Generate dataset
data = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
data = data.map(input_columns=["image"], operations=transform)
# check results with md5 comparison
filename = "random_perspective_01_result.npz"
......
tests/ut/python/dataset/test_random_posterize.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,7 @@ Testing RandomPosterize op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
from util import visualize_list, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers, diff_mse
......
tests/ut/python/dataset/test_random_resize.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@
Testing RandomResize op in DE
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from util import visualize_list, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......
tests/ut/python/dataset/test_random_resize_with_bbox.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,7 @@ Testing the random resize with bounding boxes op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \
......@@ -50,7 +50,7 @@ def test_random_resize_with_bbox_op_voc_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "random_resize_with_bbox_op_01_c_voc_result.npz"
......@@ -93,7 +93,7 @@ def test_random_resize_with_bbox_op_rand_coco_c(plot_vis=False):
dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "random_resize_with_bbox_op_01_c_coco_result.npz"
......@@ -131,13 +131,13 @@ def test_random_resize_with_bbox_op_edge_c(plot_vis=False):
# maps to convert data into valid edge case data
dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[lambda img, bboxes: (
img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))])
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[lambda img, bboxes: (
img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op])
......
tests/ut/python/dataset/test_random_rotation.py
浏览文件 @ 3bac9d37
......@@ -19,9 +19,10 @@ import numpy as np
import cv2
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
from mindspore.dataset.transforms.vision.utils import Inter
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore.dataset.vision.utils import Inter
from mindspore import log as logger
from util import visualize_image, visualize_list, diff_mse, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -31,6 +32,7 @@ SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
GENERATE_GOLDEN = False
def test_random_rotation_op_c(plot=False):
"""
Test RandomRotation in c++ transformations op
......@@ -74,16 +76,16 @@ def test_random_rotation_op_py(plot=False):
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
# use [90, 90] to force rotate 90 degrees, expand is set to be True to match output size
transform1 = py_vision.ComposeOp([py_vision.Decode(),
py_vision.RandomRotation((90, 90), expand=True),
py_vision.ToTensor()])
data1 = data1.map(input_columns=["image"], operations=transform1())
transform1 = mindspore.dataset.transforms.py_transforms.Compose([py_vision.Decode(),
py_vision.RandomRotation((90, 90), expand=True),
py_vision.ToTensor()])
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
transform2 = py_vision.ComposeOp([py_vision.Decode(),
py_vision.ToTensor()])
data2 = data2.map(input_columns=["image"], operations=transform2())
transform2 = mindspore.dataset.transforms.py_transforms.Compose([py_vision.Decode(),
py_vision.ToTensor()])
data2 = data2.map(input_columns=["image"], operations=transform2)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
......@@ -143,14 +145,14 @@ def test_random_rotation_md5():
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, shuffle=False)
transform2 = py_vision.ComposeOp([py_vision.Decode(),
py_vision.RandomRotation((0, 90),
expand=True,
resample=Inter.BILINEAR,
center=(50, 50),
fill_value=150),
py_vision.ToTensor()])
data2 = data2.map(input_columns=["image"], operations=transform2())
transform2 = mindspore.dataset.transforms.py_transforms.Compose([py_vision.Decode(),
py_vision.RandomRotation((0, 90),
expand=True,
resample=Inter.BILINEAR,
center=(50, 50),
fill_value=150),
py_vision.ToTensor()])
data2 = data2.map(input_columns=["image"], operations=transform2)
# Compare with expected md5 from images
filename1 = "random_rotation_01_c_result.npz"
......@@ -186,9 +188,9 @@ def test_rotation_diff(plot=False):
py_vision.RandomRotation((45, 45)),
py_vision.ToTensor(),
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
num_iter = 0
image_list_c, image_list_py = [], []
......@@ -206,7 +208,7 @@ def test_rotation_diff(plot=False):
logger.info("dtype of py_image: {}".format(py_image.dtype))
mse = diff_mse(c_image, py_image)
assert mse < 0.001 # Rounding error
assert mse < 0.001 # Rounding error
if plot:
visualize_list(image_list_c, image_list_py, visualize_mode=2)
......
tests/ut/python/dataset/test_random_select_subpolicy.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as ops
import mindspore.dataset.transforms.vision.c_transforms as visions
import mindspore.dataset.vision.c_transforms as visions
def test_random_select_subpolicy():
......
tests/ut/python/dataset/test_random_sharpness.py
浏览文件 @ 3bac9d37
......@@ -18,8 +18,9 @@ Testing RandomSharpness op in DE
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as F
import mindspore.dataset.vision.c_transforms as C
from mindspore import log as logger
from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -37,14 +38,14 @@ def test_random_sharpness_py(degrees=(0.7, 0.7), plot=False):
logger.info("Test RandomSharpness python op")
# Original Images
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = data.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
......@@ -57,19 +58,19 @@ def test_random_sharpness_py(degrees=(0.7, 0.7), plot=False):
axis=0)
# Random Sharpness Adjusted Images
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
py_op = F.RandomSharpness()
if degrees is not None:
py_op = F.RandomSharpness(degrees)
transforms_random_sharpness = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
py_op,
F.ToTensor()])
transforms_random_sharpness = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
py_op,
F.ToTensor()])
ds_random_sharpness = data.map(input_columns="image",
operations=transforms_random_sharpness())
operations=transforms_random_sharpness)
ds_random_sharpness = ds_random_sharpness.batch(512)
......@@ -106,11 +107,11 @@ def test_random_sharpness_py_md5():
F.RandomSharpness((20.0, 25.0)),
F.ToTensor()
]
transform = F.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# Generate dataset
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = data.map(input_columns=["image"], operations=transform())
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
data = data.map(input_columns=["image"], operations=transform)
# check results with md5 comparison
filename = "random_sharpness_py_01_result.npz"
......@@ -129,7 +130,7 @@ def test_random_sharpness_c(degrees=(1.6, 1.6), plot=False):
logger.info("Test RandomSharpness cpp op")
# Original Images
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = [C.Decode(),
C.Resize((224, 224))]
......@@ -148,7 +149,7 @@ def test_random_sharpness_c(degrees=(1.6, 1.6), plot=False):
axis=0)
# Random Sharpness Adjusted Images
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
c_op = C.RandomSharpness()
if degrees is not None:
......@@ -197,7 +198,7 @@ def test_random_sharpness_c_md5():
]
# Generate dataset
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
data = data.map(input_columns=["image"], operations=transforms)
# check results with md5 comparison
......@@ -216,7 +217,7 @@ def test_random_sharpness_c_py(degrees=(1.0, 1.0), plot=False):
logger.info("Test RandomSharpness C and python Op")
# RandomSharpness Images
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
data = data.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((200, 300))])
......@@ -224,9 +225,9 @@ def test_random_sharpness_c_py(degrees=(1.0, 1.0), plot=False):
python_op = F.RandomSharpness(degrees)
c_op = C.RandomSharpness(degrees)
transforms_op = F.ComposeOp([lambda img: F.ToPIL()(img.astype(np.uint8)),
python_op,
np.array])()
transforms_op = mindspore.dataset.transforms.py_transforms.Compose([lambda img: F.ToPIL()(img.astype(np.uint8)),
python_op,
np.array])
ds_random_sharpness_py = data.map(input_columns="image",
operations=transforms_op)
......@@ -242,7 +243,7 @@ def test_random_sharpness_c_py(degrees=(1.0, 1.0), plot=False):
image,
axis=0)
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
data = data.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((200, 300))])
......@@ -305,7 +306,7 @@ def test_random_sharpness_invalid_params():
"""
logger.info("Test RandomSharpness with invalid input parameters.")
try:
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
data = data.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -315,7 +316,7 @@ def test_random_sharpness_invalid_params():
assert "tuple" in str(error)
try:
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
data = data.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -325,7 +326,7 @@ def test_random_sharpness_invalid_params():
assert "interval" in str(error)
try:
data = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
data = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
data = data.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......
tests/ut/python/dataset/test_random_solarize_op.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ Testing RandomSolarizeOp op in DE
import pytest
import mindspore.dataset as ds
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from util import visualize_list, save_and_check_md5, config_get_set_seed, config_get_set_num_parallel_workers, \
visualize_one_channel_dataset
......
tests/ut/python/dataset/test_random_vertical_flip.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing the random vertical flip op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import save_and_check_md5, visualize_list, visualize_image, diff_mse, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -114,8 +115,8 @@ def test_random_vertical_valid_prob_py():
py_vision.RandomVerticalFlip(0.8),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
filename = "random_vertical_01_py_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -159,8 +160,8 @@ def test_random_vertical_invalid_prob_py():
py_vision.RandomVerticalFlip(1.5),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert 'Input prob is not within the required interval of (0.0 to 1.0).' in str(e)
......@@ -188,8 +189,8 @@ def test_random_vertical_comp(plot=False):
py_vision.RandomVerticalFlip(1),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
images_list_c = []
images_list_py = []
......
tests/ut/python/dataset/test_random_vertical_flip_with_bbox.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,7 @@ Testing RandomVerticalFlipWithBBox op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \
......@@ -48,7 +48,7 @@ def test_random_vertical_flip_with_bbox_op_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
unaugSamp, augSamp = [], []
......@@ -77,7 +77,7 @@ def test_random_vertical_flip_with_bbox_op_coco_c(plot_vis=False):
dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
test_op = c_vision.RandomVerticalFlipWithBBox(1)
......@@ -113,7 +113,7 @@ def test_random_vertical_flip_with_bbox_op_rand_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "random_vertical_flip_with_bbox_01_c_result.npz"
......@@ -150,13 +150,13 @@ def test_random_vertical_flip_with_bbox_op_edge_c(plot_vis=False):
# maps to convert data into valid edge case data
dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[lambda img, bboxes: (img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))])
# Test Op added to list of Operations here
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[lambda img, bboxes: (img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op])
unaugSamp, augSamp = [], []
......@@ -183,7 +183,7 @@ def test_random_vertical_flip_with_bbox_op_invalid_c():
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
for _ in dataVoc2.create_dict_iterator(num_epochs=1):
......
tests/ut/python/dataset/test_repeat.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ Test Repeat Op
import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from util import save_and_check_dict
......
tests/ut/python/dataset/test_rescale_op.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@
Testing the rescale op in DE
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from util import visualize_image, diff_mse, save_and_check_md5
......
tests/ut/python/dataset/test_resize.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,8 @@ Testing Resize op in DE
"""
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
from mindspore.dataset.transforms.vision.utils import Inter
import mindspore.dataset.vision.c_transforms as vision
from mindspore.dataset.vision.utils import Inter
from mindspore import log as logger
from util import visualize_list, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......
tests/ut/python/dataset/test_resize_with_bbox.py
浏览文件 @ 3bac9d37
......@@ -19,7 +19,7 @@ import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \
save_and_check_md5
......@@ -50,7 +50,7 @@ def test_resize_with_bbox_op_voc_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "resize_with_bbox_op_01_c_voc_result.npz"
......@@ -87,7 +87,7 @@ def test_resize_with_bbox_op_coco_c(plot_vis=False):
dataCOCO2 = dataCOCO2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "resize_with_bbox_op_01_c_coco_result.npz"
......@@ -121,13 +121,13 @@ def test_resize_with_bbox_op_edge_c(plot_vis=False):
# maps to convert data into valid edge case data
dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[lambda img, bboxes: (
img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype))])
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[lambda img, bboxes: (
img, np.array([[0, 0, img.shape[1], img.shape[0]]]).astype(bboxes.dtype)), test_op])
......
tests/ut/python/dataset/test_rgb_hsv.py
浏览文件 @ 3bac9d37
......@@ -21,8 +21,9 @@ import numpy as np
from numpy.testing import assert_allclose
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as vision
import mindspore.dataset.transforms.vision.py_transforms_util as util
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as vision
import mindspore.dataset.vision.py_transforms_util as util
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
......@@ -137,9 +138,9 @@ def test_rgb_hsv_pipeline():
vision.Resize([64, 64]),
vision.ToTensor()
]
transforms1 = vision.ComposeOp(transforms1)
transforms1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
ds1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
ds1 = ds1.map(input_columns=["image"], operations=transforms1())
ds1 = ds1.map(input_columns=["image"], operations=transforms1)
# Second dataset
transforms2 = [
......@@ -149,9 +150,9 @@ def test_rgb_hsv_pipeline():
vision.RgbToHsv(),
vision.HsvToRgb()
]
transform2 = vision.ComposeOp(transforms2)
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
ds2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
ds2 = ds2.map(input_columns=["image"], operations=transform2())
ds2 = ds2.map(input_columns=["image"], operations=transform2)
num_iter = 0
for data1, data2 in zip(ds1.create_dict_iterator(num_epochs=1), ds2.create_dict_iterator(num_epochs=1)):
......
tests/ut/python/dataset/test_serdes_dataset.py
浏览文件 @ 3bac9d37
......@@ -26,9 +26,9 @@ from util import config_get_set_num_parallel_workers
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as c
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
......@@ -47,7 +47,7 @@ def test_imagefolder(remove_json_files=True):
# Constructing DE pipeline
sampler = ds.WeightedRandomSampler(weights, 11)
data1 = ds.ImageFolderDatasetV2(data_dir, sampler=sampler)
data1 = ds.ImageFolderDataset(data_dir, sampler=sampler)
data1 = data1.repeat(1)
data1 = data1.map(input_columns=["image"], operations=[vision.Decode(True)])
rescale_op = vision.Rescale(rescale, shift)
......
tests/ut/python/dataset/test_skip.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@ import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
......
tests/ut/python/dataset/test_soft_dvpp.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@
Testing soft dvpp SoftDvppDecodeResizeJpeg and SoftDvppDecodeRandomCropResizeJpeg in DE
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from util import diff_mse, visualize_image
......
tests/ut/python/dataset/test_ten_crop.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,8 @@ import pytest
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as vision
from mindspore import log as logger
from util import visualize_list, save_and_check_md5
......@@ -37,8 +38,8 @@ def util_test_ten_crop(crop_size, vertical_flip=False, plot=False):
vision.Decode(),
vision.ToTensor(),
]
transform_1 = vision.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -47,8 +48,8 @@ def util_test_ten_crop(crop_size, vertical_flip=False, plot=False):
vision.TenCrop(crop_size, use_vertical_flip=vertical_flip),
lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 10 images
]
transform_2 = vision.ComposeOp(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2())
transform_2 = mindspore.dataset.transforms.py_transforms.Compose(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
num_iter += 1
......@@ -109,8 +110,8 @@ def test_ten_crop_md5():
vision.TenCrop((200, 100), use_vertical_flip=True),
lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 10 images
]
transform_2 = vision.ComposeOp(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2())
transform_2 = mindspore.dataset.transforms.py_transforms.Compose(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2)
# Compare with expected md5 from images
filename = "ten_crop_01_result.npz"
save_and_check_md5(data2, filename, generate_golden=GENERATE_GOLDEN)
......@@ -169,8 +170,8 @@ def test_ten_crop_wrong_img_error_msg():
vision.TenCrop(200),
vision.ToTensor()
]
transform = vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
with pytest.raises(RuntimeError) as info:
data.create_tuple_iterator(num_epochs=1).get_next()
......
tests/ut/python/dataset/test_text_basic_tokenizer.py
浏览文件 @ 3bac9d37
......@@ -106,7 +106,7 @@ def check_basic_tokenizer_with_offsets(first, last, expected_tokens, expected_of
with_offsets=True)
dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'],
columns_order=['token', 'offsets_start', 'offsets_limit'], operations=basic_tokenizer)
column_order=['token', 'offsets_start', 'offsets_limit'], operations=basic_tokenizer)
count = 0
for i in dataset.create_dict_iterator(num_epochs=1):
token = text.to_str(i['token'])
......
tests/ut/python/dataset/test_text_bert_tokenizer.py
浏览文件 @ 3bac9d37
......@@ -213,7 +213,7 @@ def check_bert_tokenizer_with_offsets(first, last, expect_str,
unknown_token=unknown_token, lower_case=lower_case, keep_whitespace=keep_whitespace,
normalization_form=normalization_form, preserve_unused_token=preserve_unused_token, with_offsets=True)
dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'],
columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op)
column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op)
count = 0
for i in dataset.create_dict_iterator(num_epochs=1):
token = text.to_str(i['token'])
......
tests/ut/python/dataset/test_text_jieba_tokenizer.py
浏览文件 @ 3bac9d37
......@@ -202,7 +202,7 @@ def test_jieba_with_offsets_1():
data = ds.TextFileDataset(DATA_FILE)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
columns_order=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
operations=jieba_op, num_parallel_workers=1)
expect = ['今天天气', '太好了', '我们', '一起', '去', '外面', '玩吧']
expected_offsets_start = [0, 12, 21, 27, 33, 36, 42]
......@@ -223,7 +223,7 @@ def test_jieba_with_offsets_1_1():
data = ds.TextFileDataset(DATA_FILE)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.HMM, with_offsets=True)
data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
columns_order=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
operations=jieba_op, num_parallel_workers=1)
expect = ['今天', '天气', '太', '好', '了', '我们', '一起', '去', '外面', '玩', '吧']
expected_offsets_start = [0, 6, 12, 15, 18, 21, 27, 33, 36, 42, 45]
......@@ -243,7 +243,7 @@ def test_jieba_with_offsets_1_2():
data = ds.TextFileDataset(DATA_FILE)
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MIX, with_offsets=True)
data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
columns_order=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
operations=jieba_op, num_parallel_workers=1)
expect = ['今天天气', '太好了', '我们', '一起', '去', '外面', '玩吧']
expected_offsets_start = [0, 12, 21, 27, 33, 36, 42]
......@@ -266,7 +266,7 @@ def test_jieba_with_offsets_2():
jieba_op.add_word("男默女泪")
expect = ['男默女泪', '市', '长江大桥']
data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
columns_order=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
operations=jieba_op, num_parallel_workers=2)
expected_offsets_start = [0, 12, 15]
expected_offsets_limit = [12, 15, 27]
......@@ -287,7 +287,7 @@ def test_jieba_with_offsets_2_1():
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_word("男默女泪", 10)
data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
columns_order=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
operations=jieba_op, num_parallel_workers=2)
expect = ['男默女泪', '市', '长江大桥']
expected_offsets_start = [0, 12, 15]
......@@ -309,7 +309,7 @@ def test_jieba_with_offsets_2_2():
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_word("江大桥", 20000)
data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
columns_order=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
operations=jieba_op, num_parallel_workers=2)
expect = ['江州', '市长', '江大桥', '参加', '了', '长江大桥', '的', '通车', '仪式']
expected_offsets_start = [0, 6, 12, 21, 27, 30, 42, 45, 51]
......@@ -334,7 +334,7 @@ def test_jieba_with_offsets_3():
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_dict(user_dict)
data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
columns_order=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
operations=jieba_op, num_parallel_workers=1)
expect = ['男默女泪', '市', '长江大桥']
expected_offsets_start = [0, 12, 15]
......@@ -360,7 +360,7 @@ def test_jieba_with_offsets_3_1():
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_dict(user_dict)
data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
columns_order=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
operations=jieba_op, num_parallel_workers=1)
expect = ['男默女泪', '市长', '江大桥']
expected_offsets_start = [0, 12, 18]
......@@ -383,7 +383,7 @@ def test_jieba_with_offsets_4():
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_dict(DICT_FILE)
data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
columns_order=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
operations=jieba_op, num_parallel_workers=1)
expect = ['今天天气', '太好了', '我们', '一起', '去', '外面', '玩吧']
expected_offsets_start = [0, 12, 21, 27, 33, 36, 42]
......@@ -406,7 +406,7 @@ def test_jieba_with_offsets_5():
jieba_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
jieba_op.add_word("江大桥", 20000)
data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
columns_order=["token", "offsets_start", "offsets_limit"],
column_order=["token", "offsets_start", "offsets_limit"],
operations=jieba_op, num_parallel_workers=1)
expect = ['江州', '市长', '江大桥', '参加', '了', '长江大桥', '的', '通车', '仪式']
expected_offsets_start = [0, 6, 12, 21, 27, 30, 42, 45, 51]
......
tests/ut/python/dataset/test_text_tokenizer.py
浏览文件 @ 3bac9d37
......@@ -60,7 +60,7 @@ def test_unicode_char_tokenizer_with_offsets():
dataset = ds.TextFileDataset(DATA_FILE, shuffle=False)
tokenizer = text.UnicodeCharTokenizer(with_offsets=True)
dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'],
columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer)
column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer)
tokens = []
expected_offsets_start = [[0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18],
[0, 3, 6, 9, 12, 15], [0, 3, 6, 9, 10, 11, 12, 13, 14, 15, 16], [0, 1]]
......@@ -107,7 +107,7 @@ def test_whitespace_tokenizer_with_offsets():
dataset = ds.TextFileDataset(DATA_FILE, shuffle=False)
tokenizer = text.WhitespaceTokenizer(with_offsets=True)
dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'],
columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer)
column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer)
tokens = []
expected_offsets_start = [[0, 8, 11], [0], [0], [0]]
expected_offsets_limit = [[7, 10, 19], [18], [17], [0]]
......@@ -173,7 +173,7 @@ def test_unicode_script_tokenizer_with_offsets():
dataset = ds.TextFileDataset(DATA_FILE, shuffle=False)
tokenizer = text.UnicodeScriptTokenizer(keep_whitespace=False, with_offsets=True)
dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'],
columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer)
column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer)
tokens = []
expected_offsets_start = [[0, 8, 11, 18], [0, 15], [0, 9, 16], [0]]
expected_offsets_limit = [[7, 10, 18, 19], [15, 18], [9, 16, 17], [0]]
......@@ -199,7 +199,7 @@ def test_unicode_script_tokenizer_with_offsets2():
dataset = ds.TextFileDataset(DATA_FILE, shuffle=False)
tokenizer = text.UnicodeScriptTokenizer(keep_whitespace=True, with_offsets=True)
dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'],
columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer)
column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer)
tokens = []
expected_offsets_start = [[0, 7, 8, 10, 11, 18], [0, 15], [0, 9, 16], [0]]
expected_offsets_limit = [[7, 8, 10, 11, 18, 19], [15, 18], [9, 16, 17], [2]]
......@@ -340,7 +340,7 @@ def test_regex_tokenizer_with_offsets():
dataset = dataset.take(last - first + 1)
tokenizer_op = text.RegexTokenizer(delim_pattern, keep_delim_pattern, with_offsets=True)
dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'],
columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op)
column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op)
out_text = []
count = 0
for i in dataset.create_dict_iterator(num_epochs=1):
......
tests/ut/python/dataset/test_text_wordpiece_tokenizer.py
浏览文件 @ 3bac9d37
......@@ -127,7 +127,7 @@ def check_wordpiece_tokenizer_with_offsets(first, last, expect_str, expected_off
tokenizer_op = text.WordpieceTokenizer(vocab=vocab, with_offsets=True, unknown_token=unknown_token,
max_bytes_per_token=max_bytes_per_token)
dataset = dataset.map(input_columns=['text'], output_columns=['token', 'offsets_start', 'offsets_limit'],
columns_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op)
column_order=['token', 'offsets_start', 'offsets_limit'], operations=tokenizer_op)
count = 0
for i in dataset.create_dict_iterator(num_epochs=1):
token = text.to_str(i['token'])
......
tests/ut/python/dataset/test_to_pil.py
浏览文件 @ 3bac9d37
......@@ -16,8 +16,9 @@
Testing ToPIL op in DE
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import save_and_check_md5
......@@ -43,8 +44,8 @@ def test_to_pil_01():
py_vision.CenterCrop(375),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "to_pil_01_result.npz"
......@@ -66,9 +67,9 @@ def test_to_pil_02():
py_vision.CenterCrop(375),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=decode_op)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "to_pil_02_result.npz"
......
tests/ut/python/dataset/test_to_type.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,8 @@ Testing ToType op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import save_and_check_md5
......@@ -41,8 +42,8 @@ def test_to_type_op():
# Note: Convert the datatype from float32 to int16
py_vision.ToType(np.int16)
]
transform1 = py_vision.ComposeOp(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1())
transform1 = mindspore.dataset.transforms.py_transforms.Compose(transforms1)
data1 = data1.map(input_columns=["image"], operations=transform1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -50,8 +51,8 @@ def test_to_type_op():
py_vision.Decode(),
py_vision.ToTensor()
]
transform2 = py_vision.ComposeOp(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2())
transform2 = mindspore.dataset.transforms.py_transforms.Compose(transforms2)
data2 = data2.map(input_columns=["image"], operations=transform2)
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
image1 = item1["image"]
......@@ -78,8 +79,8 @@ def test_to_type_01():
# Note: Convert the datatype from float32 to int32
py_vision.ToType(np.int32)
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "to_type_01_result.npz"
......@@ -99,8 +100,8 @@ def test_to_type_02():
# Note: Convert to type int
py_vision.ToType('int')
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "to_type_02_result.npz"
......@@ -121,8 +122,8 @@ def test_to_type_03():
# Note: If the object is not numpy, e.g. PIL image, TypeError will raise
py_vision.ToType(np.int32)
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
except Exception as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Numpy" in str(e)
......@@ -143,8 +144,8 @@ def test_to_type_04():
# Note: if output_type is not explicitly given
py_vision.ToType()
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
except Exception as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "missing" in str(e)
......@@ -165,8 +166,8 @@ def test_to_type_05():
# Note: if output_type is not explicitly given
py_vision.ToType('invalid')
]
transform = py_vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
except Exception as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "data type" in str(e)
......
tests/ut/python/dataset/test_type_cast.py
浏览文件 @ 3bac9d37
......@@ -20,8 +20,9 @@ import numpy as np
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as data_util
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
......@@ -50,9 +51,9 @@ def test_type_cast():
transforms = [py_vision.Decode(),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
......@@ -90,9 +91,9 @@ def test_type_cast_string():
transforms = [py_vision.Decode(),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
......
tests/ut/python/dataset/test_uniform_augment.py
浏览文件 @ 3bac9d37
......@@ -19,8 +19,9 @@ import numpy as np
import pytest
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as C
import mindspore.dataset.vision.py_transforms as F
from mindspore import log as logger
from util import visualize_list, diff_mse
......@@ -34,14 +35,14 @@ def test_uniform_augment(plot=False, num_ops=2):
logger.info("Test UniformAugment")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = ds.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
......@@ -54,7 +55,7 @@ def test_uniform_augment(plot=False, num_ops=2):
axis=0)
# UniformAugment Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transform_list = [F.RandomRotation(45),
F.RandomColor(),
......@@ -63,13 +64,15 @@ def test_uniform_augment(plot=False, num_ops=2):
F.AutoContrast(),
F.Equalize()]
transforms_ua = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.UniformAugment(transforms=transform_list, num_ops=num_ops),
F.ToTensor()])
transforms_ua = \
mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.UniformAugment(transforms=transform_list,
num_ops=num_ops),
F.ToTensor()])
ds_ua = ds.map(input_columns="image",
operations=transforms_ua())
operations=transforms_ua)
ds_ua = ds_ua.batch(512)
......@@ -98,7 +101,7 @@ def test_cpp_uniform_augment(plot=False, num_ops=2):
logger.info("Test CPP UniformAugment")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = [C.Decode(), C.Resize(size=[224, 224]),
F.ToTensor()]
......@@ -117,7 +120,7 @@ def test_cpp_uniform_augment(plot=False, num_ops=2):
axis=0)
# UniformAugment Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_ua = [C.RandomCrop(size=[224, 224], padding=[32, 32, 32, 32]),
C.RandomHorizontalFlip(),
C.RandomVerticalFlip(),
......@@ -170,7 +173,7 @@ def test_cpp_uniform_augment_exception_pyops(num_ops=2):
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Argument tensor_ops[5] with value" \
" <mindspore.dataset.transforms.vision.py_transforms.Invert" in str(e.value)
" <mindspore.dataset.vision.py_transforms.Invert" in str(e.value)
assert "is not of type (<class 'mindspore._c_dataengine.TensorOp'>,)" in str(e.value)
......
tests/ut/python/dataset/test_var_batch_map.py
浏览文件 @ 3bac9d37
......@@ -273,7 +273,7 @@ def test_var_batch_var_resize():
def add_one(batchInfo):
return batchInfo.get_batch_num() + 1
data1 = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/", num_parallel_workers=4, decode=True)
data1 = ds.ImageFolderDataset("../data/dataset/testPK/data/", num_parallel_workers=4, decode=True)
data1 = data1.batch(batch_size=add_one, drop_remainder=True, input_columns=["image"], per_batch_map=np_psedo_resize)
# i-th batch has shape [i, i^2, i^2, 3]
i = 1
......
tests/ut/python/dataset/util.py
浏览文件 @ 3bac9d37
......@@ -386,12 +386,12 @@ def check_bad_bbox(data, test_op, invalid_bbox_type, expected_error):
# map to use selected invalid bounding box type
data = data.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=lambda img, bboxes: add_bad_bbox(img, bboxes, invalid_bbox_type))
# map to apply ops
data = data.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op]) # Add column for "bbox"
for _, _ in enumerate(data.create_dict_iterator(num_epochs=1)):
break
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或