Skip to content

M
mindspore

magicwindyyd / mindspore
与 Fork 源项目一致

Fork自 MindSpore / mindspore

通知 1
Star 1
Fork 0
M
mindspore
提交 3bac9d37 编写于 作者: N nhussain
浏览文件
操作

switch input columns and operation 

change ImagefolderDV2 name

change ds.transforms.vision to ds.vision

change batch api to match map api more closely

compose op changes

test_pylint

remove compose op from vision, move to transform module, refactor map and batch to use column_order
上级 75045e3e
展开全部 隐藏空白更改
内联 并排
Showing with 1339 addition and 1107 deletion
mindspore/ccsrc/minddata/dataset/api/python/de_pipeline.cc
浏览文件 @ 3bac9d37
......@@ -733,7 +733,7 @@ Status DEPipeline::ParseMapOp(const py::dict &args, std::shared_ptr<DatasetOp> *
(void)map_builder.SetInColNames(in_col_names);
} else if (key == "output_columns") {
(void)map_builder.SetOutColNames(ToStringVector(value));
} else if (key == "columns_order") {
} else if (key == "column_order") {
project_columns = ToStringVector(value);
} else if (key == "num_parallel_workers") {
num_workers = ToInt(value);
......
mindspore/ccsrc/minddata/dataset/engine/datasetops/source/image_folder_op.cc
浏览文件 @ 3bac9d37
......@@ -113,7 +113,7 @@ Status ImageFolderOp::PrescanMasterEntry(const std::string &filedir) {
num_rows_ = image_label_pairs_.size();
if (num_rows_ == 0) {
RETURN_STATUS_UNEXPECTED(
"There is no valid data matching the dataset API ImageFolderDatasetV2.Please check file path or dataset "
"There is no valid data matching the dataset API ImageFolderDataset. Please check file path or dataset "
"API validation first.");
}
// free memory of two queues used for pre-scan
......
mindspore/ccsrc/minddata/dataset/kernels/tensor_op.h
浏览文件 @ 3bac9d37
......@@ -111,7 +111,7 @@ constexpr char kWhitespaceTokenizerOp[] = "WhitespaceTokenizerOp";
constexpr char kWordpieceTokenizerOp[] = "WordpieceTokenizerOp";
constexpr char kRandomChoiceOp[] = "RandomChoiceOp";
constexpr char kRandomApplyOp[] = "RandomApplyOp";
constexpr char kComposeOp[] = "ComposeOp";
constexpr char kComposeOp[] = "Compose";
constexpr char kRandomSelectSubpolicyOp[] = "RandomSelectSubpolicyOp";
constexpr char kSentencepieceTokenizerOp[] = "SentencepieceTokenizerOp";
......
mindspore/dataset/__init__.py
浏览文件 @ 3bac9d37
......@@ -19,7 +19,7 @@ can also create samplers with this module to sample data.
"""
from .core import config
from .engine.datasets import TFRecordDataset, ImageFolderDatasetV2, MnistDataset, MindDataset, NumpySlicesDataset, \
from .engine.datasets import TFRecordDataset, ImageFolderDataset, MnistDataset, MindDataset, NumpySlicesDataset, \
GeneratorDataset, ManifestDataset, Cifar10Dataset, Cifar100Dataset, VOCDataset, CocoDataset, CelebADataset, \
TextFileDataset, CLUEDataset, CSVDataset, Schema, Shuffle, zip, RandomDataset, PaddedDataset
from .engine.samplers import DistributedSampler, PKSampler, RandomSampler, SequentialSampler, SubsetRandomSampler, \
......@@ -28,7 +28,7 @@ from .engine.cache_client import DatasetCache
from .engine.serializer_deserializer import serialize, deserialize, show
from .engine.graphdata import GraphData
__all__ = ["config", "ImageFolderDatasetV2", "MnistDataset", "PaddedDataset",
__all__ = ["config", "ImageFolderDataset", "MnistDataset", "PaddedDataset",
"MindDataset", "GeneratorDataset", "TFRecordDataset",
"ManifestDataset", "Cifar10Dataset", "Cifar100Dataset", "CelebADataset", "NumpySlicesDataset", "VOCDataset",
"CocoDataset", "TextFileDataset", "CLUEDataset", "CSVDataset", "Schema", "DistributedSampler", "PKSampler",
......
mindspore/dataset/core/py_util_helpers.py 0 → 100644
浏览文件 @ 3bac9d37
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
General py_transforms_utils functions.
"""
import numpy as np
def is_numpy(img):
"""
Check if the input image is Numpy format.
Args:
img: Image to be checked.
Returns:
Bool, True if input is Numpy image.
"""
return isinstance(img, np.ndarray)
mindspore/dataset/engine/__init__.py
浏览文件 @ 3bac9d37
......@@ -28,7 +28,7 @@ from .serializer_deserializer import serialize, deserialize, show, compare
from .samplers import *
from ..core import config
__all__ = ["config", "zip", "ImageFolderDatasetV2", "MnistDataset",
__all__ = ["config", "zip", "ImageFolderDataset", "MnistDataset",
"MindDataset", "GeneratorDataset", "TFRecordDataset", "CLUEDataset", "CSVDataset",
"ManifestDataset", "Cifar10Dataset", "Cifar100Dataset", "CelebADataset",
"VOCDataset", "CocoDataset", "TextFileDataset", "Schema", "DistributedSampler",
......
mindspore/dataset/engine/iterators.py
浏览文件 @ 3bac9d37
......@@ -150,7 +150,7 @@ class Iterator:
op_type = OpName.SKIP
elif isinstance(dataset, de.TakeDataset):
op_type = OpName.TAKE
elif isinstance(dataset, de.ImageFolderDatasetV2):
elif isinstance(dataset, de.ImageFolderDataset):
op_type = OpName.IMAGEFOLDER
elif isinstance(dataset, de.GeneratorDataset):
op_type = OpName.GENERATOR
......
mindspore/dataset/engine/samplers.py
浏览文件 @ 3bac9d37
......@@ -41,7 +41,7 @@ class Sampler:
>>> for i in range(self.dataset_size - 1, -1, -1):
>>> yield i
>>>
>>> ds = ds.ImageFolderDatasetV2(path, sampler=ReverseSampler())
>>> ds = ds.ImageFolderDataset(path, sampler=ReverseSampler())
"""
def __init__(self, num_samples=None):
......@@ -232,7 +232,7 @@ class DistributedSampler(BuiltinSampler):
>>>
>>> # creates a distributed sampler with 10 shards total. This shard is shard 5
>>> sampler = ds.DistributedSampler(10, 5)
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If num_shards is not positive.
......@@ -315,7 +315,7 @@ class PKSampler(BuiltinSampler):
>>>
>>> # creates a PKSampler that will get 3 samples from every class.
>>> sampler = ds.PKSampler(3)
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If num_val is not positive.
......@@ -387,7 +387,7 @@ class RandomSampler(BuiltinSampler):
>>>
>>> # creates a RandomSampler
>>> sampler = ds.RandomSampler()
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If replacement is not boolean.
......@@ -447,7 +447,7 @@ class SequentialSampler(BuiltinSampler):
>>>
>>> # creates a SequentialSampler
>>> sampler = ds.SequentialSampler()
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
"""
def __init__(self, start_index=None, num_samples=None):
......@@ -510,7 +510,7 @@ class SubsetRandomSampler(BuiltinSampler):
>>>
>>> # creates a SubsetRandomSampler, will sample from the provided indices
>>> sampler = ds.SubsetRandomSampler()
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
"""
def __init__(self, indices, num_samples=None):
......@@ -573,7 +573,7 @@ class WeightedRandomSampler(BuiltinSampler):
>>>
>>> # creates a WeightedRandomSampler that will sample 4 elements without replacement
>>> sampler = ds.WeightedRandomSampler(weights, 4)
>>> data = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8, sampler=sampler)
>>> data = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8, sampler=sampler)
Raises:
ValueError: If num_samples is not positive.
......
mindspore/dataset/engine/serializer_deserializer.py
浏览文件 @ 3bac9d37
......@@ -21,9 +21,10 @@ import sys
from mindspore import log as logger
from . import datasets as de
from ..transforms.vision.utils import Inter, Border
from ..vision.utils import Inter, Border
from ..core import config
def serialize(dataset, json_filepath=None):
"""
Serialize dataset pipeline into a json file.
......@@ -44,7 +45,7 @@ def serialize(dataset, json_filepath=None):
>>> DATA_DIR = "../../data/testMnistData"
>>> data = ds.MnistDataset(DATA_DIR, 100)
>>> one_hot_encode = C.OneHot(10) # num_classes is input argument
>>> data = data.map(input_column_names="label", operation=one_hot_encode)
>>> data = data.map(operation=one_hot_encode, input_column_names="label")
>>> data = data.batch(batch_size=10, drop_remainder=True)
>>>
>>> ds.engine.serialize(data, json_filepath="mnist_dataset_pipeline.json") # serialize it to json file
......@@ -77,7 +78,7 @@ def deserialize(input_dict=None, json_filepath=None):
>>> DATA_DIR = "../../data/testMnistData"
>>> data = ds.MnistDataset(DATA_DIR, 100)
>>> one_hot_encode = C.OneHot(10) # num_classes is input argument
>>> data = data.map(input_column_names="label", operation=one_hot_encode)
>>> data = data.map(operation=one_hot_encode, input_column_names="label")
>>> data = data.batch(batch_size=10, drop_remainder=True)
>>>
>>> # Use case 1: to/from json file
......@@ -254,7 +255,7 @@ def create_node(node):
pyobj = None
# Find a matching Dataset class and call the constructor with the corresponding args.
# When a new Dataset class is introduced, another if clause and parsing code needs to be added.
if dataset_op == 'ImageFolderDatasetV2':
if dataset_op == 'ImageFolderDataset':
sampler = construct_sampler(node.get('sampler'))
pyobj = pyclass(node['dataset_dir'], node.get('num_samples'), node.get('num_parallel_workers'),
node.get('shuffle'), sampler, node.get('extensions'),
......@@ -336,8 +337,8 @@ def create_node(node):
elif dataset_op == 'MapDataset':
tensor_ops = construct_tensor_ops(node.get('operations'))
pyobj = de.Dataset().map(node.get('input_columns'), tensor_ops, node.get('output_columns'),
node.get('columns_order'), node.get('num_parallel_workers'))
pyobj = de.Dataset().map(tensor_ops, node.get('input_columns'), node.get('output_columns'),
node.get('column_order'), node.get('num_parallel_workers'))
elif dataset_op == 'ShuffleDataset':
pyobj = de.Dataset().shuffle(node.get('buffer_size'))
......
mindspore/dataset/engine/validators.py
浏览文件 @ 3bac9d37
......@@ -35,8 +35,8 @@ from . import cache_client
from .. import callback
def check_imagefolderdatasetv2(method):
"""A wrapper that wraps a parameter checker around the original Dataset(ImageFolderDatasetV2)."""
def check_imagefolderdataset(method):
"""A wrapper that wraps a parameter checker around the original Dataset(ImageFolderDataset)."""
@wraps(method)
def new_method(self, *args, **kwargs):
......@@ -474,8 +474,8 @@ def check_batch(method):
@wraps(method)
def new_method(self, *args, **kwargs):
[batch_size, drop_remainder, num_parallel_workers, per_batch_map,
input_columns, pad_info], param_dict = parse_user_args(method, *args, **kwargs)
[batch_size, drop_remainder, num_parallel_workers, per_batch_map, input_columns, output_columns,
column_order, pad_info], param_dict = parse_user_args(method, *args, **kwargs)
if not (isinstance(batch_size, int) or (callable(batch_size))):
raise TypeError("batch_size should either be an int or a callable.")
......@@ -510,6 +510,12 @@ def check_batch(method):
if len(input_columns) != (len(ins.signature(per_batch_map).parameters) - 1):
raise ValueError("the signature of per_batch_map should match with input columns")
if output_columns is not None:
raise ValueError("output_columns is currently not implemented.")
if column_order is not None:
raise ValueError("column_order is currently not implemented.")
return method(self, *args, **kwargs)
return new_method
......@@ -551,14 +557,14 @@ def check_map(method):
@wraps(method)
def new_method(self, *args, **kwargs):
[input_columns, _, output_columns, columns_order, num_parallel_workers, python_multiprocessing, cache,
[_, input_columns, output_columns, column_order, num_parallel_workers, python_multiprocessing, cache,
callbacks], _ = \
parse_user_args(method, *args, **kwargs)
nreq_param_columns = ['input_columns', 'output_columns', 'columns_order']
nreq_param_columns = ['input_columns', 'output_columns', 'column_order']
if columns_order is not None:
type_check(columns_order, (list,), "columns_order")
if column_order is not None:
type_check(column_order, (list,), "column_order")
if num_parallel_workers is not None:
check_num_parallel_workers(num_parallel_workers)
type_check(python_multiprocessing, (bool,), "python_multiprocessing")
......@@ -571,7 +577,7 @@ def check_map(method):
else:
type_check(callbacks, (callback.DSCallback,), "callbacks")
for param_name, param in zip(nreq_param_columns, [input_columns, output_columns, columns_order]):
for param_name, param in zip(nreq_param_columns, [input_columns, output_columns, column_order]):
if param is not None:
check_columns(param, param_name)
if callbacks is not None:
......
mindspore/dataset/text/transforms.py
浏览文件 @ 3bac9d37
......@@ -103,7 +103,6 @@ class SlidingWindow(cde.SlidingWindowOp):
super().__init__(width, axis)
class Ngram(cde.NgramOp):
"""
TensorOp to generate n-gram from a 1-D string Tensor.
......@@ -161,8 +160,9 @@ class JiebaTokenizer(cde.JiebaTokenizerOp):
>>> # If with_offsets=False, then output three columns {["token", dtype=str], ["offsets_start", dtype=uint32],
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = JiebaTokenizer(HMM_FILE, MP_FILE, mode=JiebaMode.MP, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> data = data.map(operations=tokenizer_op, input_columns=["text"],
>>> output_columns=["token", "offsets_start", "offsets_limit"],
>>> column_order=["token", "offsets_start", "offsets_limit"])
"""
@check_jieba_init
......@@ -281,7 +281,7 @@ class UnicodeCharTokenizer(cde.UnicodeCharTokenizerOp):
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.UnicodeCharTokenizer(True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_with_offsets
......@@ -312,7 +312,7 @@ class WordpieceTokenizer(cde.WordpieceTokenizerOp):
>>> tokenizer_op = text.WordpieceTokenizer(vocab=vocab, unknown_token=['UNK'],
>>> max_bytes_per_token=100, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_wordpiece_tokenizer
......@@ -377,7 +377,7 @@ if platform.system().lower() != 'windows':
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.WhitespaceTokenizer(True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_with_offsets
......@@ -403,7 +403,7 @@ if platform.system().lower() != 'windows':
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.UnicodeScriptTokenizerOp(keep_whitespace=True, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_unicode_script_tokenizer
......@@ -496,7 +496,7 @@ if platform.system().lower() != 'windows':
>>> # ["offsets_limit", dtype=uint32]}
>>> tokenizer_op = text.RegexTokenizer(delim_pattern, keep_delim_pattern, with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_regex_tokenizer
......@@ -539,7 +539,7 @@ if platform.system().lower() != 'windows':
>>> preserve_unused_token=True,
>>> with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_basic_tokenizer
......@@ -592,7 +592,7 @@ if platform.system().lower() != 'windows':
>>> normalization_form=NormalizeForm.NONE, preserve_unused_token=True,
>>> with_offsets=True)
>>> data = data.map(input_columns=["text"], output_columns=["token", "offsets_start", "offsets_limit"],
>>> columns_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
>>> column_order=["token", "offsets_start", "offsets_limit"], operations=tokenizer_op)
"""
@check_bert_tokenizer
......
mindspore/dataset/transforms/__init__.py
浏览文件 @ 3bac9d37
......@@ -16,6 +16,6 @@ This module is to support common augmentations. C_transforms is a high performan
image augmentation module which is developed with C++ OpenCV. Py_transforms
provide more kinds of image augmentations which is developed with Python PIL.
"""
from . import vision
from .. import vision
from . import c_transforms
from . import py_transforms
mindspore/dataset/transforms/c_transforms.py
浏览文件 @ 3bac9d37
......@@ -229,8 +229,8 @@ class Duplicate(cde.DuplicateOp):
>>> # +---------+
>>> # | [1,2,3] |
>>> # +---------+
>>> data = data.map(input_columns=["x"], operations=Duplicate(),
>>> output_columns=["x", "y"], columns_order=["x", "y"])
>>> data = data.map(operations=Duplicate(), input_columns=["x"],
>>> output_columns=["x", "y"], column_order=["x", "y"])
>>> # Data after
>>> # | x | y |
>>> # +---------+---------+
......
mindspore/dataset/transforms/py_transforms.py
浏览文件 @ 3bac9d37
......@@ -17,9 +17,8 @@
This module py_transforms is implemented basing on Python. It provides common
operations including OneHotOp.
"""
from .validators import check_one_hot_op
from .vision import py_transforms_util as util
from .validators import check_one_hot_op, check_compose_list
from . import py_transforms_util as util
class OneHotOp:
......@@ -48,3 +47,48 @@ class OneHotOp:
label (numpy.ndarray), label after being Smoothed.
"""
return util.one_hot_encoding(label, self.num_classes, self.smoothing_rate)
class Compose:
"""
Compose a list of transforms.
.. Note::
Compose takes a list of transformations either provided in py_transforms or from user-defined implementation;
each can be an initialized transformation class or a lambda function, as long as the output from the last
transformation is a single tensor of type numpy.ndarray. See below for an example of how to use Compose
with py_transforms classes and check out FiveCrop or TenCrop for the use of them in conjunction with lambda
functions.
Args:
transforms (list): List of transformations to be applied.
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.vision.py_transforms as py_transforms
>>> from mindspore.dataset.transforms.py_transforms import Compose
>>> dataset_dir = "path/to/imagefolder_directory"
>>> # create a dataset that reads all files in dataset_dir with 8 threads
>>> dataset = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8)
>>> # create a list of transformations to be applied to the image data
>>> transform = Compose([py_transforms.Decode(),
>>> py_transforms.RandomHorizontalFlip(0.5),
>>> py_transforms.ToTensor(),
>>> py_transforms.Normalize((0.491, 0.482, 0.447), (0.247, 0.243, 0.262)),
>>> py_transforms.RandomErasing()])
>>> # apply the transform to the dataset through dataset.map()
>>> dataset = dataset.map(operations=transform, input_columns="image")
"""
@check_compose_list
def __init__(self, transforms):
self.transforms = transforms
def __call__(self, img):
"""
Call method.
Returns:
lambda function, Lambda function that takes in an img to apply transformations on.
"""
return util.compose(img, self.transforms)
mindspore/dataset/transforms/py_transforms_util.py 0 → 100644
浏览文件 @ 3bac9d37
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Built-in py_transforms_utils functions.
"""
import numpy as np
from ..core.py_util_helpers import is_numpy
def compose(img, transforms):
"""
Compose a list of transforms and apply on the image.
Args:
img (numpy.ndarray): An image in Numpy ndarray.
transforms (list): A list of transform Class objects to be composed.
Returns:
img (numpy.ndarray), An augmented image in Numpy ndarray.
"""
if is_numpy(img):
for transform in transforms:
img = transform(img)
if is_numpy(img):
return img
raise TypeError('img should be Numpy ndarray. Got {}. Append ToTensor() to transforms'.format(type(img)))
raise TypeError('img should be Numpy ndarray. Got {}.'.format(type(img)))
def one_hot_encoding(label, num_classes, epsilon):
"""
Apply label smoothing transformation to the input label, and make label be more smoothing and continuous.
Args:
label (numpy.ndarray): label to be applied label smoothing.
num_classes (int): Num class of object in dataset, value should over 0.
epsilon (float): The adjustable Hyper parameter. Default is 0.0.
Returns:
img (numpy.ndarray), label after being one hot encoded and done label smoothed.
"""
if label > num_classes:
raise ValueError('the num_classes is smaller than the category number.')
num_elements = label.size
one_hot_label = np.zeros((num_elements, num_classes), dtype=int)
if isinstance(label, list) is False:
label = [label]
for index in range(num_elements):
one_hot_label[index, label[index]] = 1
return (1 - epsilon) * one_hot_label + epsilon / num_classes
mindspore/dataset/transforms/validators.py
浏览文件 @ 3bac9d37
......@@ -200,3 +200,19 @@ def check_random_transform_ops(method):
return method(self, *args, **kwargs)
return new_method
def check_compose_list(method):
"""Wrapper method to check the transform list of Compose."""
@wraps(method)
def new_method(self, *args, **kwargs):
[transforms], _ = parse_user_args(method, *args, **kwargs)
type_check(transforms, (list,), transforms)
if not transforms:
raise ValueError("transforms list is empty.")
return method(self, *args, **kwargs)
return new_method
mindspore/dataset/transforms/vision/c_transforms.py → mindspore/dataset/vision/c_transforms.py
浏览文件 @ 3bac9d37
......@@ -25,11 +25,12 @@ to improve their training models.
Examples:
>>> import mindspore.dataset as ds
>>> import mindspore.dataset.transforms.c_transforms as c_transforms
>>> import mindspore.dataset.transforms.vision.c_transforms as c_vision
>>> import mindspore.dataset.vision.c_transforms as c_vision
>>> from mindspore.dataset.transforms.vision.utils import Border, ImageBatchFormat, Inter
>>> dataset_dir = "path/to/imagefolder_directory"
>>> # create a dataset that reads all files in dataset_dir with 8 threads
>>> data1 = ds.ImageFolderDatasetV2(dataset_dir, num_parallel_workers=8)
>>> data1 = ds.ImageFolderDataset(dataset_dir, num_parallel_workers=8)
>>> # create a list of transformations to be applied to the image data
>>> transforms_list = [c_vision.Decode(),
>>> c_vision.Resize((256, 256)),
......@@ -1095,7 +1096,7 @@ class UniformAugment(cde.UniformAugOp):
num_ops (int, optional): Number of operations to be selected and applied (default=2).
Examples:
>>> import mindspore.dataset.transforms.vision.py_transforms as py_vision
>>> import mindspore.dataset.vision.py_transforms as py_vision
>>> transforms_list = [c_vision.RandomHorizontalFlip(),
>>> c_vision.RandomVerticalFlip(),
>>> c_vision.RandomColorAdjust(),
......
mindspore/dataset/transforms/vision/py_transforms_util.py → mindspore/dataset/vision/py_transforms_util.py
浏览文件 @ 3bac9d37
......@@ -24,6 +24,7 @@ import numpy as np
from PIL import Image, ImageOps, ImageEnhance, __version__
from .utils import Inter
from ..core.py_util_helpers import is_numpy
augment_error_message = 'img should be PIL image. Got {}. Use Decode() for encoded data or ToPIL() for decoded data.'
......@@ -41,39 +42,6 @@ def is_pil(img):
return isinstance(img, Image.Image)
def is_numpy(img):
"""
Check if the input image is NumPy format.
Args:
img: Image to be checked.
Returns:
Bool, True if input is NumPy image.
"""
return isinstance(img, np.ndarray)
def compose(img, transforms):
"""
Compose a list of transforms and apply on the image.
Args:
img (numpy.ndarray): An image in NumPy ndarray.
transforms (list): A list of transform Class objects to be composed.
Returns:
img (numpy.ndarray), An augmented image in NumPy ndarray.
"""
if is_numpy(img):
for transform in transforms:
img = transform(img)
if is_numpy(img):
return img
raise TypeError('img should be NumPy ndarray. Got {}. Append ToTensor() to transforms'.format(type(img)))
raise TypeError('img should be NumPy ndarray. Got {}.'.format(type(img)))
def normalize(img, mean, std):
"""
Normalize the image between [0, 1] with respect to mean and standard deviation.
......@@ -1221,32 +1189,6 @@ def random_affine(img, angle, translations, scale, shear, resample, fill_value=0
return img.transform(output_size, Image.AFFINE, matrix, resample, **kwargs)
def one_hot_encoding(label, num_classes, epsilon):
"""
Apply label smoothing transformation to the input label, and make label be more smoothing and continuous.
Args:
label (numpy.ndarray): label to be applied label smoothing.
num_classes (int): Num class of object in dataset, value should over 0.
epsilon (float): The adjustable Hyper parameter. Default is 0.0.
Returns:
img (numpy.ndarray), label after being one hot encoded and done label smoothed.
"""
if label > num_classes:
raise ValueError('the num_classes is smaller than the category number.')
num_elements = label.size
one_hot_label = np.zeros((num_elements, num_classes), dtype=int)
if isinstance(label, list) is False:
label = [label]
for index in range(num_elements):
one_hot_label[index, label[index]] = 1
return (1 - epsilon) * one_hot_label + epsilon / num_classes
def mix_up_single(batch_size, img, label, alpha=0.2):
"""
Apply mix up transformation to image and label in single batch internal, One hot encoding should done before this.
......
mindspore/dataset/transforms/vision/validators.py → mindspore/dataset/vision/validators.py
浏览文件 @ 3bac9d37
......@@ -19,10 +19,10 @@ from functools import wraps
import numpy as np
from mindspore._c_dataengine import TensorOp
from .utils import Inter, Border, ImageBatchFormat
from ...core.validator_helpers import check_value, check_uint8, FLOAT_MAX_INTEGER, check_pos_float32, \
from mindspore.dataset.core.validator_helpers import check_value, check_uint8, FLOAT_MAX_INTEGER, check_pos_float32, \
check_2tuple, check_range, check_positive, INT32_MAX, parse_user_args, type_check, type_check_list, \
check_tensor_op, UINT8_MAX, check_value_normalize_std
from .utils import Inter, Border, ImageBatchFormat
def check_crop_size(size):
......@@ -678,21 +678,6 @@ def check_positive_degrees(method):
return new_method
def check_compose_list(method):
"""Wrapper method to check the transform list of ComposeOp."""
@wraps(method)
def new_method(self, *args, **kwargs):
[transforms], _ = parse_user_args(method, *args, **kwargs)
type_check(transforms, (list,), transforms)
if not transforms:
raise ValueError("transforms list is empty.")
return method(self, *args, **kwargs)
return new_method
def check_random_select_subpolicy_op(method):
"""Wrapper method to check the parameters of RandomSelectSubpolicyOp."""
......
mindspore/train/callback/_summary_collector.py
浏览文件 @ 3bac9d37
......@@ -727,7 +727,7 @@ class SummaryCollector(Callback):
Get dataset path of MindDataset object.
Args:
output_dataset (Union[Dataset, ImageFolderDatasetV2, MnistDataset, Cifar10Dataset, Cifar100Dataset,
output_dataset (Union[Dataset, ImageFolderDataset, MnistDataset, Cifar10Dataset, Cifar100Dataset,
VOCDataset, CelebADataset, MindDataset, ManifestDataset, TFRecordDataset, TextFileDataset]):
Refer to mindspore.dataset.Dataset.
......@@ -738,7 +738,7 @@ class SummaryCollector(Callback):
IndexError: it means get dataset path failed.
"""
dataset_package = import_module('mindspore.dataset')
dataset_dir_set = (dataset_package.ImageFolderDatasetV2, dataset_package.MnistDataset,
dataset_dir_set = (dataset_package.ImageFolderDataset, dataset_package.MnistDataset,
dataset_package.Cifar10Dataset, dataset_package.Cifar100Dataset,
dataset_package.VOCDataset, dataset_package.CelebADataset)
dataset_file_set = (dataset_package.MindDataset, dataset_package.ManifestDataset)
......
model_zoo/official/cv/faster_rcnn/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -449,7 +449,7 @@ def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, devi
if is_training:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "box", "label", "valid_num"],
columns_order=["image", "image_shape", "box", "label", "valid_num"],
column_order=["image", "image_shape", "box", "label", "valid_num"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
flip = (np.random.rand() < config.flip_ratio)
......@@ -467,7 +467,7 @@ def create_fasterrcnn_dataset(mindrecord_file, batch_size=2, repeat_num=12, devi
else:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "box", "label", "valid_num"],
columns_order=["image", "image_shape", "box", "label", "valid_num"],
column_order=["image", "image_shape", "box", "label", "valid_num"],
operations=compose_map_func,
num_parallel_workers=num_parallel_workers)
......
model_zoo/official/cv/inceptionv3/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -37,10 +37,10 @@ def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
dataset
"""
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
# define map operations
if do_train:
trans = [
......
model_zoo/official/cv/maskrcnn/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -505,7 +505,7 @@ def create_maskrcnn_dataset(mindrecord_file, batch_size=2, device_num=1, rank_id
if is_training:
ds = ds.map(input_columns=["image", "annotation", "mask", "mask_shape"],
output_columns=["image", "image_shape", "box", "label", "valid_num", "mask"],
columns_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
column_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
operations=compose_map_func,
python_multiprocessing=False,
num_parallel_workers=num_parallel_workers)
......@@ -514,7 +514,7 @@ def create_maskrcnn_dataset(mindrecord_file, batch_size=2, device_num=1, rank_id
else:
ds = ds.map(input_columns=["image", "annotation", "mask", "mask_shape"],
output_columns=["image", "image_shape", "box", "label", "valid_num", "mask"],
columns_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
column_order=["image", "image_shape", "box", "label", "valid_num", "mask"],
operations=compose_map_func,
num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
......
model_zoo/official/cv/mobilenetv2/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -26,6 +26,7 @@ import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
def create_dataset(dataset_path, do_train, config, repeat_num=1):
"""
create a train or eval dataset
......@@ -44,20 +45,19 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1):
rank_size = int(os.getenv("RANK_SIZE", '1'))
rank_id = int(os.getenv("RANK_ID", '0'))
if rank_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
elif config.platform == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
elif config.platform == "CPU":
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
resize_height = config.image_height
resize_width = config.image_width
......@@ -71,7 +71,8 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1):
resize_op = C.Resize((256, 256))
center_crop = C.CenterCrop(resize_width)
rescale_op = C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
normalize_op = C.Normalize(mean=[0.485*255, 0.456*255, 0.406*255], std=[0.229*255, 0.224*255, 0.225*255])
normalize_op = C.Normalize(mean=[0.485 * 255, 0.456 * 255, 0.406 * 255],
std=[0.229 * 255, 0.224 * 255, 0.225 * 255])
change_swap_op = C.HWC2CHW()
if do_train:
......@@ -95,6 +96,7 @@ def create_dataset(dataset_path, do_train, config, repeat_num=1):
return ds
def extract_features(net, dataset_path, config):
features_folder = dataset_path + '_features'
if not os.path.exists(features_folder):
......@@ -110,13 +112,13 @@ def extract_features(net, dataset_path, config):
for data in pbar:
features_path = os.path.join(features_folder, f"feature_{i}.npy")
label_path = os.path.join(features_folder, f"label_{i}.npy")
if not(os.path.exists(features_path) and os.path.exists(label_path)):
if not (os.path.exists(features_path) and os.path.exists(label_path)):
image = data["image"]
label = data["label"]
features = model.predict(Tensor(image))
np.save(features_path, features.asnumpy())
np.save(label_path, label)
pbar.set_description("Process dataset batch: %d"%(i+1))
pbar.set_description("Process dataset batch: %d" % (i + 1))
i += 1
return step_size
model_zoo/official/cv/mobilenetv2_quant/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,8 @@ import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.transforms.vision.py_transforms as P
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as P
def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1, batch_size=32):
......@@ -44,7 +45,7 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1,
if config.data_load_mode == "mindrecord":
load_func = partial(de.MindDataset, dataset_path, columns_list)
else:
load_func = partial(de.ImageFolderDatasetV2, dataset_path)
load_func = partial(de.ImageFolderDataset, dataset_path)
if do_train:
if rank_size == 1:
ds = load_func(num_parallel_workers=8, shuffle=True)
......@@ -56,10 +57,10 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1,
elif device_target == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
raise ValueError("Unsupported device_target.")
......@@ -118,12 +119,12 @@ def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num=
rank_id = int(os.getenv("RANK_ID"))
if do_train:
if rank_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=rank_size, shard_id=rank_id)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=False)
else:
raise ValueError("Unsupported device target.")
......@@ -149,9 +150,9 @@ def create_dataset_py(dataset_path, do_train, config, device_target, repeat_num=
else:
trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op]
compose = P.ComposeOp(trans)
compose = mindspore.dataset.transforms.py_transforms.Compose(trans)
ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True)
ds = ds.map(input_columns="image", operations=compose, num_parallel_workers=8, python_multiprocessing=True)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
......
model_zoo/official/cv/mobilenetv3/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -37,10 +37,10 @@ def create_dataset(dataset_path, do_train, config, device_target, repeat_num=1,
if device_target == "GPU":
if do_train:
from mindspore.communication.management import get_rank, get_group_size
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=get_group_size(), shard_id=get_rank())
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
raise ValueError("Unsupported device_target.")
......
model_zoo/official/cv/nasnet/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -37,24 +37,24 @@ def create_dataset(dataset_path, config, do_train, repeat_num=1):
rank = config.rank
group_size = config.group_size
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=config.work_nums, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=config.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=config.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(config.image_size),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode
#C.RandomColorAdjust(brightness=0.4, contrast=0.5, saturation=0.5, hue=0.2)
]
C.RandomColorAdjust(brightness=0.4, saturation=0.5) # fast mode
# C.RandomColorAdjust(brightness=0.4, contrast=0.5, saturation=0.5, hue=0.2)
]
else:
trans = [
C.Decode(),
C.Resize(int(config.image_size/0.875)),
C.Resize(int(config.image_size / 0.875)),
C.CenterCrop(config.image_size)
]
]
trans += [
C.Rescale(1.0 / 255.0, 0.0),
C.Normalize(mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5]),
......
model_zoo/official/cv/resnet/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -98,10 +98,10 @@ def create_dataset2(dataset_path, do_train, repeat_num=1, batch_size=32, target=
device_num = get_group_size()
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
......@@ -153,10 +153,10 @@ def create_dataset3(dataset_path, do_train, repeat_num=1, batch_size=32, target=
device_num, rank_id = _get_rank_info()
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.475 * 255, 0.451 * 255, 0.392 * 255]
std = [0.275 * 255, 0.267 * 255, 0.278 * 255]
......@@ -207,10 +207,10 @@ def create_dataset4(dataset_path, do_train, repeat_num=1, batch_size=32, target=
if target == "Ascend":
device_num, rank_id = _get_rank_info()
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=12, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=12, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=12, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [123.68, 116.78, 103.94]
std = [1.0, 1.0, 1.0]
......
model_zoo/official/cv/resnet50_quant/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,8 @@ import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.transforms.vision.py_transforms as P
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as P
from mindspore.communication.management import init, get_rank, get_group_size
from src.config import config_quant
......@@ -54,7 +55,7 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="
if config.data_load_mode == "mindrecord":
load_func = partial(de.MindDataset, dataset_path, columns_list)
else:
load_func = partial(de.ImageFolderDatasetV2, dataset_path)
load_func = partial(de.ImageFolderDataset, dataset_path)
if device_num == 1:
ds = load_func(num_parallel_workers=8, shuffle=True)
else:
......@@ -120,12 +121,12 @@ def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, targe
if do_train:
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=False)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=False)
image_size = 224
......@@ -145,8 +146,8 @@ def create_dataset_py(dataset_path, do_train, repeat_num=1, batch_size=32, targe
else:
trans = [decode_op, resize_op, center_crop, to_tensor, normalize_op]
compose = P.ComposeOp(trans)
ds = ds.map(input_columns="image", operations=compose(), num_parallel_workers=8, python_multiprocessing=True)
compose = mindspore.dataset.transforms.py_transforms.Compose(trans)
ds = ds.map(input_columns="image", operations=compose, num_parallel_workers=8, python_multiprocessing=True)
# apply batch operations
ds = ds.batch(batch_size, drop_remainder=True)
......
model_zoo/official/cv/resnet_thor/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -47,10 +47,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="
num_parallels = 4
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=num_parallels, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=num_parallels, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=num_parallels, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=num_parallels, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
......@@ -86,6 +86,7 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32, target="
return ds
def _get_rank_info():
"""
get rank size and rank id
......
model_zoo/official/cv/resnext50/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -134,9 +134,9 @@ def classification_dataset(data_dir, image_size, per_batch_size, max_epoch, rank
transform_label = target_transform
if input_mode == 'folder':
de_dataset = de.ImageFolderDatasetV2(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
de_dataset = de.ImageFolderDataset(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
else:
dataset = TxtDataset(root, data_dir)
sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle)
......
model_zoo/official/cv/shufflenetv2/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -30,6 +30,7 @@ class toBGR():
img = np.ascontiguousarray(img)
return img
def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
"""
create a train or eval dataset
......@@ -45,23 +46,23 @@ def create_dataset(dataset_path, do_train, rank, group_size, repeat_num=1):
dataset
"""
if group_size == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=cfg.work_nums, shuffle=True,
num_shards=group_size, shard_id=rank)
# define map operations
if do_train:
trans = [
C.RandomCropDecodeResize(224),
C.RandomHorizontalFlip(prob=0.5),
C.RandomColorAdjust(brightness=0.4, contrast=0.4, saturation=0.4)
]
]
else:
trans = [
C.Decode(),
C.Resize(256),
C.CenterCrop(224)
]
]
trans += [
toBGR(),
C.Rescale(1.0 / 255.0, 0.0),
......
model_zoo/official/cv/ssd/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -403,7 +403,7 @@ def create_ssd_dataset(mindrecord_file, batch_size=32, repeat_num=10, device_num
output_columns = ["img_id", "image", "image_shape"]
trans = [normalize_op, change_swap_op]
ds = ds.map(input_columns=["img_id", "image", "annotation"],
output_columns=output_columns, columns_order=output_columns,
output_columns=output_columns, column_order=output_columns,
operations=compose_map_func, python_multiprocessing=is_training,
num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=["image"], operations=trans, python_multiprocessing=is_training,
......
model_zoo/official/cv/vgg16/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -149,9 +149,9 @@ def classification_dataset(data_dir, image_size, per_batch_size, rank=0, group_s
transform_label = target_transform
if input_mode == 'folder':
de_dataset = de.ImageFolderDatasetV2(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
de_dataset = de.ImageFolderDataset(data_dir, num_parallel_workers=num_parallel_workers,
shuffle=shuffle, sampler=sampler, class_indexing=class_indexing,
num_shards=group_size, shard_id=rank)
else:
dataset = TxtDataset(root, data_dir)
sampler = DistributedSampler(dataset, rank, group_size, shuffle=shuffle)
......
model_zoo/official/cv/yolov3_darknet53/src/yolo_dataset.py
浏览文件 @ 3bac9d37
......@@ -178,7 +178,7 @@ def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num,
compose_map_func = (lambda image, img_id: reshape_fn(image, img_id, config))
ds = ds.map(input_columns=["image", "img_id"],
output_columns=["image", "image_shape", "img_id"],
columns_order=["image", "image_shape", "img_id"],
column_order=["image", "image_shape", "img_id"],
operations=compose_map_func, num_parallel_workers=8)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8)
ds = ds.batch(batch_size, drop_remainder=True)
......
model_zoo/official/cv/yolov3_darknet53_quant/src/yolo_dataset.py
浏览文件 @ 3bac9d37
......@@ -175,7 +175,7 @@ def create_yolo_dataset(image_dir, anno_path, batch_size, max_epoch, device_num,
compose_map_func = (lambda image, img_id: reshape_fn(image, img_id, config))
ds = ds.map(input_columns=["image", "img_id"],
output_columns=["image", "image_shape", "img_id"],
columns_order=["image", "image_shape", "img_id"],
column_order=["image", "image_shape", "img_id"],
operations=compose_map_func, num_parallel_workers=8)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8)
ds = ds.batch(batch_size, drop_remainder=True)
......
model_zoo/official/cv/yolov3_resnet18/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -303,7 +303,7 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=1, device_num=
hwc_to_chw = C.HWC2CHW()
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
columns_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
column_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
......@@ -311,6 +311,6 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=1, device_num=
else:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "annotation"],
columns_order=["image", "image_shape", "annotation"],
column_order=["image", "image_shape", "annotation"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
return ds
model_zoo/official/nlp/bert/src/clue_classification_dataset_process.py
浏览文件 @ 3bac9d37
......@@ -43,7 +43,7 @@ def process_tnews_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
### Processing label
if data_usage == 'test':
dataset = dataset.map(input_columns=["id"], output_columns=["id", "label_id"],
columns_order=["id", "label_id", "sentence"], operations=ops.Duplicate())
column_order=["id", "label_id", "sentence"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["label_id"], operations=ops.Fill(0))
else:
label_vocab = text.Vocab.from_list(label_list)
......@@ -61,10 +61,10 @@ def process_tnews_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
dataset = dataset.map(input_columns=["sentence"], output_columns=["text_ids"], operations=lookup)
dataset = dataset.map(input_columns=["text_ids"], operations=ops.PadEnd([max_seq_len], 0))
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "mask_ids"],
columns_order=["text_ids", "mask_ids", "label_id"], operations=ops.Duplicate())
column_order=["text_ids", "mask_ids", "label_id"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["mask_ids"], operations=ops.Mask(ops.Relational.NE, 0, mstype.int32))
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "segment_ids"],
columns_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
column_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["segment_ids"], operations=ops.Fill(0))
dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
return dataset
......@@ -87,7 +87,7 @@ def process_cmnli_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
### Processing label
if data_usage == 'test':
dataset = dataset.map(input_columns=["id"], output_columns=["id", "label_id"],
columns_order=["id", "label_id", "sentence1", "sentence2"], operations=ops.Duplicate())
column_order=["id", "label_id", "sentence1", "sentence2"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["label_id"], operations=ops.Fill(0))
else:
label_vocab = text.Vocab.from_list(label_list)
......@@ -110,26 +110,26 @@ def process_cmnli_clue_dataset(data_dir, label_list, bert_vocab_path, data_usage
operations=ops.Concatenate(append=np.array(["[SEP]"], dtype='S')))
### Generating segment_ids
dataset = dataset.map(input_columns=["sentence1"], output_columns=["sentence1", "type_sentence1"],
columns_order=["sentence1", "type_sentence1", "sentence2", "label_id"],
column_order=["sentence1", "type_sentence1", "sentence2", "label_id"],
operations=ops.Duplicate())
dataset = dataset.map(input_columns=["sentence2"], output_columns=["sentence2", "type_sentence2"],
columns_order=["sentence1", "type_sentence1", "sentence2", "type_sentence2", "label_id"],
column_order=["sentence1", "type_sentence1", "sentence2", "type_sentence2", "label_id"],
operations=ops.Duplicate())
dataset = dataset.map(input_columns=["type_sentence1"], operations=[lookup, ops.Fill(0)])
dataset = dataset.map(input_columns=["type_sentence2"], operations=[lookup, ops.Fill(1)])
dataset = dataset.map(input_columns=["type_sentence1", "type_sentence2"], output_columns=["segment_ids"],
columns_order=["sentence1", "sentence2", "segment_ids", "label_id"],
column_order=["sentence1", "sentence2", "segment_ids", "label_id"],
operations=ops.Concatenate())
dataset = dataset.map(input_columns=["segment_ids"], operations=ops.PadEnd([max_seq_len], 0))
### Generating text_ids
dataset = dataset.map(input_columns=["sentence1", "sentence2"], output_columns=["text_ids"],
columns_order=["text_ids", "segment_ids", "label_id"],
column_order=["text_ids", "segment_ids", "label_id"],
operations=ops.Concatenate())
dataset = dataset.map(input_columns=["text_ids"], operations=lookup)
dataset = dataset.map(input_columns=["text_ids"], operations=ops.PadEnd([max_seq_len], 0))
### Generating mask_ids
dataset = dataset.map(input_columns=["text_ids"], output_columns=["text_ids", "mask_ids"],
columns_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
column_order=["text_ids", "mask_ids", "segment_ids", "label_id"], operations=ops.Duplicate())
dataset = dataset.map(input_columns=["mask_ids"], operations=ops.Mask(ops.Relational.NE, 0, mstype.int32))
dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)
return dataset
model_zoo/official/recommend/deepfm/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -213,7 +213,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds
......@@ -261,7 +261,7 @@ def _get_tf_dataset(directory, train_mode=True, epochs=1, batch_size=1000,
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds
......
model_zoo/official/recommend/wide_and_deep/src/datasets.py
浏览文件 @ 3bac9d37
......@@ -230,7 +230,7 @@ def _get_tf_dataset(data_dir, train_mode=True, epochs=1, batch_size=1000,
ds = ds.map(operations=_padding_func(batch_size, manual_shape, target_column),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
# if train_mode:
ds = ds.repeat(epochs)
return ds
......@@ -270,7 +270,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100
ds = ds.batch(int(batch_size / line_per_sample), drop_remainder=True)
ds = ds.map(_padding_func(batch_size, manual_shape, target_column),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds
......
model_zoo/official/recommend/wide_and_deep_multitable/src/datasets.py
浏览文件 @ 3bac9d37
......@@ -263,7 +263,7 @@ def _get_tf_dataset(data_dir,
'multi_doc_ad_topic_id_mask', 'ad_id', 'display_ad_and_is_leak',
'display_id', 'is_leak'
],
columns_order=[
column_order=[
'label', 'continue_val', 'indicator_id', 'emb_128_id',
'emb_64_single_id', 'multi_doc_ad_category_id',
'multi_doc_ad_category_id_mask', 'multi_doc_event_entity_id',
......
tests/st/mem_reuse/resnet_cifar_memreuse.py
浏览文件 @ 3bac9d37
......@@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype
import mindspore.context as context
import mindspore.dataset as de
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.communication.management import init
......
tests/st/mem_reuse/resnet_cifar_normal.py
浏览文件 @ 3bac9d37
......@@ -22,7 +22,7 @@ import mindspore.common.dtype as mstype
import mindspore.context as context
import mindspore.dataset as de
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.communication.management import init
......
tests/st/model_zoo_tests/wide_and_deep/python_file_for_ci/datasets.py
浏览文件 @ 3bac9d37
......@@ -57,7 +57,7 @@ def _get_tf_dataset(data_dir, train_mode=True, epochs=1, batch_size=1000,
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
column_order=['feat_ids', 'feat_vals', 'label'], num_parallel_workers=8)
#if train_mode:
ds = ds.repeat(epochs)
return ds
......@@ -97,7 +97,7 @@ def _get_mindrecord_dataset(directory, train_mode=True, epochs=1, batch_size=100
np.array(y).flatten().reshape(batch_size, 39),
np.array(z).flatten().reshape(batch_size, 1))),
input_columns=['feat_ids', 'feat_vals', 'label'],
columns_order=['feat_ids', 'feat_vals', 'label'],
column_order=['feat_ids', 'feat_vals', 'label'],
num_parallel_workers=8)
ds = ds.repeat(epochs)
return ds
......
tests/st/model_zoo_tests/yolov3/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -22,7 +22,7 @@ from matplotlib.colors import rgb_to_hsv, hsv_to_rgb
from PIL import Image
import mindspore.dataset as de
from mindspore.mindrecord import FileWriter
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
from src.config import ConfigYOLOV3ResNet18
iter_cnt = 0
......@@ -305,7 +305,7 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=10, device_num
hwc_to_chw = C.HWC2CHW()
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
columns_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
column_order=["image", "bbox_1", "bbox_2", "bbox_3", "gt_box1", "gt_box2", "gt_box3"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
ds = ds.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=num_parallel_workers)
ds = ds.batch(batch_size, drop_remainder=True)
......@@ -313,6 +313,6 @@ def create_yolo_dataset(mindrecord_dir, batch_size=32, repeat_num=10, device_num
else:
ds = ds.map(input_columns=["image", "annotation"],
output_columns=["image", "image_shape", "annotation"],
columns_order=["image", "image_shape", "annotation"],
column_order=["image", "image_shape", "annotation"],
operations=compose_map_func, num_parallel_workers=num_parallel_workers)
return ds
tests/st/networks/models/deeplabv3/src/md_dataset.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
"""Dataset module."""
from PIL import Image
import mindspore.dataset as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
import numpy as np
from .ei_dataset import HwVocRawDataset
......
tests/st/networks/models/resnet50/src/dataset.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@
import os
import mindspore.common.dtype as mstype
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
import mindspore.dataset.transforms.c_transforms as C2
......@@ -39,10 +39,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
......
tests/st/networks/models/resnet50/src_thor/dataset.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,7 @@ import mindspore.common.dtype as mstype
import mindspore.dataset as dataset
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.c_transforms as C2
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.vision.c_transforms as C
dataset.config.set_seed(1)
......@@ -43,10 +43,10 @@ def create_dataset(dataset_path, do_train, repeat_num=1, batch_size=32):
device_num = int(os.getenv("RANK_SIZE"))
rank_id = int(os.getenv("RANK_ID"))
if device_num == 1:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True)
else:
ds = de.ImageFolderDatasetV2(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
ds = de.ImageFolderDataset(dataset_path, num_parallel_workers=8, shuffle=True,
num_shards=device_num, shard_id=rank_id)
image_size = 224
mean = [0.485 * 255, 0.456 * 255, 0.406 * 255]
......
tests/st/networks/test_gpu_lenet.py
浏览文件 @ 3bac9d37
......@@ -21,11 +21,11 @@ import pytest
import mindspore.context as context
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import Tensor
from mindspore.common import dtype as mstype
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.nn import Dense, TrainOneStepCell, WithLossCell
from mindspore.nn.metrics import Accuracy
from mindspore.nn.optim import Momentum
......
tests/st/ops/ascend/test_tdt_data_ms.py
浏览文件 @ 3bac9d37
......@@ -17,11 +17,11 @@ import numpy as np
import mindspore.context as context
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore.common.api import _executor
from mindspore.common.tensor import Tensor
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.ops import operations as P
context.set_context(mode=context.GRAPH_MODE, device_target="Ascend")
......@@ -83,8 +83,6 @@ if __name__ == '__main__':
class dataiter(nn.Cell):
def __init__(self):
super(dataiter, self).__init__()
def construct(self):
input_, _ = get_next()
......
tests/st/probability/dataset.py
浏览文件 @ 3bac9d37
......@@ -17,9 +17,9 @@ Produce the dataset
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.dataset.transforms.c_transforms as C
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.common import dtype as mstype
......
tests/st/probability/test_gpu_svi_cvae.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@ import os
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context, Tensor
from mindspore.ops import operations as P
......
tests/st/probability/test_gpu_svi_vae.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@ import os
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context, Tensor
from mindspore.ops import operations as P
......
tests/st/probability/test_gpu_vae_gan.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ The VAE interface can be called to construct VAE-GAN network.
import os
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context
from mindspore.ops import operations as P
......
tests/st/probability/test_uncertainty.py
浏览文件 @ 3bac9d37
......@@ -15,12 +15,12 @@
""" test uncertainty toolbox """
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore import context, Tensor
from mindspore.common import dtype as mstype
from mindspore.common.initializer import TruncatedNormal
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.nn.probability.toolbox.uncertainty_evaluation import UncertaintyEvaluation
from mindspore.train.serialization import load_checkpoint, load_param_into_net
......
tests/st/ps/full_ps/test_full_ps_lenet.py
浏览文件 @ 3bac9d37
......@@ -19,10 +19,10 @@ import argparse
import mindspore.context as context
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.nn as nn
from mindspore.common import dtype as mstype
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.nn.metrics import Accuracy
from mindspore.train import Model
from mindspore.train.callback import LossMonitor
......
tests/st/pynative/test_pynative_resnet50.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,7 @@ import pytest
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
import mindspore.ops.functional as F
......
tests/st/quantization/lenet_quant/dataset.py
浏览文件 @ 3bac9d37
......@@ -17,9 +17,9 @@ Produce the dataset
"""
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as CV
import mindspore.dataset.vision.c_transforms as CV
import mindspore.dataset.transforms.c_transforms as C
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
from mindspore.common import dtype as mstype
......
tests/st/summary/test_summary.py
浏览文件 @ 3bac9d37
......@@ -25,8 +25,8 @@ from mindspore import nn, Tensor, context
from mindspore.nn.metrics import Accuracy
from mindspore.nn.optim import Momentum
from mindspore.dataset.transforms import c_transforms as C
from mindspore.dataset.transforms.vision import c_transforms as CV
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import c_transforms as CV
from mindspore.dataset.vision import Inter
from mindspore.common import dtype as mstype
from mindspore.common.initializer import TruncatedNormal
from mindspore.ops import operations as P
......
tests/st/tbe_networks/resnet_cifar.py
浏览文件 @ 3bac9d37
......@@ -24,7 +24,7 @@ from mindspore.train.callback import ModelCheckpoint, CheckpointConfig, LossMoni
from mindspore.train.serialization import load_checkpoint, load_param_into_net
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
from mindspore import Tensor
from mindspore import context
......
tests/st/tbe_networks/test_resnet_cifar_1p.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,7 @@ from resnet import resnet50
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
import mindspore.ops.functional as F
from mindspore import Tensor
......
tests/st/tbe_networks/test_resnet_cifar_8p.py
浏览文件 @ 3bac9d37
......@@ -22,7 +22,7 @@ from resnet import resnet50
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.nn as nn
import mindspore.ops.functional as F
from mindspore import Tensor
......
tests/ut/python/dataset/test_HWC2CHW.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing HWC2CHW op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, visualize_list, save_and_check_md5
......@@ -99,8 +100,8 @@ def test_HWC2CHW_comp(plot=False):
py_vision.ToTensor(),
py_vision.HWC2CHW()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
image_c_transposed = []
image_py_transposed = []
......
tests/ut/python/dataset/test_apply.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = "../data/dataset/testPK/data"
......@@ -46,8 +46,8 @@ def test_apply_generator_case():
def test_apply_imagefolder_case():
# apply dataset map operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
data2 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
decode_op = vision.Decode()
normalize_op = vision.Normalize([121.0, 115.0, 100.0], [70.0, 68.0, 71.0])
......
tests/ut/python/dataset/test_autocontrast.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing AutoContrast op in DE
"""
import numpy as np
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as F
import mindspore.dataset.vision.c_transforms as C
from mindspore import log as logger
from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5
......@@ -35,14 +36,14 @@ def test_auto_contrast_py(plot=False):
logger.info("Test AutoContrast Python Op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = ds.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
......@@ -55,15 +56,16 @@ def test_auto_contrast_py(plot=False):
axis=0)
# AutoContrast Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_auto_contrast = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=10.0, ignore=[10, 20]),
F.ToTensor()])
transforms_auto_contrast = \
mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=10.0, ignore=[10, 20]),
F.ToTensor()])
ds_auto_contrast = ds.map(input_columns="image",
operations=transforms_auto_contrast())
operations=transforms_auto_contrast)
ds_auto_contrast = ds_auto_contrast.batch(512)
......@@ -96,15 +98,15 @@ def test_auto_contrast_c(plot=False):
logger.info("Test AutoContrast C Op")
# AutoContrast Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224))])
python_op = F.AutoContrast(cutoff=10.0, ignore=[10, 20])
c_op = C.AutoContrast(cutoff=10.0, ignore=[10, 20])
transforms_op = F.ComposeOp([lambda img: F.ToPIL()(img.astype(np.uint8)),
python_op,
np.array])()
transforms_op = mindspore.dataset.transforms.py_transforms.Compose([lambda img: F.ToPIL()(img.astype(np.uint8)),
python_op,
np.array])
ds_auto_contrast_py = ds.map(input_columns="image",
operations=transforms_op)
......@@ -119,7 +121,7 @@ def test_auto_contrast_c(plot=False):
image,
axis=0)
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224))])
......@@ -159,17 +161,18 @@ def test_auto_contrast_one_channel_c(plot=False):
logger.info("Test AutoContrast C Op With One Channel Images")
# AutoContrast Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224))])
python_op = F.AutoContrast()
c_op = C.AutoContrast()
# not using F.ToTensor() since it converts to floats
transforms_op = F.ComposeOp([lambda img: (np.array(img)[:, :, 0]).astype(np.uint8),
F.ToPIL(),
python_op,
np.array])()
transforms_op = mindspore.dataset.transforms.py_transforms.Compose(
[lambda img: (np.array(img)[:, :, 0]).astype(np.uint8),
F.ToPIL(),
python_op,
np.array])
ds_auto_contrast_py = ds.map(input_columns="image",
operations=transforms_op)
......@@ -184,7 +187,7 @@ def test_auto_contrast_one_channel_c(plot=False):
image,
axis=0)
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -248,7 +251,7 @@ def test_auto_contrast_invalid_ignore_param_c():
"""
logger.info("Test AutoContrast C Op with invalid ignore parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -260,7 +263,7 @@ def test_auto_contrast_invalid_ignore_param_c():
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Argument ignore with value 255.5 is not of type" in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -279,7 +282,7 @@ def test_auto_contrast_invalid_cutoff_param_c():
"""
logger.info("Test AutoContrast C Op with invalid cutoff parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -291,7 +294,7 @@ def test_auto_contrast_invalid_cutoff_param_c():
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -310,22 +313,22 @@ def test_auto_contrast_invalid_ignore_param_py():
"""
logger.info("Test AutoContrast python Op with invalid ignore parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=255.5),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=255.5),
F.ToTensor()])])
except TypeError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Argument ignore with value 255.5 is not of type" in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=(10, 100)),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(ignore=(10, 100)),
F.ToTensor()])])
except TypeError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Argument ignore with value (10,100) is not of type" in str(error)
......@@ -337,22 +340,22 @@ def test_auto_contrast_invalid_cutoff_param_py():
"""
logger.info("Test AutoContrast python Op with invalid cutoff parameter")
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=-10.0),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=-10.0),
F.ToTensor()])])
except ValueError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=120.0),
F.ToTensor()])])
operations=[mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.AutoContrast(cutoff=120.0),
F.ToTensor()])])
except ValueError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
assert "Input cutoff is not within the required interval of (0 to 100)." in str(error)
......
tests/ut/python/dataset/test_batch.py
浏览文件 @ 3bac9d37
......@@ -449,6 +449,22 @@ def test_batch_exception_13():
logger.info("Got an exception in DE: {}".format(str(e)))
assert "shard_id" in str(e)
# test non-functional parameters
try:
data1 = data1.batch(batch_size, output_columns="3")
sum([1 for _ in data1])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "output_columns is currently not implemented." in str(e)
try:
data1 = data1.batch(batch_size, column_order="3")
sum([1 for _ in data1])
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "column_order is currently not implemented." in str(e)
if __name__ == '__main__':
test_batch_01()
......
tests/ut/python/dataset/test_bounding_box_augment.py
浏览文件 @ 3bac9d37
......@@ -19,7 +19,7 @@ Testing the bounding box augment op in DE
import numpy as np
import mindspore.log as logger
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from util import visualize_with_bounding_boxes, InvalidBBoxType, check_bad_bbox, \
config_get_set_seed, config_get_set_num_parallel_workers, save_and_check_md5
......@@ -51,7 +51,7 @@ def test_bounding_box_augment_with_rotation_op(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "bounding_box_augment_rotation_c_result.npz"
......@@ -90,7 +90,7 @@ def test_bounding_box_augment_with_crop_op(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "bounding_box_augment_crop_c_result.npz"
......@@ -128,7 +128,7 @@ def test_bounding_box_augment_valid_ratio_c(plot_vis=False):
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op]) # Add column for "bbox"
filename = "bounding_box_augment_valid_ratio_c_result.npz"
......@@ -165,7 +165,7 @@ def test_bounding_box_augment_op_coco_c(plot_vis=False):
dataCoco2 = dataCoco2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
unaugSamp, augSamp = [], []
......@@ -197,17 +197,17 @@ def test_bounding_box_augment_valid_edge_c(plot_vis=False):
# Add column for "bbox"
dataVoc1 = dataVoc1.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=lambda img, bbox:
(img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32)))
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=lambda img, bbox:
(img, np.array([[0, 0, img.shape[1], img.shape[0], 0, 0, 0]]).astype(np.float32)))
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op])
filename = "bounding_box_augment_valid_edge_c_result.npz"
save_and_check_md5(dataVoc2, filename, generate_golden=GENERATE_GOLDEN)
......@@ -240,7 +240,7 @@ def test_bounding_box_augment_invalid_ratio_c():
# map to apply ops
dataVoc2 = dataVoc2.map(input_columns=["image", "bbox"],
output_columns=["image", "bbox"],
columns_order=["image", "bbox"],
column_order=["image", "bbox"],
operations=[test_op]) # Add column for "bbox"
except ValueError as error:
logger.info("Got an exception in DE: {}".format(str(error)))
......
tests/ut/python/dataset/test_cache_map.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ Testing cache operator with mappable datasets
import os
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
from util import save_and_check_md5
......@@ -46,7 +46,7 @@ def test_cache_map_basic1():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.map(input_columns=["image"], operations=decode_op)
ds1 = ds1.repeat(4)
......@@ -75,7 +75,7 @@ def test_cache_map_basic2():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR)
decode_op = c_vision.Decode()
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
ds1 = ds1.repeat(4)
......@@ -104,7 +104,7 @@ def test_cache_map_basic3():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR)
decode_op = c_vision.Decode()
ds1 = ds1.repeat(4)
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
......@@ -128,7 +128,7 @@ def test_cache_map_basic4():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.repeat(4)
ds1 = ds1.map(input_columns=["image"], operations=decode_op)
......@@ -165,7 +165,7 @@ def test_cache_map_failure1():
some_cache = ds.DatasetCache(session_id=1, size=0, spilling=True)
# This DATA_DIR only has 2 images in it
ds1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR, cache=some_cache)
ds1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR, cache=some_cache)
decode_op = c_vision.Decode()
ds1 = ds1.map(input_columns=["image"], operations=decode_op, cache=some_cache)
ds1 = ds1.repeat(4)
......
tests/ut/python/dataset/test_cache_nomap.py
浏览文件 @ 3bac9d37
......@@ -19,7 +19,7 @@ import os
import pytest
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.vision.c_transforms as c_vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
......
tests/ut/python/dataset/test_center_crop.py
浏览文件 @ 3bac9d37
......@@ -17,8 +17,9 @@ Testing CenterCrop op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, visualize_list, save_and_check_md5
......@@ -93,8 +94,8 @@ def test_center_crop_comp(height=375, width=375, plot=False):
py_vision.CenterCrop([height, width]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
image_c_cropped = []
image_py_cropped = []
......@@ -123,9 +124,9 @@ def test_crop_grayscale(height=375, width=375):
(lambda image: (image.transpose(1, 2, 0) * 255).astype(np.uint8))
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=transform)
# If input is grayscale, the output dimensions should be single channel
crop_gray = vision.CenterCrop([height, width])
......
tests/ut/python/dataset/test_concat.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,8 @@ import numpy as np
import mindspore.common.dtype as mstype
import mindspore.dataset as ds
import mindspore.dataset.transforms.c_transforms as C
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as F
from mindspore import log as logger
......@@ -317,15 +318,15 @@ def test_concat_14():
DATA_DIR = "../data/dataset/testPK/data"
DATA_DIR2 = "../data/dataset/testImageNetData/train/"
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=3)
data2 = ds.ImageFolderDatasetV2(DATA_DIR2, num_samples=2)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=3)
data2 = ds.ImageFolderDataset(DATA_DIR2, num_samples=2)
transforms1 = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms1 = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
data1 = data1.map(input_columns=["image"], operations=transforms1())
data2 = data2.map(input_columns=["image"], operations=transforms1())
data1 = data1.map(input_columns=["image"], operations=transforms1)
data2 = data2.map(input_columns=["image"], operations=transforms1)
data3 = data1 + data2
expected, output = [], []
......@@ -351,7 +352,7 @@ def test_concat_15():
DATA_DIR = "../data/dataset/testPK/data"
DATA_DIR2 = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
data1 = ds.ImageFolderDatasetV2(DATA_DIR)
data1 = ds.ImageFolderDataset(DATA_DIR)
data2 = ds.TFRecordDataset(DATA_DIR2, columns_list=["image"])
data1 = data1.project(["image"])
......
tests/ut/python/dataset/test_concatenate_op.py
浏览文件 @ 3bac9d37
......@@ -74,7 +74,7 @@ def test_concatenate_op_multi_input_string():
concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor, append=append_tensor)
data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"],
data = data.map(input_columns=["col1", "col2"], column_order=["out1"], output_columns=["out1"],
operations=concatenate_op)
expected = np.array(["dw", "df", "1", "2", "d", "3", "4", "e", "dwsdf", "df"], dtype='S')
for data_row in data:
......@@ -89,7 +89,7 @@ def test_concatenate_op_multi_input_numeric():
concatenate_op = data_trans.Concatenate(0, prepend=prepend_tensor)
data = data.map(input_columns=["col1", "col2"], columns_order=["out1"], output_columns=["out1"],
data = data.map(input_columns=["col1", "col2"], column_order=["out1"], output_columns=["out1"],
operations=concatenate_op)
expected = np.array([3, 5, 1, 2, 3, 4])
for data_row in data:
......
tests/ut/python/dataset/test_config.py
浏览文件 @ 3bac9d37
......@@ -21,8 +21,9 @@ import glob
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c_vision
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c_vision
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import dataset_equal
......@@ -283,8 +284,8 @@ def test_deterministic_python_seed():
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor(),
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
data1_output = []
# config.set_seed() calls random.seed()
for data_one in data1.create_dict_iterator(num_epochs=1):
......@@ -292,7 +293,7 @@ def test_deterministic_python_seed():
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
# config.set_seed() calls random.seed(), resets seed for next dataset iterator
ds.config.set_seed(0)
......@@ -326,8 +327,8 @@ def test_deterministic_python_seed_multi_thread():
py_vision.RandomCrop([512, 512], [200, 200, 200, 200]),
py_vision.ToTensor(),
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform(), python_multiprocessing=True)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform, python_multiprocessing=True)
data1_output = []
# config.set_seed() calls random.seed()
for data_one in data1.create_dict_iterator(num_epochs=1):
......@@ -336,7 +337,7 @@ def test_deterministic_python_seed_multi_thread():
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
# If seed is set up on constructor
data2 = data2.map(input_columns=["image"], operations=transform(), python_multiprocessing=True)
data2 = data2.map(input_columns=["image"], operations=transform, python_multiprocessing=True)
# config.set_seed() calls random.seed()
ds.config.set_seed(0)
......
tests/ut/python/dataset/test_cut_out.py
浏览文件 @ 3bac9d37
......@@ -18,8 +18,9 @@ Testing CutOut op in DE
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as c
import mindspore.dataset.transforms.vision.py_transforms as f
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as c
import mindspore.dataset.vision.py_transforms as f
from mindspore import log as logger
from util import visualize_image, visualize_list, diff_mse, save_and_check_md5, \
config_get_set_seed, config_get_set_num_parallel_workers
......@@ -43,8 +44,8 @@ def test_cut_out_op(plot=False):
f.ToTensor(),
f.RandomErasing(value='random')
]
transform_1 = f.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -89,8 +90,8 @@ def test_cut_out_op_multicut(plot=False):
f.Decode(),
f.ToTensor(),
]
transform_1 = f.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -144,8 +145,8 @@ def test_cut_out_md5():
f.ToTensor(),
f.Cutout(100)
]
transform = f.ComposeOp(transforms)
data2 = data2.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data2 = data2.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename1 = "cut_out_01_c_result.npz"
......@@ -172,8 +173,8 @@ def test_cut_out_comp(plot=False):
f.ToTensor(),
f.Cutout(200)
]
transform_1 = f.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......
tests/ut/python/dataset/test_cutmix_batch_op.py
浏览文件 @ 3bac9d37
......@@ -18,9 +18,9 @@ Testing the CutMixBatch op in DE
import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
import mindspore.dataset.transforms.c_transforms as data_trans
import mindspore.dataset.transforms.vision.utils as mode
import mindspore.dataset.vision.utils as mode
from mindspore import log as logger
from util import save_and_check_md5, diff_mse, visualize_list, config_get_set_seed, \
config_get_set_num_parallel_workers
......@@ -119,11 +119,11 @@ def test_cutmix_batch_success2(plot=False):
def test_cutmix_batch_success3(plot=False):
"""
Test CutMixBatch op with default values for alpha and prob on a batch of HWC images on ImageFolderDatasetV2
Test CutMixBatch op with default values for alpha and prob on a batch of HWC images on ImageFolderDataset
"""
logger.info("test_cutmix_batch_success3")
ds_original = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False)
ds_original = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False)
decode_op = vision.Decode()
ds_original = ds_original.map(input_columns=["image"], operations=[decode_op])
ds_original = ds_original.batch(4, pad_info={}, drop_remainder=True)
......@@ -136,7 +136,7 @@ def test_cutmix_batch_success3(plot=False):
images_original = np.append(images_original, image, axis=0)
# CutMix Images
data1 = ds.ImageFolderDatasetV2(dataset_dir=DATA_DIR2, shuffle=False)
data1 = ds.ImageFolderDataset(dataset_dir=DATA_DIR2, shuffle=False)
decode_op = vision.Decode()
data1 = data1.map(input_columns=["image"], operations=[decode_op])
......
tests/ut/python/dataset/test_dataset_numpy_slices.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ import numpy as np
import pandas as pd
import mindspore.dataset as de
from mindspore import log as logger
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
def test_numpy_slices_list_1():
......
tests/ut/python/dataset/test_datasets_celeba.py
浏览文件 @ 3bac9d37
......@@ -12,9 +12,9 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from mindspore.dataset.transforms.vision import Inter
from mindspore.dataset.vision import Inter
DATA_DIR = "../data/dataset/testCelebAData/"
......
tests/ut/python/dataset/test_datasets_coco.py
浏览文件 @ 3bac9d37
......@@ -14,7 +14,7 @@
# ==============================================================================
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
DATA_DIR = "../data/dataset/testCOCO/train/"
DATA_DIR_2 = "../data/dataset/testCOCO/train"
......
tests/ut/python/dataset/test_datasets_generator.py
浏览文件 @ 3bac9d37
......@@ -244,7 +244,7 @@ def test_generator_8():
data1 = data1.map(input_columns="col0", output_columns="out0", operations=(lambda x: x * 3),
num_parallel_workers=2)
data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x * 7, x)),
num_parallel_workers=2, columns_order=["out0", "out1", "out2"])
num_parallel_workers=2, column_order=["out0", "out1", "out2"])
data1 = data1.map(input_columns="out2", output_columns="out2", operations=(lambda x: x + 1),
num_parallel_workers=2)
......@@ -299,7 +299,7 @@ def test_generator_10():
# apply dataset operations
data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"])
data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)),
columns_order=['col0', 'out1', 'out2'], num_parallel_workers=2)
column_order=['col0', 'out1', 'out2'], num_parallel_workers=2)
# Expected column order is |col0|out1|out2|
i = 0
......@@ -318,17 +318,17 @@ def test_generator_11():
Test map column order when len(input_columns) != len(output_columns).
"""
logger.info("Test map column order when len(input_columns) != len(output_columns), "
"and columns_order drops some columns.")
"and column_order drops some columns.")
# apply dataset operations
data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"])
data1 = data1.map(input_columns="col1", output_columns=["out1", "out2"], operations=(lambda x: (x, x * 5)),
columns_order=['out1', 'out2'], num_parallel_workers=2)
column_order=['out1', 'out2'], num_parallel_workers=2)
# Expected column order is |out1|out2|
i = 0
for item in data1.create_tuple_iterator(num_epochs=1):
# len should be 2 because col0 is dropped (not included in columns_order)
# len should be 2 because col0 is dropped (not included in column_order)
assert len(item) == 2
golden = np.array([[i, i + 1], [i + 2, i + 3]])
np.testing.assert_array_equal(item[0], golden)
......@@ -358,7 +358,7 @@ def test_generator_12():
i = i + 1
data1 = ds.GeneratorDataset(generator_mc(2048), ["col0", "col1"])
data1 = data1.map(operations=(lambda x: (x * 5)), columns_order=["col1", "col0"], num_parallel_workers=2)
data1 = data1.map(operations=(lambda x: (x * 5)), column_order=["col1", "col0"], num_parallel_workers=2)
# Expected column order is |col0|col1|
i = 0
......@@ -392,7 +392,7 @@ def test_generator_13():
i = i + 1
for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary
# len should be 2 because col0 is dropped (not included in columns_order)
# len should be 2 because col0 is dropped (not included in column_order)
assert len(item) == 2
golden = np.array([i * 5])
np.testing.assert_array_equal(item["out0"], golden)
......@@ -508,7 +508,7 @@ def test_generator_error_3():
for _ in data1:
pass
assert "When (len(input_columns) != len(output_columns)), columns_order must be specified." in str(info.value)
assert "When (len(input_columns) != len(output_columns)), column_order must be specified." in str(info.value)
def test_generator_error_4():
......
tests/ut/python/dataset/test_datasets_get_dataset_size.py
浏览文件 @ 3bac9d37
......@@ -27,16 +27,16 @@ CIFAR100_DATA_DIR = "../data/dataset/testCifar100Data"
def test_imagenet_rawdata_dataset_size():
ds_total = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR)
ds_total = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR)
assert ds_total.get_dataset_size() == 6
ds_shard_1_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=1, shard_id=0)
ds_shard_1_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=1, shard_id=0)
assert ds_shard_1_0.get_dataset_size() == 6
ds_shard_2_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=2, shard_id=0)
ds_shard_2_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=2, shard_id=0)
assert ds_shard_2_0.get_dataset_size() == 3
ds_shard_3_0 = ds.ImageFolderDatasetV2(IMAGENET_RAWDATA_DIR, num_shards=3, shard_id=0)
ds_shard_3_0 = ds.ImageFolderDataset(IMAGENET_RAWDATA_DIR, num_shards=3, shard_id=0)
assert ds_shard_3_0.get_dataset_size() == 2
......
tests/ut/python/dataset/test_datasets_imagefolder.py
浏览文件 @ 3bac9d37
......@@ -24,7 +24,7 @@ def test_imagefolder_basic():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR)
data1 = ds.ImageFolderDataset(DATA_DIR)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -44,7 +44,7 @@ def test_imagefolder_numsamples():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10, num_parallel_workers=2)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10, num_parallel_workers=2)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -58,7 +58,7 @@ def test_imagefolder_numsamples():
assert num_iter == 10
random_sampler = ds.RandomSampler(num_samples=3, replacement=True)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
num_iter = 0
for item in data1.create_dict_iterator(num_epochs=1):
......@@ -67,7 +67,7 @@ def test_imagefolder_numsamples():
assert num_iter == 3
random_sampler = ds.RandomSampler(num_samples=3, replacement=False)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, num_parallel_workers=2, sampler=random_sampler)
num_iter = 0
for item in data1.create_dict_iterator(num_epochs=1):
......@@ -82,7 +82,7 @@ def test_imagefolder_numshards():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=3)
data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=3)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -102,7 +102,7 @@ def test_imagefolder_shardid():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_shards=4, shard_id=1)
data1 = ds.ImageFolderDataset(DATA_DIR, num_shards=4, shard_id=1)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -122,7 +122,7 @@ def test_imagefolder_noshuffle():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, shuffle=False)
data1 = ds.ImageFolderDataset(DATA_DIR, shuffle=False)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -142,7 +142,7 @@ def test_imagefolder_extrashuffle():
repeat_count = 2
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, shuffle=True)
data1 = ds.ImageFolderDataset(DATA_DIR, shuffle=True)
data1 = data1.shuffle(buffer_size=5)
data1 = data1.repeat(repeat_count)
......@@ -164,7 +164,7 @@ def test_imagefolder_classindex():
# apply dataset operations
class_index = {"class3": 333, "class1": 111}
data1 = ds.ImageFolderDatasetV2(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = ds.ImageFolderDataset(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = data1.repeat(repeat_count)
golden = [111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111,
......@@ -189,7 +189,7 @@ def test_imagefolder_negative_classindex():
# apply dataset operations
class_index = {"class3": -333, "class1": 111}
data1 = ds.ImageFolderDatasetV2(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = ds.ImageFolderDataset(DATA_DIR, class_indexing=class_index, shuffle=False)
data1 = data1.repeat(repeat_count)
golden = [111, 111, 111, 111, 111, 111, 111, 111, 111, 111, 111,
......@@ -214,7 +214,7 @@ def test_imagefolder_extensions():
# apply dataset operations
ext = [".jpg", ".JPEG"]
data1 = ds.ImageFolderDatasetV2(DATA_DIR, extensions=ext)
data1 = ds.ImageFolderDataset(DATA_DIR, extensions=ext)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -235,7 +235,7 @@ def test_imagefolder_decode():
# apply dataset operations
ext = [".jpg", ".JPEG"]
data1 = ds.ImageFolderDatasetV2(DATA_DIR, extensions=ext, decode=True)
data1 = ds.ImageFolderDataset(DATA_DIR, extensions=ext, decode=True)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -262,7 +262,7 @@ def test_sequential_sampler():
# apply dataset operations
sampler = ds.SequentialSampler()
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
result = []
......@@ -283,7 +283,7 @@ def test_random_sampler():
# apply dataset operations
sampler = ds.RandomSampler()
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -304,7 +304,7 @@ def test_distributed_sampler():
# apply dataset operations
sampler = ds.DistributedSampler(10, 1)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -325,7 +325,7 @@ def test_pk_sampler():
# apply dataset operations
sampler = ds.PKSampler(3)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -347,7 +347,7 @@ def test_subset_random_sampler():
# apply dataset operations
indices = [0, 1, 2, 3, 4, 5, 12, 13, 14, 15, 16, 11]
sampler = ds.SubsetRandomSampler(indices)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -369,7 +369,7 @@ def test_weighted_random_sampler():
# apply dataset operations
weights = [1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1]
sampler = ds.WeightedRandomSampler(weights, 11)
data1 = ds.ImageFolderDatasetV2(DATA_DIR, sampler=sampler)
data1 = ds.ImageFolderDataset(DATA_DIR, sampler=sampler)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -389,7 +389,7 @@ def test_imagefolder_rename():
repeat_count = 1
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10)
data1 = data1.repeat(repeat_count)
num_iter = 0
......@@ -421,8 +421,8 @@ def test_imagefolder_zip():
repeat_count = 2
# apply dataset operations
data1 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10)
data2 = ds.ImageFolderDatasetV2(DATA_DIR, num_samples=10)
data1 = ds.ImageFolderDataset(DATA_DIR, num_samples=10)
data2 = ds.ImageFolderDataset(DATA_DIR, num_samples=10)
data1 = data1.repeat(repeat_count)
# rename dataset2 for no conflict
......
tests/ut/python/dataset/test_datasets_sharding.py
浏览文件 @ 3bac9d37
......@@ -20,9 +20,9 @@ def test_imagefolder_shardings(print_res=False):
image_folder_dir = "../data/dataset/testPK/data"
def sharding_config(num_shards, shard_id, num_samples, shuffle, class_index, repeat_cnt=1):
data1 = ds.ImageFolderDatasetV2(image_folder_dir, num_samples=num_samples, num_shards=num_shards,
shard_id=shard_id,
shuffle=shuffle, class_indexing=class_index, decode=True)
data1 = ds.ImageFolderDataset(image_folder_dir, num_samples=num_samples, num_shards=num_shards,
shard_id=shard_id,
shuffle=shuffle, class_indexing=class_index, decode=True)
data1 = data1.repeat(repeat_cnt)
res = []
for item in data1.create_dict_iterator(num_epochs=1): # each data is a dictionary
......
tests/ut/python/dataset/test_datasets_voc.py
浏览文件 @ 3bac9d37
......@@ -13,7 +13,7 @@
# limitations under the License.
# ==============================================================================
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
DATA_DIR = "../data/dataset/testVOC2012"
IMAGE_SHAPE = [2268, 2268, 2268, 2268, 642, 607, 561, 596, 612, 2268]
......
tests/ut/python/dataset/test_decode.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,7 @@ Testing Decode op in DE
import cv2
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
from util import diff_mse
......
tests/ut/python/dataset/test_deviceop_cpu.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
import time
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
......
tests/ut/python/dataset/test_duplicate_op.py
浏览文件 @ 3bac9d37
......@@ -24,7 +24,7 @@ import mindspore.dataset.transforms.c_transforms as ops
def compare(array):
data = ds.NumpySlicesDataset([array], column_names="x")
array = np.array(array)
data = data.map(input_columns=["x"], output_columns=["x", "y"], columns_order=["x", "y"],
data = data.map(input_columns=["x"], output_columns=["x", "y"], column_order=["x", "y"],
operations=ops.Duplicate())
for d in data.create_dict_iterator(num_epochs=1):
np.testing.assert_array_equal(array, d["x"])
......
tests/ut/python/dataset/test_epoch_ctrl.py
浏览文件 @ 3bac9d37
......@@ -21,7 +21,7 @@ import numpy as np
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
......
tests/ut/python/dataset/test_equalize.py
浏览文件 @ 3bac9d37
......@@ -18,8 +18,9 @@ Testing Equalize op in DE
import numpy as np
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.c_transforms as C
import mindspore.dataset.vision.py_transforms as F
from mindspore import log as logger
from util import visualize_list, visualize_one_channel_dataset, diff_mse, save_and_check_md5
......@@ -36,14 +37,14 @@ def test_equalize_py(plot=False):
logger.info("Test Equalize")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = ds.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
......@@ -56,15 +57,15 @@ def test_equalize_py(plot=False):
axis=0)
# Color Equalized Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_equalize = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.Equalize(),
F.ToTensor()])
transforms_equalize = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.Equalize(),
F.ToTensor()])
ds_equalize = ds.map(input_columns="image",
operations=transforms_equalize())
operations=transforms_equalize)
ds_equalize = ds_equalize.batch(512)
......@@ -93,7 +94,7 @@ def test_equalize_c(plot=False):
logger.info("Test Equalize cpp op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = [C.Decode(), C.Resize(size=[224, 224])]
......@@ -111,7 +112,7 @@ def test_equalize_c(plot=False):
axis=0)
# Equalize Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transform_equalize = [C.Decode(), C.Resize(size=[224, 224]),
C.Equalize()]
......@@ -145,7 +146,7 @@ def test_equalize_py_c(plot=False):
logger.info("Test Equalize cpp and python op")
# equalize Images in cpp
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
......@@ -163,17 +164,17 @@ def test_equalize_py_c(plot=False):
axis=0)
# Equalize images in python
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
transforms_p_equalize = F.ComposeOp([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Equalize(),
np.array])
transforms_p_equalize = mindspore.dataset.transforms.py_transforms.Compose([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Equalize(),
np.array])
ds_p_equalize = ds.map(input_columns="image",
operations=transforms_p_equalize())
operations=transforms_p_equalize)
ds_p_equalize = ds_p_equalize.batch(512)
......@@ -204,7 +205,7 @@ def test_equalize_one_channel():
c_op = C.Equalize()
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -253,12 +254,12 @@ def test_equalize_md5_py():
logger.info("Test Equalize")
# First dataset
data1 = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
transforms = F.ComposeOp([F.Decode(),
F.Equalize(),
F.ToTensor()])
data1 = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Equalize(),
F.ToTensor()])
data1 = data1.map(input_columns="image", operations=transforms())
data1 = data1.map(input_columns="image", operations=transforms)
# Compare with expected md5 from images
filename = "equalize_01_result.npz"
save_and_check_md5(data1, filename, generate_golden=GENERATE_GOLDEN)
......@@ -271,7 +272,7 @@ def test_equalize_md5_c():
logger.info("Test Equalize cpp op with md5 check")
# Generate dataset
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_equalize = [C.Decode(),
C.Resize(size=[224, 224]),
......
tests/ut/python/dataset/test_exceptions.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
import pytest
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
from mindspore import log as logger
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
......
tests/ut/python/dataset/test_filterop.py
浏览文件 @ 3bac9d37
......@@ -16,7 +16,7 @@
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as cde
import mindspore.dataset.vision.c_transforms as cde
DATA_DIR = ["../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"]
SCHEMA_DIR = "../data/dataset/test_tf_file_3_images/datasetSchema.json"
......
tests/ut/python/dataset/test_five_crop.py
浏览文件 @ 3bac9d37
......@@ -18,7 +18,8 @@ import pytest
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as vision
from mindspore import log as logger
from util import visualize_list, save_and_check_md5
......@@ -39,8 +40,8 @@ def test_five_crop_op(plot=False):
vision.Decode(),
vision.ToTensor(),
]
transform_1 = vision.ComposeOp(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1())
transform_1 = mindspore.dataset.transforms.py_transforms.Compose(transforms_1)
data1 = data1.map(input_columns=["image"], operations=transform_1)
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
......@@ -49,8 +50,8 @@ def test_five_crop_op(plot=False):
vision.FiveCrop(200),
lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 5 images
]
transform_2 = vision.ComposeOp(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2())
transform_2 = mindspore.dataset.transforms.py_transforms.Compose(transforms_2)
data2 = data2.map(input_columns=["image"], operations=transform_2)
num_iter = 0
for item1, item2 in zip(data1.create_dict_iterator(num_epochs=1), data2.create_dict_iterator(num_epochs=1)):
......@@ -83,8 +84,8 @@ def test_five_crop_error_msg():
vision.FiveCrop(200),
vision.ToTensor()
]
transform = vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
with pytest.raises(RuntimeError) as info:
for _ in data:
......@@ -108,8 +109,8 @@ def test_five_crop_md5():
vision.FiveCrop(100),
lambda images: np.stack([vision.ToTensor()(image) for image in images]) # 4D stack of 5 images
]
transform = vision.ComposeOp(transforms)
data = data.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data = data.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "five_crop_01_result.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......
tests/ut/python/dataset/test_flat_map.py
浏览文件 @ 3bac9d37
......@@ -27,7 +27,7 @@ def test_flat_map_1():
def flat_map_func(x):
data_dir = x[0].item().decode('utf8')
d = ds.ImageFolderDatasetV2(data_dir)
d = ds.ImageFolderDataset(data_dir)
return d
data = ds.TextFileDataset(DATA_FILE)
......@@ -47,7 +47,7 @@ def test_flat_map_2():
def flat_map_func_1(x):
data_dir = x[0].item().decode('utf8')
d = ds.ImageFolderDatasetV2(data_dir)
d = ds.ImageFolderDataset(data_dir)
return d
def flat_map_func_2(x):
......
tests/ut/python/dataset/test_get_col_names.py
浏览文件 @ 3bac9d37
......@@ -15,7 +15,7 @@
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.c_transforms as vision
import mindspore.dataset.vision.c_transforms as vision
CELEBA_DIR = "../data/dataset/testCelebAData"
CIFAR10_DIR = "../data/dataset/testCifar10Data"
......@@ -75,7 +75,7 @@ def test_get_column_name_generator():
def test_get_column_name_imagefolder():
data = ds.ImageFolderDatasetV2(IMAGE_FOLDER_DIR)
data = ds.ImageFolderDataset(IMAGE_FOLDER_DIR)
assert data.get_col_names() == ["image", "label"]
......@@ -105,7 +105,7 @@ def test_get_column_name_map():
assert data.get_col_names() == ["col1", "label"]
data = ds.Cifar10Dataset(CIFAR10_DIR)
data = data.map(input_columns=["image"], operations=center_crop_op, output_columns=["col1", "col2"],
columns_order=["col2", "col1"])
column_order=["col2", "col1"])
assert data.get_col_names() == ["col2", "col1"]
......
tests/ut/python/dataset/test_get_size.py
浏览文件 @ 3bac9d37
......@@ -150,13 +150,13 @@ def test_manifest():
def test_imagefolder():
data = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/")
data = ds.ImageFolderDataset("../data/dataset/testPK/data/")
assert data.get_dataset_size() == 44
assert data.num_classes() == 4
data = data.shuffle(100)
assert data.num_classes() == 4
data = ds.ImageFolderDatasetV2("../data/dataset/testPK/data/", num_samples=10)
data = ds.ImageFolderDataset("../data/dataset/testPK/data/", num_samples=10)
assert data.get_dataset_size() == 10
assert data.num_classes() == 4
......
tests/ut/python/dataset/test_invert.py
浏览文件 @ 3bac9d37
......@@ -18,8 +18,9 @@ Testing Invert op in DE
import numpy as np
import mindspore.dataset.engine as de
import mindspore.dataset.transforms.vision.py_transforms as F
import mindspore.dataset.transforms.vision.c_transforms as C
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as F
import mindspore.dataset.vision.c_transforms as C
from mindspore import log as logger
from util import visualize_list, save_and_check_md5, diff_mse
......@@ -35,14 +36,14 @@ def test_invert_py(plot=False):
logger.info("Test Invert Python op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
transforms_original = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.ToTensor()])
ds_original = ds.map(input_columns="image",
operations=transforms_original())
operations=transforms_original)
ds_original = ds_original.batch(512)
......@@ -55,15 +56,15 @@ def test_invert_py(plot=False):
axis=0)
# Color Inverted Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_invert = F.ComposeOp([F.Decode(),
F.Resize((224, 224)),
F.Invert(),
F.ToTensor()])
transforms_invert = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Resize((224, 224)),
F.Invert(),
F.ToTensor()])
ds_invert = ds.map(input_columns="image",
operations=transforms_invert())
operations=transforms_invert)
ds_invert = ds_invert.batch(512)
......@@ -92,7 +93,7 @@ def test_invert_c(plot=False):
logger.info("Test Invert cpp op")
# Original Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_original = [C.Decode(), C.Resize(size=[224, 224])]
......@@ -110,7 +111,7 @@ def test_invert_c(plot=False):
axis=0)
# Invert Images
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transform_invert = [C.Decode(), C.Resize(size=[224, 224]),
C.Invert()]
......@@ -144,7 +145,7 @@ def test_invert_py_c(plot=False):
logger.info("Test Invert cpp and python op")
# Invert Images in cpp
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
......@@ -162,17 +163,17 @@ def test_invert_py_c(plot=False):
axis=0)
# invert images in python
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(), C.Resize((224, 224))])
transforms_p_invert = F.ComposeOp([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Invert(),
np.array])
transforms_p_invert = mindspore.dataset.transforms.py_transforms.Compose([lambda img: img.astype(np.uint8),
F.ToPIL(),
F.Invert(),
np.array])
ds_p_invert = ds.map(input_columns="image",
operations=transforms_p_invert())
operations=transforms_p_invert)
ds_p_invert = ds_p_invert.batch(512)
......@@ -203,7 +204,7 @@ def test_invert_one_channel():
c_op = C.Invert()
try:
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
ds = ds.map(input_columns=["image"],
operations=[C.Decode(),
C.Resize((224, 224)),
......@@ -224,13 +225,13 @@ def test_invert_md5_py():
logger.info("Test Invert python op with md5 check")
# Generate dataset
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_invert = F.ComposeOp([F.Decode(),
F.Invert(),
F.ToTensor()])
transforms_invert = mindspore.dataset.transforms.py_transforms.Compose([F.Decode(),
F.Invert(),
F.ToTensor()])
data = ds.map(input_columns="image", operations=transforms_invert())
data = ds.map(input_columns="image", operations=transforms_invert)
# Compare with expected md5 from images
filename = "invert_01_result_py.npz"
save_and_check_md5(data, filename, generate_golden=GENERATE_GOLDEN)
......@@ -243,7 +244,7 @@ def test_invert_md5_c():
logger.info("Test Invert cpp op with md5 check")
# Generate dataset
ds = de.ImageFolderDatasetV2(dataset_dir=DATA_DIR, shuffle=False)
ds = de.ImageFolderDataset(dataset_dir=DATA_DIR, shuffle=False)
transforms_invert = [C.Decode(),
C.Resize(size=[224, 224]),
......
tests/ut/python/dataset/test_linear_transformation.py
浏览文件 @ 3bac9d37
......@@ -17,7 +17,8 @@ Testing LinearTransformation op in DE
"""
import numpy as np
import mindspore.dataset as ds
import mindspore.dataset.transforms.vision.py_transforms as py_vision
import mindspore.dataset.transforms.py_transforms
import mindspore.dataset.vision.py_transforms as py_vision
from mindspore import log as logger
from util import diff_mse, visualize_list, save_and_check_md5
......@@ -46,11 +47,11 @@ def test_linear_transformation_op(plot=False):
py_vision.CenterCrop([height, weight]),
py_vision.ToTensor()
]
transform = py_vision.ComposeOp(transforms)
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
# First dataset
data1 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data1 = data1.map(input_columns=["image"], operations=transform())
data1 = data1.map(input_columns=["image"], operations=transform)
# Note: if transformation matrix is diagonal matrix with all 1 in diagonal,
# the output matrix in expected to be the same as the input matrix.
data1 = data1.map(input_columns=["image"],
......@@ -58,7 +59,7 @@ def test_linear_transformation_op(plot=False):
# Second dataset
data2 = ds.TFRecordDataset(DATA_DIR, SCHEMA_DIR, columns_list=["image"], shuffle=False)
data2 = data2.map(input_columns=["image"], operations=transform())
data2 = data2.map(input_columns=["image"], operations=transform)
image_transformed = []
image = []
......@@ -96,8 +97,8 @@ def test_linear_transformation_md5():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
# Compare with expected md5 from images
filename = "linear_transformation_01_result.npz"
......@@ -126,8 +127,8 @@ def test_linear_transformation_exception_01():
py_vision.ToTensor(),
py_vision.LinearTransformation(None, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except TypeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Argument transformation_matrix with value None is not of type (<class 'numpy.ndarray'>,)" in str(e)
......@@ -155,8 +156,8 @@ def test_linear_transformation_exception_02():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, None)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except TypeError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "Argument mean_vector with value None is not of type (<class 'numpy.ndarray'>,)" in str(e)
......@@ -185,8 +186,8 @@ def test_linear_transformation_exception_03():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "square matrix" in str(e)
......@@ -215,8 +216,8 @@ def test_linear_transformation_exception_04():
py_vision.ToTensor(),
py_vision.LinearTransformation(transformation_matrix, mean_vector)
]
transform = py_vision.ComposeOp(transforms)
data1 = data1.map(input_columns=["image"], operations=transform())
transform = mindspore.dataset.transforms.py_transforms.Compose(transforms)
data1 = data1.map(input_columns=["image"], operations=transform)
except ValueError as e:
logger.info("Got an exception in DE: {}".format(str(e)))
assert "should match" in str(e)
......
tests/ut/python/dataset/test_opt.py
浏览文件 @ 3bac9d37
此差异已折叠。
tests/ut/python/dataset/test_pad.py
浏览文件 @ 3bac9d37
此差异已折叠。
tests/ut/python/dataset/util.py
浏览文件 @ 3bac9d37
此差异已折叠。
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或