Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
正统之独孤求败
mindspore
提交
7322839b
M
mindspore
项目概览
正统之独孤求败
/
mindspore
与 Fork 源项目一致
Fork自
MindSpore / mindspore
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
mindspore
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
7322839b
编写于
5月 19, 2020
作者:
T
Tinazhang
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add UTs for LinearTransformation, ToPIL, ToType
上级
6f733ec1
变更
8
显示空白变更内容
内联
并排
Showing
8 changed file
with
484 addition
and
0 deletion
+484
-0
tests/ut/data/dataset/golden/linear_transformation_01_result.npz
...t/data/dataset/golden/linear_transformation_01_result.npz
+0
-0
tests/ut/data/dataset/golden/to_pil_01_result.npz
tests/ut/data/dataset/golden/to_pil_01_result.npz
+0
-0
tests/ut/data/dataset/golden/to_pil_02_result.npz
tests/ut/data/dataset/golden/to_pil_02_result.npz
+0
-0
tests/ut/data/dataset/golden/to_type_01_result.npz
tests/ut/data/dataset/golden/to_type_01_result.npz
+0
-0
tests/ut/data/dataset/golden/to_type_02_result.npz
tests/ut/data/dataset/golden/to_type_02_result.npz
+0
-0
tests/ut/python/dataset/test_linear_transformation.py
tests/ut/python/dataset/test_linear_transformation.py
+225
-0
tests/ut/python/dataset/test_to_pil.py
tests/ut/python/dataset/test_to_pil.py
+79
-0
tests/ut/python/dataset/test_to_type.py
tests/ut/python/dataset/test_to_type.py
+180
-0
未找到文件。
tests/ut/data/dataset/golden/linear_transformation_01_result.npz
0 → 100644
浏览文件 @
7322839b
文件已添加
tests/ut/data/dataset/golden/to_pil_01_result.npz
0 → 100644
浏览文件 @
7322839b
文件已添加
tests/ut/data/dataset/golden/to_pil_02_result.npz
0 → 100644
浏览文件 @
7322839b
文件已添加
tests/ut/data/dataset/golden/to_type_01_result.npz
0 → 100644
浏览文件 @
7322839b
文件已添加
tests/ut/data/dataset/golden/to_type_02_result.npz
0 → 100644
浏览文件 @
7322839b
文件已添加
tests/ut/python/dataset/test_linear_transformation.py
0 → 100644
浏览文件 @
7322839b
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Testing LinearTransformation op in DE
"""
import
numpy
as
np
import
mindspore.dataset
as
ds
import
mindspore.dataset.transforms.vision.py_transforms
as
py_vision
from
mindspore
import
log
as
logger
from
util
import
diff_mse
,
visualize
,
save_and_check_md5
GENERATE_GOLDEN
=
False
DATA_DIR
=
[
"../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"
]
SCHEMA_DIR
=
"../data/dataset/test_tf_file_3_images/datasetSchema.json"
def
test_linear_transformation_op
(
plot
=
False
):
"""
Test LinearTransformation op: verify if images transform correctly
"""
logger
.
info
(
"test_linear_transformation_01"
)
# Initialize parameters
height
=
50
weight
=
50
dim
=
3
*
height
*
weight
transformation_matrix
=
np
.
eye
(
dim
)
mean_vector
=
np
.
zeros
(
dim
)
# Define operations
transforms
=
[
py_vision
.
Decode
(),
py_vision
.
CenterCrop
([
height
,
weight
]),
py_vision
.
ToTensor
()
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
# First dataset
data1
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
# Note: if transformation matrix is diagonal matrix with all 1 in diagonal,
# the output matrix in expected to be the same as the input matrix.
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
py_vision
.
LinearTransformation
(
transformation_matrix
,
mean_vector
))
# Second dataset
data2
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
data2
=
data2
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
image_transformed
=
[]
image
=
[]
for
item1
,
item2
in
zip
(
data1
.
create_dict_iterator
(),
data2
.
create_dict_iterator
()):
image1
=
(
item1
[
"image"
].
transpose
(
1
,
2
,
0
)
*
255
).
astype
(
np
.
uint8
)
image2
=
(
item2
[
"image"
].
transpose
(
1
,
2
,
0
)
*
255
).
astype
(
np
.
uint8
)
image_transformed
.
append
(
image1
)
image
.
append
(
image2
)
mse
=
diff_mse
(
image1
,
image2
)
assert
mse
==
0
if
plot
:
visualize
(
image
,
image_transformed
)
def
test_linear_transformation_md5_01
():
"""
Test LinearTransformation op: valid params (transformation_matrix, mean_vector)
Expected to pass
"""
logger
.
info
(
"test_linear_transformation_md5_01"
)
# Initialize parameters
height
=
50
weight
=
50
dim
=
3
*
height
*
weight
transformation_matrix
=
np
.
ones
([
dim
,
dim
])
mean_vector
=
np
.
zeros
(
dim
)
# Generate dataset
data1
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
transforms
=
[
py_vision
.
Decode
(),
py_vision
.
CenterCrop
([
height
,
weight
]),
py_vision
.
ToTensor
(),
py_vision
.
LinearTransformation
(
transformation_matrix
,
mean_vector
)
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
# Compare with expected md5 from images
filename
=
"linear_transformation_01_result.npz"
save_and_check_md5
(
data1
,
filename
,
generate_golden
=
GENERATE_GOLDEN
)
def
test_linear_transformation_md5_02
():
"""
Test LinearTransformation op: transformation_matrix is not provided
Expected to raise ValueError
"""
logger
.
info
(
"test_linear_transformation_md5_02"
)
# Initialize parameters
height
=
50
weight
=
50
dim
=
3
*
height
*
weight
mean_vector
=
np
.
zeros
(
dim
)
# Generate dataset
data1
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
try
:
transforms
=
[
py_vision
.
Decode
(),
py_vision
.
CenterCrop
([
height
,
weight
]),
py_vision
.
ToTensor
(),
py_vision
.
LinearTransformation
(
None
,
mean_vector
)
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
except
ValueError
as
e
:
logger
.
info
(
"Got an exception in DE: {}"
.
format
(
str
(
e
)))
assert
"not provided"
in
str
(
e
)
def
test_linear_transformation_md5_03
():
"""
Test LinearTransformation op: mean_vector is not provided
Expected to raise ValueError
"""
logger
.
info
(
"test_linear_transformation_md5_03"
)
# Initialize parameters
height
=
50
weight
=
50
dim
=
3
*
height
*
weight
transformation_matrix
=
np
.
ones
([
dim
,
dim
])
# Generate dataset
data1
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
try
:
transforms
=
[
py_vision
.
Decode
(),
py_vision
.
CenterCrop
([
height
,
weight
]),
py_vision
.
ToTensor
(),
py_vision
.
LinearTransformation
(
transformation_matrix
,
None
)
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
except
ValueError
as
e
:
logger
.
info
(
"Got an exception in DE: {}"
.
format
(
str
(
e
)))
assert
"not provided"
in
str
(
e
)
def
test_linear_transformation_md5_04
():
"""
Test LinearTransformation op: transformation_matrix is not a square matrix
Expected to raise ValueError
"""
logger
.
info
(
"test_linear_transformation_md5_04"
)
# Initialize parameters
height
=
50
weight
=
50
dim
=
3
*
height
*
weight
transformation_matrix
=
np
.
ones
([
dim
,
dim
-
1
])
mean_vector
=
np
.
zeros
(
dim
)
# Generate dataset
data1
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
try
:
transforms
=
[
py_vision
.
Decode
(),
py_vision
.
CenterCrop
([
height
,
weight
]),
py_vision
.
ToTensor
(),
py_vision
.
LinearTransformation
(
transformation_matrix
,
mean_vector
)
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
except
ValueError
as
e
:
logger
.
info
(
"Got an exception in DE: {}"
.
format
(
str
(
e
)))
assert
"square matrix"
in
str
(
e
)
def
test_linear_transformation_md5_05
():
"""
Test LinearTransformation op: mean_vector does not match dimension of transformation_matrix
Expected to raise ValueError
"""
logger
.
info
(
"test_linear_transformation_md5_05"
)
# Initialize parameters
height
=
50
weight
=
50
dim
=
3
*
height
*
weight
transformation_matrix
=
np
.
ones
([
dim
,
dim
])
mean_vector
=
np
.
zeros
(
dim
-
1
)
# Generate dataset
data1
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
try
:
transforms
=
[
py_vision
.
Decode
(),
py_vision
.
CenterCrop
([
height
,
weight
]),
py_vision
.
ToTensor
(),
py_vision
.
LinearTransformation
(
transformation_matrix
,
mean_vector
)
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
except
ValueError
as
e
:
logger
.
info
(
"Got an exception in DE: {}"
.
format
(
str
(
e
)))
assert
"should match"
in
str
(
e
)
if
__name__
==
'__main__'
:
test_linear_transformation_op
(
True
)
test_linear_transformation_md5_01
()
test_linear_transformation_md5_02
()
test_linear_transformation_md5_03
()
test_linear_transformation_md5_04
()
test_linear_transformation_md5_05
()
tests/ut/python/dataset/test_to_pil.py
0 → 100644
浏览文件 @
7322839b
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Testing ToPIL op in DE
"""
import
mindspore.dataset
as
ds
import
mindspore.dataset.transforms.vision.c_transforms
as
c_vision
import
mindspore.dataset.transforms.vision.py_transforms
as
py_vision
from
mindspore
import
log
as
logger
from
util
import
save_and_check_md5
GENERATE_GOLDEN
=
False
DATA_DIR
=
[
"../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"
]
SCHEMA_DIR
=
"../data/dataset/test_tf_file_3_images/datasetSchema.json"
def
test_to_pil_01
():
"""
Test ToPIL Op with md5 comparison: input is already PIL image
Expected to pass
"""
logger
.
info
(
"test_to_pil_01"
)
# Generate dataset
data1
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
transforms
=
[
py_vision
.
Decode
(),
# If input is already PIL image.
py_vision
.
ToPIL
(),
py_vision
.
CenterCrop
(
375
),
py_vision
.
ToTensor
()
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
# Compare with expected md5 from images
filename
=
"to_pil_01_result.npz"
save_and_check_md5
(
data1
,
filename
,
generate_golden
=
GENERATE_GOLDEN
)
def
test_to_pil_02
():
"""
Test ToPIL Op with md5 comparison: input is not PIL image
Expected to pass
"""
logger
.
info
(
"test_to_pil_02"
)
# Generate dataset
data1
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
decode_op
=
c_vision
.
Decode
()
transforms
=
[
# If input type is not PIL.
py_vision
.
ToPIL
(),
py_vision
.
CenterCrop
(
375
),
py_vision
.
ToTensor
()
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
decode_op
)
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
# Compare with expected md5 from images
filename
=
"to_pil_02_result.npz"
save_and_check_md5
(
data1
,
filename
,
generate_golden
=
GENERATE_GOLDEN
)
if
__name__
==
"__main__"
:
test_to_pil_01
()
test_to_pil_02
()
tests/ut/python/dataset/test_to_type.py
0 → 100644
浏览文件 @
7322839b
# Copyright 2020 Huawei Technologies Co., Ltd
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================
"""
Testing ToType op in DE
"""
import
numpy
as
np
import
mindspore.dataset
as
ds
import
mindspore.dataset.transforms.vision.py_transforms
as
py_vision
from
mindspore
import
log
as
logger
from
util
import
save_and_check_md5
GENERATE_GOLDEN
=
False
DATA_DIR
=
[
"../data/dataset/test_tf_file_3_images/train-0000-of-0001.data"
]
SCHEMA_DIR
=
"../data/dataset/test_tf_file_3_images/datasetSchema.json"
def
test_to_type_op
():
"""
Test ToType Op
"""
logger
.
info
(
"test_to_type_op"
)
# First dataset
data1
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
transforms1
=
[
py_vision
.
Decode
(),
py_vision
.
ToTensor
(),
# Note: Convert the datatype from float32 to int16
py_vision
.
ToType
(
np
.
int16
)
]
transform1
=
py_vision
.
ComposeOp
(
transforms1
)
data1
=
data1
.
map
(
input_columns
=
[
"image"
],
operations
=
transform1
())
# Second dataset
data2
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
transforms2
=
[
py_vision
.
Decode
(),
py_vision
.
ToTensor
()
]
transform2
=
py_vision
.
ComposeOp
(
transforms2
)
data2
=
data2
.
map
(
input_columns
=
[
"image"
],
operations
=
transform2
())
for
item1
,
item2
in
zip
(
data1
.
create_dict_iterator
(),
data2
.
create_dict_iterator
()):
image1
=
item1
[
"image"
]
image2
=
item2
[
"image"
]
assert
isinstance
(
image1
,
np
.
ndarray
)
assert
isinstance
(
image2
,
np
.
ndarray
)
assert
image1
.
dtype
==
np
.
int16
assert
image2
.
dtype
==
np
.
float32
assert
image1
.
shape
==
image2
.
shape
def
test_to_type_01
():
"""
Test ToType Op with md5 comparison: valid input (Numpy dtype)
Expect to pass
"""
logger
.
info
(
"test_to_type_01"
)
# Generate dataset
data
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
transforms
=
[
py_vision
.
Decode
(),
py_vision
.
ToTensor
(),
# Note: Convert the datatype from float32 to int32
py_vision
.
ToType
(
np
.
int32
)
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data
=
data
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
# Compare with expected md5 from images
filename
=
"to_type_01_result.npz"
save_and_check_md5
(
data
,
filename
,
generate_golden
=
GENERATE_GOLDEN
)
def
test_to_type_02
():
"""
Test ToType Op with md5 comparison: valid input (str)
Expect to pass
"""
logger
.
info
(
"test_to_type_02"
)
# Generate dataset
data
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
transforms
=
[
py_vision
.
Decode
(),
py_vision
.
ToTensor
(),
# Note: Convert to type int
py_vision
.
ToType
(
'int'
)
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data
=
data
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
# Compare with expected md5 from images
filename
=
"to_type_02_result.npz"
save_and_check_md5
(
data
,
filename
,
generate_golden
=
GENERATE_GOLDEN
)
def
test_to_type_03
():
"""
Test ToType Op: invalid input image type
Expect to raise error
"""
logger
.
info
(
"test_to_type_03"
)
try
:
# Generate dataset
data
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
transforms
=
[
py_vision
.
Decode
(),
# Note: If the object is not numpy, e.g. PIL image, TypeError will raise
py_vision
.
ToType
(
np
.
int32
)
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data
=
data
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
except
BaseException
as
e
:
logger
.
info
(
"Got an exception in DE: {}"
.
format
(
str
(
e
)))
assert
"Numpy"
in
str
(
e
)
def
test_to_type_04
():
"""
Test ToType Op: no output_type given
Expect to raise error
"""
logger
.
info
(
"test_to_type_04"
)
try
:
# Generate dataset
data
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
transforms
=
[
py_vision
.
Decode
(),
py_vision
.
ToTensor
(),
# Note: if output_type is not explicitly given
py_vision
.
ToType
()
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data
=
data
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
except
BaseException
as
e
:
logger
.
info
(
"Got an exception in DE: {}"
.
format
(
str
(
e
)))
assert
"missing"
in
str
(
e
)
def
test_to_type_05
():
"""
Test ToType Op: invalid output_type
Expect to raise error
"""
logger
.
info
(
"test_to_type_05"
)
try
:
# Generate dataset
data
=
ds
.
TFRecordDataset
(
DATA_DIR
,
SCHEMA_DIR
,
columns_list
=
[
"image"
],
shuffle
=
False
)
transforms
=
[
py_vision
.
Decode
(),
py_vision
.
ToTensor
(),
# Note: if output_type is not explicitly given
py_vision
.
ToType
(
'invalid'
)
]
transform
=
py_vision
.
ComposeOp
(
transforms
)
data
=
data
.
map
(
input_columns
=
[
"image"
],
operations
=
transform
())
except
BaseException
as
e
:
logger
.
info
(
"Got an exception in DE: {}"
.
format
(
str
(
e
)))
assert
"data type"
in
str
(
e
)
if
__name__
==
"__main__"
:
test_to_type_op
()
test_to_type_01
()
test_to_type_02
()
test_to_type_03
()
test_to_type_04
()
test_to_type_05
()
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录