Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
MindSpore
mindspore
提交
34e42bd6
M
mindspore
项目概览
MindSpore
/
mindspore
通知
35
Star
15
Fork
15
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
mindspore
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
34e42bd6
编写于
3月 30, 2020
作者:
J
jonyguo
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
1. add more log info for dataset & mindrecord, 2. add two new testcase for MindDataset
上级
c24252b2
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
63 addition
and
10 deletion
+63
-10
mindspore/dataset/engine/iterators.py
mindspore/dataset/engine/iterators.py
+10
-10
mindspore/mindrecord/filewriter.py
mindspore/mindrecord/filewriter.py
+7
-0
tests/ut/python/dataset/test_minddataset.py
tests/ut/python/dataset/test_minddataset.py
+46
-0
未找到文件。
mindspore/dataset/engine/iterators.py
浏览文件 @
34e42bd6
...
...
@@ -66,11 +66,10 @@ def _alter_node(node):
class
Iterator
:
"""
General Iterator over a dataset.
Attributes:
dataset: Dataset to be iterated over
General Iterator over a dataset.
Attributes:
dataset: Dataset to be iterated over
"""
def
__init__
(
self
,
dataset
):
...
...
@@ -86,6 +85,7 @@ class Iterator:
root
=
self
.
__convert_node_postorder
(
self
.
dataset
)
self
.
depipeline
.
AssignRootNode
(
root
)
self
.
depipeline
.
LaunchTreeExec
()
self
.
_index
=
0
def
__is_tree_node
(
self
,
node
):
"""Check if a node is tree node."""
...
...
@@ -185,10 +185,7 @@ class Iterator:
Iterator
.
__print_local
(
input_op
,
level
+
1
)
def
print
(
self
):
"""
Print the dataset tree
"""
"""Print the dataset tree"""
self
.
__print_local
(
self
.
dataset
,
0
)
def
release
(
self
):
...
...
@@ -202,7 +199,10 @@ class Iterator:
def
__next__
(
self
):
data
=
self
.
get_next
()
if
not
data
:
if
self
.
_index
==
0
:
logger
.
warning
(
"No records available."
)
raise
StopIteration
self
.
_index
+=
1
return
data
def
get_output_shapes
(
self
):
...
...
@@ -234,7 +234,7 @@ class DictIterator(Iterator):
def
get_next
(
self
):
"""
Returns the next record in the dataset as dictionary
Returns the next record in the dataset as dictionary
Returns:
Dict, the next record in the dataset.
...
...
@@ -260,7 +260,7 @@ class TupleIterator(Iterator):
def
get_next
(
self
):
"""
Returns the next record in the dataset as a list
Returns the next record in the dataset as a list
Returns:
List, the next record in the dataset.
...
...
mindspore/mindrecord/filewriter.py
浏览文件 @
34e42bd6
...
...
@@ -328,13 +328,20 @@ class FileWriter:
self
.
_generator
.
build
()
self
.
_generator
.
write_to_db
()
mindrecord_files
=
[]
index_files
=
[]
# change the file mode to 600
for
item
in
self
.
_paths
:
if
os
.
path
.
exists
(
item
):
os
.
chmod
(
item
,
stat
.
S_IRUSR
|
stat
.
S_IWUSR
)
mindrecord_files
.
append
(
item
)
index_file
=
item
+
".db"
if
os
.
path
.
exists
(
index_file
):
os
.
chmod
(
index_file
,
stat
.
S_IRUSR
|
stat
.
S_IWUSR
)
index_files
.
append
(
index_file
)
logger
.
info
(
"The list of mindrecord files created are: {}, and the list of index files are: {}"
.
format
(
mindrecord_files
,
index_files
))
return
ret
...
...
tests/ut/python/dataset/test_minddataset.py
浏览文件 @
34e42bd6
...
...
@@ -25,6 +25,7 @@ import mindspore.dataset.transforms.vision.c_transforms as vision
import
numpy
as
np
import
pytest
from
mindspore._c_dataengine
import
InterpolationMode
from
mindspore.dataset.transforms.vision
import
Inter
from
mindspore
import
log
as
logger
import
mindspore.dataset
as
ds
...
...
@@ -151,6 +152,51 @@ def test_cv_minddataset_dataset_size(add_and_remove_cv_file):
assert
data_set
.
get_dataset_size
()
==
3
def
test_cv_minddataset_repeat_reshuffle
(
add_and_remove_cv_file
):
"""tutorial for cv minddataset."""
columns_list
=
[
"data"
,
"label"
]
num_readers
=
4
data_set
=
ds
.
MindDataset
(
CV_FILE_NAME
+
"0"
,
columns_list
,
num_readers
)
decode_op
=
vision
.
Decode
()
data_set
=
data_set
.
map
(
input_columns
=
[
"data"
],
operations
=
decode_op
,
num_parallel_workers
=
2
)
resize_op
=
vision
.
Resize
((
32
,
32
),
interpolation
=
Inter
.
LINEAR
)
data_set
=
data_set
.
map
(
input_columns
=
"data"
,
operations
=
resize_op
,
num_parallel_workers
=
2
)
data_set
=
data_set
.
batch
(
2
)
data_set
=
data_set
.
repeat
(
2
)
num_iter
=
0
labels
=
[]
for
item
in
data_set
.
create_dict_iterator
():
logger
.
info
(
"-------------- get dataset size {} -----------------"
.
format
(
num_iter
))
logger
.
info
(
"-------------- item[label]: {} ---------------------"
.
format
(
item
[
"label"
]))
logger
.
info
(
"-------------- item[data]: {} ----------------------"
.
format
(
item
[
"data"
]))
num_iter
+=
1
labels
.
append
(
item
[
"label"
])
assert
num_iter
==
10
logger
.
info
(
"repeat shuffle: {}"
.
format
(
labels
))
assert
len
(
labels
)
==
10
assert
labels
[
0
:
5
]
==
labels
[
0
:
5
]
assert
labels
[
0
:
5
]
!=
labels
[
5
:
5
]
def
test_cv_minddataset_batch_size_larger_than_records
(
add_and_remove_cv_file
):
"""tutorial for cv minddataset."""
columns_list
=
[
"data"
,
"label"
]
num_readers
=
4
data_set
=
ds
.
MindDataset
(
CV_FILE_NAME
+
"0"
,
columns_list
,
num_readers
)
decode_op
=
vision
.
Decode
()
data_set
=
data_set
.
map
(
input_columns
=
[
"data"
],
operations
=
decode_op
,
num_parallel_workers
=
2
)
resize_op
=
vision
.
Resize
((
32
,
32
),
interpolation
=
Inter
.
LINEAR
)
data_set
=
data_set
.
map
(
input_columns
=
"data"
,
operations
=
resize_op
,
num_parallel_workers
=
2
)
data_set
=
data_set
.
batch
(
32
,
drop_remainder
=
True
)
num_iter
=
0
for
item
in
data_set
.
create_dict_iterator
():
logger
.
info
(
"-------------- get dataset size {} -----------------"
.
format
(
num_iter
))
logger
.
info
(
"-------------- item[label]: {} ---------------------"
.
format
(
item
[
"label"
]))
logger
.
info
(
"-------------- item[data]: {} ----------------------"
.
format
(
item
[
"data"
]))
num_iter
+=
1
assert
num_iter
==
0
def
test_cv_minddataset_issue_888
(
add_and_remove_cv_file
):
"""issue 888 test."""
columns_list
=
[
"data"
,
"label"
]
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录