Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
magicwindyyd
mindspore
提交
cb9c6fad
M
mindspore
项目概览
magicwindyyd
/
mindspore
与 Fork 源项目一致
Fork自
MindSpore / mindspore
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
M
mindspore
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
cb9c6fad
编写于
7月 28, 2020
作者:
X
xiefangqi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix numpyslice issue
上级
7cb567eb
变更
1
显示空白变更内容
内联
并排
Showing
1 changed file
with
31 addition
and
29 deletion
+31
-29
mindspore/dataset/engine/datasets.py
mindspore/dataset/engine/datasets.py
+31
-29
未找到文件。
mindspore/dataset/engine/datasets.py
浏览文件 @
cb9c6fad
...
...
@@ -3219,33 +3219,9 @@ class GeneratorDataset(MappableDataset):
def
__init__
(
self
,
source
,
column_names
=
None
,
column_types
=
None
,
schema
=
None
,
num_samples
=
None
,
num_parallel_workers
=
1
,
shuffle
=
None
,
sampler
=
None
,
num_shards
=
None
,
shard_id
=
None
):
super
().
__init__
(
num_parallel_workers
)
self
.
source
=
source
self
.
sampler
=
_select_sampler
(
num_samples
,
sampler
,
shuffle
,
num_shards
,
shard_id
)
if
self
.
sampler
is
not
None
and
hasattr
(
source
,
"__getitem__"
):
if
isinstance
(
self
.
sampler
,
(
samplers
.
SequentialSampler
,
samplers
.
DistributedSampler
,
samplers
.
RandomSampler
,
samplers
.
SubsetRandomSampler
,
samplers
.
WeightedRandomSampler
,
samplers
.
Sampler
)):
sampler_instance
=
self
.
sampler
.
create
()
sampler_instance
.
set_num_rows
(
len
(
source
))
sampler_instance
.
initialize
()
if
num_parallel_workers
>
1
:
self
.
source
=
(
lambda
:
_cpp_sampler_fn_mp
(
sampler_instance
,
source
,
num_parallel_workers
))
else
:
self
.
source
=
(
lambda
:
_cpp_sampler_fn
(
sampler_instance
,
source
))
else
:
if
num_parallel_workers
>
1
:
self
.
source
=
(
lambda
:
_py_sampler_fn_mp
(
self
.
sampler
,
num_samples
,
source
,
num_parallel_workers
))
else
:
self
.
source
=
(
lambda
:
_py_sampler_fn
(
self
.
sampler
,
num_samples
,
source
))
else
:
try
:
iter
(
source
)
except
TypeError
:
# Use generator function if input callable
self
.
source
=
(
lambda
:
_generator_fn
(
source
,
num_samples
))
else
:
# Use iterator function if input is iterable
# Random accessible input is also iterable
self
.
source
=
(
lambda
:
_iter_fn
(
source
,
num_samples
))
self
.
num_samples
=
num_samples
if
column_names
is
not
None
and
not
isinstance
(
column_names
,
list
):
column_names
=
[
column_names
]
...
...
@@ -3310,9 +3286,35 @@ class GeneratorDataset(MappableDataset):
new_op
.
num_parallel_workers
=
copy
.
deepcopy
(
self
.
num_parallel_workers
,
memodict
)
new_op
.
column_types
=
copy
.
deepcopy
(
self
.
column_types
,
memodict
)
new_op
.
column_names
=
copy
.
deepcopy
(
self
.
column_names
,
memodict
)
new_op
.
num_samples
=
copy
.
deepcopy
(
self
.
num_samples
,
memodict
)
new_op
.
source
=
self
.
source
new_op
.
sampler
=
self
.
sampler
new_op
.
sampler
=
copy
.
deepcopy
(
self
.
sampler
)
if
new_op
.
sampler
is
not
None
and
hasattr
(
self
.
source
,
"__getitem__"
):
if
isinstance
(
new_op
.
sampler
,
(
samplers
.
SequentialSampler
,
samplers
.
DistributedSampler
,
samplers
.
RandomSampler
,
samplers
.
SubsetRandomSampler
,
samplers
.
WeightedRandomSampler
,
samplers
.
Sampler
)):
sampler_instance
=
new_op
.
sampler
.
create
()
sampler_instance
.
set_num_rows
(
len
(
self
.
source
))
sampler_instance
.
initialize
()
if
new_op
.
num_parallel_workers
>
1
:
new_op
.
source
=
(
lambda
:
_cpp_sampler_fn_mp
(
sampler_instance
,
self
.
source
,
new_op
.
num_parallel_workers
))
else
:
new_op
.
source
=
(
lambda
:
_cpp_sampler_fn
(
sampler_instance
,
self
.
source
))
else
:
if
new_op
.
num_parallel_workers
>
1
:
new_op
.
source
=
(
lambda
:
_py_sampler_fn_mp
(
new_op
.
sampler
,
new_op
.
num_samples
,
self
.
source
,
new_op
.
num_parallel_workers
))
else
:
new_op
.
source
=
(
lambda
:
_py_sampler_fn
(
new_op
.
sampler
,
new_op
.
num_samples
,
self
.
source
))
else
:
try
:
iter
(
self
.
source
)
except
TypeError
:
# Use generator function if input callable
new_op
.
source
=
(
lambda
:
_generator_fn
(
self
.
source
,
new_op
.
num_samples
))
else
:
# Use iterator function if input is iterable
# Random accessible input is also iterable
new_op
.
source
=
(
lambda
:
_iter_fn
(
self
.
source
,
new_op
.
num_samples
))
return
new_op
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录