Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
PaddlePaddle
Paddle
提交
dd3df693
P
Paddle
项目概览
PaddlePaddle
/
Paddle
接近 2 年 前同步成功
通知
2323
Star
20933
Fork
5424
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1423
列表
看板
标记
里程碑
合并请求
543
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1,423
Issue
1,423
列表
看板
标记
里程碑
合并请求
543
合并请求
543
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
dd3df693
编写于
8月 24, 2020
作者:
K
Kaipeng Deng
提交者:
GitHub
8月 24, 2020
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix RandomSampler & BatchSampler. test=develop (#26559)
* fix RandomSampler & BatchSampler. test=develop
上级
d6e888ca
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
25 addition
and
8 deletion
+25
-8
python/paddle/fluid/dataloader/batch_sampler.py
python/paddle/fluid/dataloader/batch_sampler.py
+7
-6
python/paddle/fluid/dataloader/sampler.py
python/paddle/fluid/dataloader/sampler.py
+6
-2
python/paddle/fluid/tests/unittests/test_batch_sampler.py
python/paddle/fluid/tests/unittests/test_batch_sampler.py
+12
-0
未找到文件。
python/paddle/fluid/dataloader/batch_sampler.py
浏览文件 @
dd3df693
...
@@ -16,7 +16,7 @@ from __future__ import print_function
...
@@ -16,7 +16,7 @@ from __future__ import print_function
from
__future__
import
division
from
__future__
import
division
import
numpy
as
np
import
numpy
as
np
from
.sampler
import
Sampler
,
SequenceSampler
from
.sampler
import
Sampler
,
SequenceSampler
,
RandomSampler
from
.dataset
import
Dataset
,
IterableDataset
from
.dataset
import
Dataset
,
IterableDataset
__all__
=
[
"BatchSampler"
]
__all__
=
[
"BatchSampler"
]
...
@@ -86,7 +86,6 @@ class BatchSampler(Sampler):
...
@@ -86,7 +86,6 @@ class BatchSampler(Sampler):
# init with sampler
# init with sampler
sampler = RandomSampler(RandomDataset(100))
sampler = RandomSampler(RandomDataset(100))
bs = BatchSampler(sampler=sampler,
bs = BatchSampler(sampler=sampler,
shuffle=True,
batch_size=8,
batch_size=8,
drop_last=True)
drop_last=True)
...
@@ -118,14 +117,16 @@ class BatchSampler(Sampler):
...
@@ -118,14 +117,16 @@ class BatchSampler(Sampler):
"dataset should not be a paddle.io.IterableDataset"
"dataset should not be a paddle.io.IterableDataset"
assert
sampler
is
None
,
\
assert
sampler
is
None
,
\
"should not set both dataset and sampler"
"should not set both dataset and sampler"
self
.
sampler
=
SequenceSampler
(
dataset
)
assert
isinstance
(
shuffle
,
bool
),
\
"shuffle should be a boolean value, but got {}"
.
format
(
type
(
shuffle
))
if
shuffle
:
self
.
sampler
=
RandomSampler
(
dataset
)
else
:
self
.
sampler
=
SequenceSampler
(
dataset
)
assert
isinstance
(
batch_size
,
int
)
and
batch_size
>
0
,
\
assert
isinstance
(
batch_size
,
int
)
and
batch_size
>
0
,
\
"batch_size should be a positive integer, but got {}"
.
format
(
batch_size
)
"batch_size should be a positive integer, but got {}"
.
format
(
batch_size
)
self
.
batch_size
=
batch_size
self
.
batch_size
=
batch_size
assert
isinstance
(
shuffle
,
bool
),
\
"shuffle should be a boolean value, but got {}"
.
format
(
type
(
shuffle
))
self
.
shuffle
=
shuffle
assert
isinstance
(
drop_last
,
bool
),
\
assert
isinstance
(
drop_last
,
bool
),
\
"drop_last should be a boolean value, but got {}"
.
format
(
type
(
drop_last
))
"drop_last should be a boolean value, but got {}"
.
format
(
type
(
drop_last
))
self
.
drop_last
=
drop_last
self
.
drop_last
=
drop_last
...
...
python/paddle/fluid/dataloader/sampler.py
浏览文件 @
dd3df693
...
@@ -177,7 +177,7 @@ class RandomSampler(Sampler):
...
@@ -177,7 +177,7 @@ class RandomSampler(Sampler):
def __len__(self):
def __len__(self):
return self.num_samples
return self.num_samples
sampler = RandomSampler(data_souce=RandomDataset(100))
sampler = RandomSampler(data_sou
r
ce=RandomDataset(100))
for index in sampler:
for index in sampler:
print(index)
print(index)
...
@@ -216,7 +216,11 @@ class RandomSampler(Sampler):
...
@@ -216,7 +216,11 @@ class RandomSampler(Sampler):
def
__iter__
(
self
):
def
__iter__
(
self
):
n
=
len
(
self
.
data_source
)
n
=
len
(
self
.
data_source
)
if
self
.
generator
:
if
self
.
generator
:
for
index
in
self
.
generator
:
for
i
in
range
(
self
.
num_samples
):
try
:
index
=
next
(
self
.
generator
)
except
StopIteration
:
return
yield
index
yield
index
else
:
else
:
if
self
.
replacement
:
if
self
.
replacement
:
...
...
python/paddle/fluid/tests/unittests/test_batch_sampler.py
浏览文件 @
dd3df693
...
@@ -88,6 +88,18 @@ class TestRandomSampler(unittest.TestCase):
...
@@ -88,6 +88,18 @@ class TestRandomSampler(unittest.TestCase):
rets
.
append
(
i
)
rets
.
append
(
i
)
assert
tuple
(
sorted
(
rets
))
==
tuple
(
range
(
0
,
60
))
assert
tuple
(
sorted
(
rets
))
==
tuple
(
range
(
0
,
60
))
def
test_with_generator_num_samples
(
self
):
dataset
=
RandomDataset
(
100
,
10
)
generator
=
iter
(
range
(
0
,
60
))
sampler
=
RandomSampler
(
dataset
,
generator
=
generator
,
num_samples
=
50
,
replacement
=
True
)
assert
len
(
sampler
)
==
50
rets
=
[]
for
i
in
iter
(
sampler
):
rets
.
append
(
i
)
assert
tuple
(
sorted
(
rets
))
==
tuple
(
range
(
0
,
50
))
class
TestBatchSampler
(
unittest
.
TestCase
):
class
TestBatchSampler
(
unittest
.
TestCase
):
def
setUp
(
self
):
def
setUp
(
self
):
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录