Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
csdn_franckjun
Paddle
提交
4594f186
P
Paddle
项目概览
csdn_franckjun
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
4594f186
编写于
7月 10, 2018
作者:
W
Wojciech Uss
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
added cycling the dataset for cifar and flowers
上级
26ae6111
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
52 addition
and
25 deletion
+52
-25
python/paddle/dataset/cifar.py
python/paddle/dataset/cifar.py
+18
-9
python/paddle/dataset/flowers.py
python/paddle/dataset/flowers.py
+34
-16
未找到文件。
python/paddle/dataset/cifar.py
浏览文件 @
4594f186
...
...
@@ -43,7 +43,7 @@ CIFAR100_URL = URL_PREFIX + 'cifar-100-python.tar.gz'
CIFAR100_MD5
=
'eb9058c3a382ffc7106e4002c42a8d85'
def
reader_creator
(
filename
,
sub_name
):
def
reader_creator
(
filename
,
sub_name
,
cycle
=
False
):
def
read_batch
(
batch
):
data
=
batch
[
'data'
]
labels
=
batch
.
get
(
'labels'
,
batch
.
get
(
'fine_labels'
,
None
))
...
...
@@ -56,10 +56,13 @@ def reader_creator(filename, sub_name):
names
=
(
each_item
.
name
for
each_item
in
f
if
sub_name
in
each_item
.
name
)
for
name
in
names
:
batch
=
cPickle
.
load
(
f
.
extractfile
(
name
))
for
item
in
read_batch
(
batch
):
yield
item
while
True
:
for
name
in
names
:
batch
=
cPickle
.
load
(
f
.
extractfile
(
name
))
for
item
in
read_batch
(
batch
):
yield
item
if
not
cycle
:
break
return
reader
...
...
@@ -94,34 +97,40 @@ def test100():
'test'
)
def
train10
():
def
train10
(
cycle
=
False
):
"""
CIFAR-10 training set creator.
It returns a reader creator, each sample in the reader is image pixels in
[0, 1] and label in [0, 9].
:param cycle: whether to cycle through the dataset
:type cycle: bool
:return: Training reader creator
:rtype: callable
"""
return
reader_creator
(
paddle
.
dataset
.
common
.
download
(
CIFAR10_URL
,
'cifar'
,
CIFAR10_MD5
),
'data_batch'
)
'data_batch'
,
cycle
=
cycle
)
def
test10
():
def
test10
(
cycle
=
False
):
"""
CIFAR-10 test set creator.
It returns a reader creator, each sample in the reader is image pixels in
[0, 1] and label in [0, 9].
:param cycle: whether to cycle through the dataset
:type cycle: bool
:return: Test reader creator.
:rtype: callable
"""
return
reader_creator
(
paddle
.
dataset
.
common
.
download
(
CIFAR10_URL
,
'cifar'
,
CIFAR10_MD5
),
'test_batch'
)
'test_batch'
,
cycle
=
cycle
)
def
fetch
():
...
...
python/paddle/dataset/flowers.py
浏览文件 @
4594f186
...
...
@@ -76,7 +76,8 @@ def reader_creator(data_file,
dataset_name
,
mapper
,
buffered_size
=
1024
,
use_xmap
=
True
):
use_xmap
=
True
,
cycle
=
False
):
'''
1. read images from tar file and
merge images into batch files in 102flowers.tgz_batch/
...
...
@@ -96,6 +97,8 @@ def reader_creator(data_file,
:type mapper: callable
:param buffered_size: the size of buffer used to process images
:type buffered_size: int
:param cycle: whether to cycle through the dataset
:type cycle: bool
:return: data reader
:rtype: callable
'''
...
...
@@ -108,15 +111,18 @@ def reader_creator(data_file,
file_list
=
batch_images_from_tar
(
data_file
,
dataset_name
,
img2label
)
def
reader
():
for
file
in
open
(
file_list
):
file
=
file
.
strip
()
batch
=
None
with
open
(
file
,
'r'
)
as
f
:
batch
=
cPickle
.
load
(
f
)
data
=
batch
[
'data'
]
labels
=
batch
[
'label'
]
for
sample
,
label
in
itertools
.
izip
(
data
,
batch
[
'label'
]):
yield
sample
,
int
(
label
)
-
1
while
True
:
for
file
in
open
(
file_list
):
file
=
file
.
strip
()
batch
=
None
with
open
(
file
,
'r'
)
as
f
:
batch
=
cPickle
.
load
(
f
)
data
=
batch
[
'data'
]
labels
=
batch
[
'label'
]
for
sample
,
label
in
itertools
.
izip
(
data
,
batch
[
'label'
]):
yield
sample
,
int
(
label
)
-
1
if
not
cycle
:
break
if
use_xmap
:
cpu_num
=
int
(
os
.
environ
.
get
(
'CPU_NUM'
,
cpu_count
()))
...
...
@@ -125,7 +131,7 @@ def reader_creator(data_file,
return
map_readers
(
mapper
,
reader
)
def
train
(
mapper
=
train_mapper
,
buffered_size
=
1024
,
use_xmap
=
True
):
def
train
(
mapper
=
train_mapper
,
buffered_size
=
1024
,
use_xmap
=
True
,
cycle
=
False
):
'''
Create flowers training set reader.
It returns a reader, each sample in the reader is
...
...
@@ -138,17 +144,23 @@ def train(mapper=train_mapper, buffered_size=1024, use_xmap=True):
:type mapper: callable
:param buffered_size: the size of buffer used to process images
:type buffered_size: int
:param cycle: whether to cycle through the dataset
:type cycle: bool
:return: train data reader
:rtype: callable
'''
return
reader_creator
(
download
(
DATA_URL
,
'flowers'
,
DATA_MD5
),
download
(
LABEL_URL
,
'flowers'
,
LABEL_MD5
),
download
(
SETID_URL
,
'flowers'
,
SETID_MD5
),
TRAIN_FLAG
,
mapper
,
buffered_size
,
use_xmap
)
download
(
SETID_URL
,
'flowers'
,
SETID_MD5
),
TRAIN_FLAG
,
mapper
,
buffered_size
,
use_xmap
,
cycle
=
cycle
)
def
test
(
mapper
=
test_mapper
,
buffered_size
=
1024
,
use_xmap
=
True
):
def
test
(
mapper
=
test_mapper
,
buffered_size
=
1024
,
use_xmap
=
True
,
cycle
=
False
):
'''
Create flowers test set reader.
It returns a reader, each sample in the reader is
...
...
@@ -161,14 +173,20 @@ def test(mapper=test_mapper, buffered_size=1024, use_xmap=True):
:type mapper: callable
:param buffered_size: the size of buffer used to process images
:type buffered_size: int
:param cycle: whether to cycle through the dataset
:type cycle: bool
:return: test data reader
:rtype: callable
'''
return
reader_creator
(
download
(
DATA_URL
,
'flowers'
,
DATA_MD5
),
download
(
LABEL_URL
,
'flowers'
,
LABEL_MD5
),
download
(
SETID_URL
,
'flowers'
,
SETID_MD5
),
TEST_FLAG
,
mapper
,
buffered_size
,
use_xmap
)
download
(
SETID_URL
,
'flowers'
,
SETID_MD5
),
TEST_FLAG
,
mapper
,
buffered_size
,
use_xmap
,
cycle
=
cycle
)
def
valid
(
mapper
=
test_mapper
,
buffered_size
=
1024
,
use_xmap
=
True
):
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录