Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
夏有凉风,冬有雪
Paddle
提交
97270b9f
P
Paddle
项目概览
夏有凉风,冬有雪
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
97270b9f
编写于
6月 26, 2017
作者:
R
root
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add convert function
上级
90a327f5
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
141 addition
and
42 deletion
+141
-42
python/paddle/v2/dataset/cifar.py
python/paddle/v2/dataset/cifar.py
+22
-7
python/paddle/v2/dataset/common.py
python/paddle/v2/dataset/common.py
+4
-1
python/paddle/v2/dataset/conll05.py
python/paddle/v2/dataset/conll05.py
+25
-11
python/paddle/v2/dataset/imdb.py
python/paddle/v2/dataset/imdb.py
+11
-0
python/paddle/v2/dataset/imikolov.py
python/paddle/v2/dataset/imikolov.py
+12
-2
python/paddle/v2/dataset/mnist.py
python/paddle/v2/dataset/mnist.py
+8
-0
python/paddle/v2/dataset/movielens.py
python/paddle/v2/dataset/movielens.py
+11
-3
python/paddle/v2/dataset/sentiment.py
python/paddle/v2/dataset/sentiment.py
+15
-5
python/paddle/v2/dataset/uci_housing.py
python/paddle/v2/dataset/uci_housing.py
+13
-5
python/paddle/v2/dataset/wmt14.py
python/paddle/v2/dataset/wmt14.py
+20
-8
未找到文件。
python/paddle/v2/dataset/cifar.py
浏览文件 @
97270b9f
...
...
@@ -31,7 +31,7 @@ images per class.
import
cPickle
import
itertools
import
numpy
from
common
import
download
import
paddle.v2.dataset.common
import
tarfile
__all__
=
[
'train100'
,
'test100'
,
'train10'
,
'test10'
]
...
...
@@ -75,7 +75,8 @@ def train100():
:rtype: callable
"""
return
reader_creator
(
download
(
CIFAR100_URL
,
'cifar'
,
CIFAR100_MD5
),
'train'
)
paddle
.
v2
.
dataset
.
common
.
download
(
CIFAR100_URL
,
'cifar'
,
CIFAR100_MD5
),
'train'
)
def
test100
():
...
...
@@ -88,7 +89,9 @@ def test100():
:return: Test reader creator.
:rtype: callable
"""
return
reader_creator
(
download
(
CIFAR100_URL
,
'cifar'
,
CIFAR100_MD5
),
'test'
)
return
reader_creator
(
paddle
.
v2
.
dataset
.
common
.
download
(
CIFAR100_URL
,
'cifar'
,
CIFAR100_MD5
),
'test'
)
def
train10
():
...
...
@@ -102,7 +105,8 @@ def train10():
:rtype: callable
"""
return
reader_creator
(
download
(
CIFAR10_URL
,
'cifar'
,
CIFAR10_MD5
),
'data_batch'
)
paddle
.
v2
.
dataset
.
common
.
download
(
CIFAR10_URL
,
'cifar'
,
CIFAR10_MD5
),
'data_batch'
)
def
test10
():
...
...
@@ -116,9 +120,20 @@ def test10():
:rtype: callable
"""
return
reader_creator
(
download
(
CIFAR10_URL
,
'cifar'
,
CIFAR10_MD5
),
'test_batch'
)
paddle
.
v2
.
dataset
.
common
.
download
(
CIFAR10_URL
,
'cifar'
,
CIFAR10_MD5
),
'test_batch'
)
def
fetch
():
download
(
CIFAR10_URL
,
'cifar'
,
CIFAR10_MD5
)
download
(
CIFAR100_URL
,
'cifar'
,
CIFAR100_MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
CIFAR10_URL
,
'cifar'
,
CIFAR10_MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
CIFAR100_URL
,
'cifar'
,
CIFAR100_MD5
)
def
convert
(
path
):
"""
Converts dataset to recordio format
"""
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
train100
(),
10
,
"cifar_train100"
)
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
test100
(),
10
,
"cifar_test100"
)
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
train10
(),
10
,
"cifar_train10"
)
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
test10
(),
10
,
"cifar_test10"
)
python/paddle/v2/dataset/common.py
浏览文件 @
97270b9f
...
...
@@ -23,7 +23,10 @@ import paddle.v2.dataset
import
cPickle
import
glob
__all__
=
[
'DATA_HOME'
,
'download'
,
'md5file'
,
'split'
,
'cluster_files_reader'
]
__all__
=
[
'DATA_HOME'
,
'download'
,
'md5file'
,
'split'
,
'cluster_files_reader'
,
'convert'
]
DATA_HOME
=
os
.
path
.
expanduser
(
'~/.cache/paddle/dataset'
)
...
...
python/paddle/v2/dataset/conll05.py
浏览文件 @
97270b9f
...
...
@@ -23,7 +23,7 @@ to initialize SRL model.
import
tarfile
import
gzip
import
itertools
from
common
import
download
import
paddle.v2.dataset.common
__all__
=
[
'test, get_dict'
,
'get_embedding'
]
...
...
@@ -182,9 +182,15 @@ def get_dict():
"""
Get the word, verb and label dictionary of Wikipedia corpus.
"""
word_dict
=
load_dict
(
download
(
WORDDICT_URL
,
'conll05st'
,
WORDDICT_MD5
))
verb_dict
=
load_dict
(
download
(
VERBDICT_URL
,
'conll05st'
,
VERBDICT_MD5
))
label_dict
=
load_dict
(
download
(
TRGDICT_URL
,
'conll05st'
,
TRGDICT_MD5
))
word_dict
=
load_dict
(
paddle
.
v2
.
dataset
.
common
.
download
(
WORDDICT_URL
,
'conll05st'
,
WORDDICT_MD5
))
verb_dict
=
load_dict
(
paddle
.
v2
.
dataset
.
common
.
download
(
VERBDICT_URL
,
'conll05st'
,
VERBDICT_MD5
))
label_dict
=
load_dict
(
paddle
.
v2
.
dataset
.
common
.
download
(
TRGDICT_URL
,
'conll05st'
,
TRGDICT_MD5
))
return
word_dict
,
verb_dict
,
label_dict
...
...
@@ -192,7 +198,7 @@ def get_embedding():
"""
Get the trained word vector based on Wikipedia corpus.
"""
return
download
(
EMB_URL
,
'conll05st'
,
EMB_MD5
)
return
paddle
.
v2
.
dataset
.
common
.
download
(
EMB_URL
,
'conll05st'
,
EMB_MD5
)
def
test
():
...
...
@@ -209,15 +215,23 @@ def test():
"""
word_dict
,
verb_dict
,
label_dict
=
get_dict
()
reader
=
corpus_reader
(
download
(
DATA_URL
,
'conll05st'
,
DATA_MD5
),
paddle
.
v2
.
dataset
.
common
.
download
(
DATA_URL
,
'conll05st'
,
DATA_MD5
),
words_name
=
'conll05st-release/test.wsj/words/test.wsj.words.gz'
,
props_name
=
'conll05st-release/test.wsj/props/test.wsj.props.gz'
)
return
reader_creator
(
reader
,
word_dict
,
verb_dict
,
label_dict
)
def
fetch
():
download
(
WORDDICT_URL
,
'conll05st'
,
WORDDICT_MD5
)
download
(
VERBDICT_URL
,
'conll05st'
,
VERBDICT_MD5
)
download
(
TRGDICT_URL
,
'conll05st'
,
TRGDICT_MD5
)
download
(
EMB_URL
,
'conll05st'
,
EMB_MD5
)
download
(
DATA_URL
,
'conll05st'
,
DATA_MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
WORDDICT_URL
,
'conll05st'
,
WORDDICT_MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
VERBDICT_URL
,
'conll05st'
,
VERBDICT_MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
TRGDICT_URL
,
'conll05st'
,
TRGDICT_MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
EMB_URL
,
'conll05st'
,
EMB_MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
DATA_URL
,
'conll05st'
,
DATA_MD5
)
def
convert
():
"""
Converts dataset to recordio format
"""
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
test
(),
10
,
"conl105_train"
)
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
test
(),
10
,
"conl105_test"
)
python/paddle/v2/dataset/imdb.py
浏览文件 @
97270b9f
...
...
@@ -166,3 +166,14 @@ def word_dict():
def
fetch
():
paddle
.
v2
.
dataset
.
common
.
download
(
URL
,
'imdb'
,
MD5
)
def
convert
():
"""
Converts dataset to recordio format
"""
word_dict
=
ds
.
imdb
.
word_dict
()
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
lambda
:
train
(
word_dict
),
10
,
"imdb_train"
)
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
lambda
:
test
(
word_dict
),
10
,
"imdb_test"
)
python/paddle/v2/dataset/imikolov.py
浏览文件 @
97270b9f
...
...
@@ -18,7 +18,7 @@ This module will download dataset from
http://www.fit.vutbr.cz/~imikolov/rnnlm/ and parse training set and test set
into paddle reader creators.
"""
import
paddle.v2.dataset.common
import
paddle.v2.dataset.common
as
common
import
collections
import
tarfile
...
...
@@ -145,4 +145,14 @@ def test(word_idx, n, data_type=DataType.NGRAM):
def
fetch
():
paddle
.
v2
.
dataset
.
common
.
download
(
URL
,
"imikolov"
,
MD5
)
common
.
download
(
URL
,
"imikolov"
,
MD5
)
def
convert
(
path
):
"""
Converts dataset to recordio format
"""
N
=
5
word_dict
=
build_dict
()
common
.
convert
(
path
,
train
(
word_dict
,
N
),
10
,
"imikolov_train"
)
common
.
convert
(
path
,
test
(
word_dict
,
N
),
10
,
"imikolov_test"
)
python/paddle/v2/dataset/mnist.py
浏览文件 @
97270b9f
...
...
@@ -113,3 +113,11 @@ def fetch():
paddle
.
v2
.
dataset
.
common
.
download
(
TRAIN_LABEL_URL
,
'mnist'
,
TRAIN_LABEL_MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
TEST_IMAGE_URL
,
'mnist'
,
TEST_IMAGE_MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
TEST_LABEL_URL
,
'mnist'
,
TRAIN_LABEL_MD5
)
def
convert
(
path
):
"""
Converts dataset to recordio format
"""
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
train
(),
10
,
"minist_train"
)
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
test
(),
10
,
"minist_test"
)
python/paddle/v2/dataset/movielens.py
浏览文件 @
97270b9f
...
...
@@ -23,7 +23,7 @@ set and test set into paddle reader creators.
"""
import
zipfile
from
common
import
download
import
paddle.v2.dataset.common
import
re
import
random
import
functools
...
...
@@ -99,7 +99,7 @@ USER_INFO = None
def
__initialize_meta_info__
():
fn
=
download
(
URL
,
"movielens"
,
MD5
)
fn
=
paddle
.
v2
.
dataset
.
common
.
download
(
URL
,
"movielens"
,
MD5
)
global
MOVIE_INFO
if
MOVIE_INFO
is
None
:
pattern
=
re
.
compile
(
r
'^(.*)\((\d+)\)$'
)
...
...
@@ -246,7 +246,15 @@ def unittest():
def
fetch
():
download
(
URL
,
"movielens"
,
MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
URL
,
"movielens"
,
MD5
)
def
convert
(
path
):
"""
Converts dataset to recordio format
"""
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
train
(),
10
,
"movielens_train"
)
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
test
(),
10
,
"movielens_test"
)
if
__name__
==
'__main__'
:
...
...
python/paddle/v2/dataset/sentiment.py
浏览文件 @
97270b9f
...
...
@@ -26,7 +26,7 @@ from itertools import chain
import
nltk
from
nltk.corpus
import
movie_reviews
import
common
import
paddle.v2.dataset.
common
__all__
=
[
'train'
,
'test'
,
'get_word_dict'
]
NUM_TRAINING_INSTANCES
=
1600
...
...
@@ -39,12 +39,13 @@ def download_data_if_not_yet():
"""
try
:
# make sure that nltk can find the data
if
common
.
DATA_HOME
not
in
nltk
.
data
.
path
:
nltk
.
data
.
path
.
append
(
common
.
DATA_HOME
)
if
paddle
.
v2
.
dataset
.
common
.
DATA_HOME
not
in
nltk
.
data
.
path
:
nltk
.
data
.
path
.
append
(
paddle
.
v2
.
dataset
.
common
.
DATA_HOME
)
movie_reviews
.
categories
()
except
LookupError
:
print
"Downloading movie_reviews data set, please wait....."
nltk
.
download
(
'movie_reviews'
,
download_dir
=
common
.
DATA_HOME
)
nltk
.
download
(
'movie_reviews'
,
download_dir
=
paddle
.
v2
.
dataset
.
common
.
DATA_HOME
)
print
"Download data set success....."
print
"Path is "
+
nltk
.
data
.
find
(
'corpora/movie_reviews'
).
path
...
...
@@ -128,4 +129,13 @@ def test():
def
fetch
():
nltk
.
download
(
'movie_reviews'
,
download_dir
=
common
.
DATA_HOME
)
nltk
.
download
(
'movie_reviews'
,
download_dir
=
paddle
.
v2
.
dataset
.
common
.
DATA_HOME
)
def
convert
(
path
):
"""
Converts dataset to recordio format
"""
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
train
,
10
,
"sentiment_train"
)
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
test
,
10
,
"sentiment_test"
)
python/paddle/v2/dataset/uci_housing.py
浏览文件 @
97270b9f
...
...
@@ -14,14 +14,14 @@
"""
UCI Housing dataset.
This module will download dataset from
This module will
paddle.v2.dataset.common.
download dataset from
https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ and
parse training set and test set into paddle reader creators.
"""
import
numpy
as
np
import
os
from
common
import
download
import
paddle.v2.dataset.common
__all__
=
[
'train'
,
'test'
]
...
...
@@ -82,7 +82,7 @@ def train():
:rtype: callable
"""
global
UCI_TRAIN_DATA
load_data
(
download
(
URL
,
'uci_housing'
,
MD5
))
load_data
(
paddle
.
v2
.
dataset
.
common
.
download
(
URL
,
'uci_housing'
,
MD5
))
def
reader
():
for
d
in
UCI_TRAIN_DATA
:
...
...
@@ -102,7 +102,7 @@ def test():
:rtype: callable
"""
global
UCI_TEST_DATA
load_data
(
download
(
URL
,
'uci_housing'
,
MD5
))
load_data
(
paddle
.
v2
.
dataset
.
common
.
download
(
URL
,
'uci_housing'
,
MD5
))
def
reader
():
for
d
in
UCI_TEST_DATA
:
...
...
@@ -112,4 +112,12 @@ def test():
def
fetch
():
download
(
URL
,
'uci_housing'
,
MD5
)
paddle
.
v2
.
dataset
.
common
.
download
(
URL
,
'uci_housing'
,
MD5
)
def
convert
(
path
):
"""
Converts dataset to recordio format
"""
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
train
(),
10
,
"uci_housing_train"
)
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
test
(),
10
,
"uci_houseing_test"
)
python/paddle/v2/dataset/wmt14.py
浏览文件 @
97270b9f
...
...
@@ -22,7 +22,7 @@ parse training set and test set into paddle reader creators.
import
tarfile
import
gzip
from
paddle.v2.dataset.common
import
download
import
paddle.v2.dataset.common
from
paddle.v2.parameters
import
Parameters
__all__
=
[
'train'
,
'test'
,
'build_dict'
]
...
...
@@ -115,7 +115,8 @@ def train(dict_size):
:rtype: callable
"""
return
reader_creator
(
download
(
URL_TRAIN
,
'wmt14'
,
MD5_TRAIN
),
'train/train'
,
dict_size
)
paddle
.
v2
.
dataset
.
common
.
download
(
URL_TRAIN
,
'wmt14'
,
MD5_TRAIN
),
'train/train'
,
dict_size
)
def
test
(
dict_size
):
...
...
@@ -130,16 +131,18 @@ def test(dict_size):
:rtype: callable
"""
return
reader_creator
(
download
(
URL_TRAIN
,
'wmt14'
,
MD5_TRAIN
),
'test/test'
,
dict_size
)
paddle
.
v2
.
dataset
.
common
.
download
(
URL_TRAIN
,
'wmt14'
,
MD5_TRAIN
),
'test/test'
,
dict_size
)
def
gen
(
dict_size
):
return
reader_creator
(
download
(
URL_TRAIN
,
'wmt14'
,
MD5_TRAIN
),
'gen/gen'
,
dict_size
)
paddle
.
v2
.
dataset
.
common
.
download
(
URL_TRAIN
,
'wmt14'
,
MD5_TRAIN
),
'gen/gen'
,
dict_size
)
def
model
():
tar_file
=
download
(
URL_MODEL
,
'wmt14'
,
MD5_MODEL
)
tar_file
=
paddle
.
v2
.
dataset
.
common
.
download
(
URL_MODEL
,
'wmt14'
,
MD5_MODEL
)
with
gzip
.
open
(
tar_file
,
'r'
)
as
f
:
parameters
=
Parameters
.
from_tar
(
f
)
return
parameters
...
...
@@ -148,7 +151,7 @@ def model():
def
get_dict
(
dict_size
,
reverse
=
True
):
# if reverse = False, return dict = {'a':'001', 'b':'002', ...}
# else reverse = true, return dict = {'001':'a', '002':'b', ...}
tar_file
=
download
(
URL_TRAIN
,
'wmt14'
,
MD5_TRAIN
)
tar_file
=
paddle
.
v2
.
dataset
.
common
.
download
(
URL_TRAIN
,
'wmt14'
,
MD5_TRAIN
)
src_dict
,
trg_dict
=
__read_to_dict__
(
tar_file
,
dict_size
)
if
reverse
:
src_dict
=
{
v
:
k
for
k
,
v
in
src_dict
.
items
()}
...
...
@@ -157,5 +160,14 @@ def get_dict(dict_size, reverse=True):
def
fetch
():
download
(
URL_TRAIN
,
'wmt14'
,
MD5_TRAIN
)
download
(
URL_MODEL
,
'wmt14'
,
MD5_MODEL
)
paddle
.
v2
.
dataset
.
common
.
download
(
URL_TRAIN
,
'wmt14'
,
MD5_TRAIN
)
paddle
.
v2
.
dataset
.
common
.
download
(
URL_MODEL
,
'wmt14'
,
MD5_MODEL
)
def
convert
(
path
):
"""
Converts dataset to recordio format
"""
dict_size
=
30000
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
train
(
dict_size
),
10
,
"wmt14_train"
)
paddle
.
v2
.
dataset
.
common
.
convert
(
path
,
test
(
dict_size
),
10
,
"wmt14_test"
)
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录