Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
wmsofts
Paddle
提交
bbeb826f
P
Paddle
项目概览
wmsofts
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
bbeb826f
编写于
11月 03, 2017
作者:
L
Luo Tao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
remove ProtoDataProvider in test_PyDataProvider.cpp
上级
b7df7f9e
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
0 addition
and
121 deletion
+0
-121
paddle/trainer/tests/pydata_provider_wrapper_dir/test_pydata_provider_wrapper.proto_data
...vider_wrapper_dir/test_pydata_provider_wrapper.proto_data
+0
-0
paddle/trainer/tests/pydata_provider_wrapper_dir/test_pydata_provider_wrapper.protolist
...ovider_wrapper_dir/test_pydata_provider_wrapper.protolist
+0
-1
paddle/trainer/tests/testPyDataWrapper.py
paddle/trainer/tests/testPyDataWrapper.py
+0
-24
paddle/trainer/tests/test_PyDataProviderWrapper.cpp
paddle/trainer/tests/test_PyDataProviderWrapper.cpp
+0
-96
未找到文件。
paddle/trainer/tests/pydata_provider_wrapper_dir/test_pydata_provider_wrapper.proto_data
已删除
100644 → 0
浏览文件 @
b7df7f9e
文件已删除
paddle/trainer/tests/pydata_provider_wrapper_dir/test_pydata_provider_wrapper.protolist
已删除
100644 → 0
浏览文件 @
b7df7f9e
./trainer/tests/pydata_provider_wrapper_dir/test_pydata_provider_wrapper.proto_data
paddle/trainer/tests/testPyDataWrapper.py
浏览文件 @
bbeb826f
...
...
@@ -20,28 +20,6 @@ import random
import
json
import
string
@
provider
(
slots
=
[
SparseNonValueSlot
(
10
),
DenseSlot
(
2
),
SparseValueSlot
(
10
),
StringSlot
(
1
),
IndexSlot
(
3
)
])
def
processNonSequenceData
(
obj
,
filename
):
with
open
(
filename
,
"rb"
)
as
f
:
for
line
in
f
:
slots_str
=
line
.
split
(
';'
)
index
=
int
(
slots_str
[
0
])
non_values
=
map
(
int
,
slots_str
[
1
].
split
()[
1
:])
dense
=
map
(
float
,
slots_str
[
2
].
split
()[
1
:])
strs
=
slots_str
[
4
].
strip
().
split
(
' '
,
1
)[
1
]
def
__values_mapper__
(
s
):
s
=
s
.
split
(
":"
)
return
int
(
s
[
0
]),
float
(
s
[
1
])
values
=
map
(
__values_mapper__
,
slots_str
[
3
].
split
()[
1
:])
yield
[
non_values
,
dense
,
values
,
strs
,
index
]
SPARSE_ID_LIMIT
=
1000
SPARSE_ID_COUNT
=
100
SEQUENCE_LIMIT
=
50
...
...
@@ -146,8 +124,6 @@ def processSubSeqAndGenerateData(obj, name):
if
__name__
==
"__main__"
:
pvd
=
processNonSequenceData
(
"test.txt"
)
print
pvd
.
getNextBatch
(
100
)
pvd
=
processSeqAndGenerateData
(
"_"
)
print
pvd
.
getNextBatch
(
100
)
pvd
=
processSubSeqAndGenerateData
(
"_"
)
...
...
paddle/trainer/tests/test_PyDataProviderWrapper.cpp
浏览文件 @
bbeb826f
...
...
@@ -25,45 +25,9 @@ limitations under the License. */
#include <unordered_set>
#include "picojson.h"
void
checkEqual
(
const
paddle
::
Argument
&
expect
,
const
paddle
::
Argument
&
actual
);
void
checkValue
(
std
::
vector
<
paddle
::
Argument
>&
arguments
,
picojson
::
array
&
arr
);
const
std
::
string
kDir
=
"./trainer/tests/pydata_provider_wrapper_dir/"
;
TEST
(
PyDataProviderWrapper
,
NoSequenceData
)
{
paddle
::
DataConfig
conf
;
conf
.
set_type
(
"py"
);
conf
.
set_load_data_module
(
std
::
string
(
"testPyDataWrapper"
));
conf
.
set_load_data_object
(
std
::
string
(
"processNonSequenceData"
));
conf
.
set_async_load_data
(
false
);
conf
.
clear_files
();
conf
.
set_files
(
kDir
+
"test_pydata_provider_wrapper.list"
);
paddle
::
DataProviderPtr
provider
(
paddle
::
DataProvider
::
create
(
conf
,
false
));
provider
->
setSkipShuffle
();
provider
->
reset
();
paddle
::
DataBatch
batchFromPy
;
provider
->
getNextBatch
(
100
,
&
batchFromPy
);
paddle
::
DataConfig
conf2
;
conf2
.
set_type
(
"proto"
);
conf2
.
set_async_load_data
(
false
);
conf2
.
clear_files
();
conf2
.
set_files
(
kDir
+
"test_pydata_provider_wrapper.protolist"
);
provider
.
reset
(
paddle
::
DataProvider
::
create
(
conf2
,
false
));
provider
->
setSkipShuffle
();
provider
->
reset
();
paddle
::
DataBatch
batchFromProto
;
provider
->
getNextBatch
(
100
,
&
batchFromProto
);
std
::
vector
<
paddle
::
Argument
>&
pyArguments
=
batchFromPy
.
getStreams
();
std
::
vector
<
paddle
::
Argument
>&
protoArguments
=
batchFromProto
.
getStreams
();
EXPECT_EQ
(
pyArguments
.
size
(),
protoArguments
.
size
());
for
(
size_t
i
=
0
;
i
<
pyArguments
.
size
();
++
i
)
{
checkEqual
(
protoArguments
[
i
],
pyArguments
[
i
]);
}
}
TEST
(
PyDataProviderWrapper
,
SequenceData
)
{
paddle
::
DataConfig
conf
;
conf
.
set_type
(
"py"
);
...
...
@@ -148,66 +112,6 @@ int main(int argc, char** argv) {
return
RUN_ALL_TESTS
();
}
void
checkEqual
(
const
paddle
::
Argument
&
expect
,
const
paddle
::
Argument
&
actual
)
{
if
(
expect
.
value
)
{
EXPECT_TRUE
(
actual
.
value
!=
nullptr
);
paddle
::
Matrix
*
e
=
expect
.
value
.
get
();
paddle
::
Matrix
*
a
=
actual
.
value
.
get
();
EXPECT_EQ
(
e
->
getWidth
(),
a
->
getWidth
());
EXPECT_EQ
(
e
->
getHeight
(),
a
->
getHeight
());
if
(
dynamic_cast
<
paddle
::
CpuSparseMatrix
*>
(
e
))
{
paddle
::
CpuSparseMatrix
*
se
=
dynamic_cast
<
paddle
::
CpuSparseMatrix
*>
(
e
);
paddle
::
CpuSparseMatrix
*
sa
=
dynamic_cast
<
paddle
::
CpuSparseMatrix
*>
(
a
);
EXPECT_EQ
(
se
->
getFormat
(),
sa
->
getFormat
());
EXPECT_EQ
(
se
->
getElementCnt
(),
sa
->
getElementCnt
());
size_t
rowSize
=
se
->
getFormat
()
==
paddle
::
SPARSE_CSC
?
se
->
getElementCnt
()
:
se
->
getHeight
()
+
1
;
size_t
colSize
=
se
->
getFormat
()
==
paddle
::
SPARSE_CSC
?
se
->
getWidth
()
+
1
:
se
->
getElementCnt
();
for
(
size_t
i
=
0
;
i
<
rowSize
;
++
i
)
{
EXPECT_EQ
(
se
->
getRows
()[
i
],
sa
->
getRows
()[
i
]);
}
for
(
size_t
i
=
0
;
i
<
colSize
;
++
i
)
{
EXPECT_EQ
(
se
->
getCols
()[
i
],
sa
->
getCols
()[
i
]);
}
if
(
se
->
getValueType
()
==
paddle
::
FLOAT_VALUE
)
{
EXPECT_EQ
(
paddle
::
FLOAT_VALUE
,
sa
->
getValueType
());
for
(
size_t
i
=
0
;
i
<
se
->
getElementCnt
();
++
i
)
{
EXPECT_EQ
(
se
->
getValue
()[
i
],
sa
->
getValue
()[
i
]);
}
}
}
else
if
(
dynamic_cast
<
paddle
::
CpuMatrix
*>
(
e
))
{
EXPECT_EQ
(
e
->
getElementCnt
(),
a
->
getElementCnt
());
for
(
size_t
i
=
0
;
i
<
e
->
getElementCnt
();
++
i
)
{
EXPECT_EQ
(
e
->
getData
()[
i
],
a
->
getData
()[
i
]);
}
}
}
if
(
expect
.
ids
)
{
EXPECT_TRUE
(
actual
.
ids
!=
nullptr
);
paddle
::
VectorT
<
int
>*
e
=
expect
.
ids
.
get
();
paddle
::
VectorT
<
int
>*
a
=
actual
.
ids
.
get
();
EXPECT_EQ
(
e
->
getSize
(),
a
->
getSize
());
for
(
size_t
i
=
0
;
i
<
e
->
getSize
();
++
i
)
{
EXPECT_EQ
(
e
->
getData
()[
i
],
a
->
getData
()[
i
]);
}
}
if
(
expect
.
strs
)
{
EXPECT_TRUE
(
actual
.
strs
!=
nullptr
);
std
::
vector
<
std
::
string
>*
e
=
expect
.
strs
.
get
();
std
::
vector
<
std
::
string
>*
a
=
actual
.
strs
.
get
();
EXPECT_EQ
(
e
->
size
(),
a
->
size
());
for
(
size_t
i
=
0
;
i
<
e
->
size
();
++
i
)
{
EXPECT_EQ
((
*
e
)[
i
],
(
*
a
)[
i
]);
}
}
}
void
checkValue
(
std
::
vector
<
paddle
::
Argument
>&
arguments
,
picojson
::
array
&
arr
)
{
// CHECK SLOT 0, Sparse Value.
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录