Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
橘蓝
Paddle
提交
c1349d98
P
Paddle
项目概览
橘蓝
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c1349d98
编写于
2月 07, 2018
作者:
F
fengjiayi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix compile errors
上级
b00cae60
变更
6
显示空白变更内容
内联
并排
Showing
6 changed file
with
31 addition
and
17 deletion
+31
-17
paddle/framework/reader.cc
paddle/framework/reader.cc
+2
-0
paddle/framework/reader.h
paddle/framework/reader.h
+9
-2
paddle/operators/CMakeLists.txt
paddle/operators/CMakeLists.txt
+1
-1
paddle/operators/create_reader_op.cc
paddle/operators/create_reader_op.cc
+12
-10
paddle/operators/read_op.cc
paddle/operators/read_op.cc
+4
-1
python/paddle/v2/fluid/tests/test_cpp_reader.py
python/paddle/v2/fluid/tests/test_cpp_reader.py
+3
-3
未找到文件。
paddle/framework/reader.cc
浏览文件 @
c1349d98
...
...
@@ -38,6 +38,8 @@ void ShuffleReader::ReadNext(std::vector<LoDTensor>* out) {
break
;
}
}
// TODO(fengjiayi): 'std::random_shuffle' can be very slow. It needs to be
// optimize.
std
::
random_shuffle
(
buffer_
.
begin
(),
buffer_
.
end
());
iteration_pos_
=
0
;
}
...
...
paddle/framework/reader.h
浏览文件 @
c1349d98
...
...
@@ -28,6 +28,8 @@ class ReaderBase {
virtual
void
ReadNext
(
std
::
vector
<
LoDTensor
>*
out
)
=
0
;
virtual
bool
HasNext
()
const
=
0
;
virtual
void
ReInit
()
=
0
;
DDim
shape
(
size_t
idx
)
const
;
std
::
vector
<
DDim
>
shapes
()
const
{
return
shapes_
;
}
void
set_shapes
(
const
std
::
vector
<
DDim
>&
shapes
)
{
shapes_
=
shapes
;
}
...
...
@@ -52,6 +54,8 @@ class DecoratedReader : public ReaderBase {
bool
HasNext
()
const
override
{
return
reader_
->
HasNext
();
}
void
ReInit
()
override
{
reader_
->
ReInit
();
}
protected:
ReaderBase
*
reader_
;
};
...
...
@@ -59,9 +63,9 @@ class DecoratedReader : public ReaderBase {
// file readers
template
<
typename
T
>
class
Random
Reade
r
:
public
FileReader
{
class
Random
DataGenerato
r
:
public
FileReader
{
public:
Random
Reade
r
(
const
std
::
vector
<
DDim
>&
shapes
,
float
min
,
float
max
)
Random
DataGenerato
r
(
const
std
::
vector
<
DDim
>&
shapes
,
float
min
,
float
max
)
:
FileReader
(
shapes
),
min_
(
min
),
max_
(
max
)
{
PADDLE_ENFORCE_LE
(
min
,
max
,
"'min' shouldn't be greater than 'max'.(%f vs %f)"
,
min
,
max
);
...
...
@@ -91,6 +95,8 @@ class RandomReader : public FileReader {
bool
HasNext
()
const
override
{
return
true
;
}
void
ReInit
()
override
{
return
;
}
private:
float
min_
;
float
max_
;
...
...
@@ -139,6 +145,7 @@ class ReaderHolder {
void
ReadNext
(
std
::
vector
<
LoDTensor
>*
out
)
{
reader_
->
ReadNext
(
out
);
}
bool
HasNext
()
const
{
return
reader_
->
HasNext
();
}
void
ReInit
()
{
reader_
->
ReInit
();
}
DDim
shape
(
size_t
idx
)
const
{
return
reader_
->
shape
(
idx
);
}
std
::
vector
<
DDim
>
shapes
()
const
{
return
reader_
->
shapes
();
}
...
...
paddle/operators/CMakeLists.txt
浏览文件 @
c1349d98
...
...
@@ -186,7 +186,7 @@ list(REMOVE_ITEM GENERAL_OPS ${DEPS_OPS})
foreach
(
src
${
GENERAL_OPS
}
)
op_library
(
${
src
}
)
endforeach
()
file
(
APPEND
${
pybind_file
}
"USE_OP(less_than);
\n
USE_OP(logical_and);
\n
USE_NO_KERNEL_OP(read_from_array);
\n
USE_NO_KERNEL_OP(create_random_
reade
r);
\n
"
)
file
(
APPEND
${
pybind_file
}
"USE_OP(less_than);
\n
USE_OP(logical_and);
\n
USE_NO_KERNEL_OP(read_from_array);
\n
USE_NO_KERNEL_OP(create_random_
data_generato
r);
\n
"
)
set
(
GLOB_OP_LIB
${
OP_LIBRARY
}
CACHE INTERNAL
"Global OP library"
)
...
...
paddle/operators/create_reader_op.cc
浏览文件 @
c1349d98
...
...
@@ -18,8 +18,8 @@
namespace
paddle
{
namespace
operators
{
st
d
::
vector
<
framework
::
DDim
>
RestoreShapes
(
const
std
::
vector
<
int
>&
shape_concat
,
const
std
::
vector
<
int
>&
ranks
)
{
st
atic
std
::
vector
<
framework
::
DDim
>
RestoreShapes
(
const
std
::
vector
<
int
>&
shape_concat
,
const
std
::
vector
<
int
>&
ranks
)
{
std
::
vector
<
framework
::
DDim
>
res
;
int
offset
=
0
;
for
(
int
len
:
ranks
)
{
...
...
@@ -69,7 +69,7 @@ class CreateReaderInferVarType : public framework::VarTypeInference {
};
template
<
typename
T
>
class
CreateRandom
Reade
rOp
:
public
framework
::
OperatorBase
{
class
CreateRandom
DataGenerato
rOp
:
public
framework
::
OperatorBase
{
public:
using
framework
::
OperatorBase
::
OperatorBase
;
void
Run
(
const
framework
::
Scope
&
scope
,
...
...
@@ -84,14 +84,15 @@ class CreateRandomReaderOp : public framework::OperatorBase {
std
::
vector
<
framework
::
DDim
>
shapes
=
RestoreShapes
(
shape_concat
,
ranks
);
auto
*
out
=
scope
.
FindVar
(
Output
(
"Out"
))
->
template
GetMutable
<
framework
::
ReaderHolder
>();
out
->
Reset
(
new
framework
::
Random
Reade
r
<
T
>
(
shapes
,
Attr
<
float
>
(
"min"
),
out
->
Reset
(
new
framework
::
Random
DataGenerato
r
<
T
>
(
shapes
,
Attr
<
float
>
(
"min"
),
Attr
<
float
>
(
"max"
)));
}
};
class
CreateRandomReaderOpMaker
:
public
framework
::
OpProtoAndCheckerMaker
{
class
CreateRandomDataGeneratorOpMaker
:
public
framework
::
OpProtoAndCheckerMaker
{
public:
CreateRandom
Reade
rOpMaker
(
OpProto
*
op_proto
,
OpAttrChecker
*
op_checker
)
CreateRandom
DataGenerato
rOpMaker
(
OpProto
*
op_proto
,
OpAttrChecker
*
op_checker
)
:
OpProtoAndCheckerMaker
(
op_proto
,
op_checker
)
{
AddOutput
(
"Out"
,
"(ReaderHolder) The created random reader."
);
AddAttr
<
std
::
vector
<
int
>>
(
"shape_concat"
,
...
...
@@ -107,7 +108,7 @@ class CreateRandomReaderOpMaker : public framework::OpProtoAndCheckerMaker {
AddAttr
<
float
>
(
"min"
,
"The lower bound of reader's uniform distribution."
);
AddAttr
<
float
>
(
"max"
,
"The upper bound of reader's uniform distribution."
);
AddComment
(
R"DOC(
CreateRandom
Reade
r Operator
CreateRandom
DataGenerato
r Operator
This Op creates a random reader.
The reader generates random data instead of really reading from files.
...
...
@@ -186,9 +187,10 @@ class CreateBatchReaderOpMaker : public framework::OpProtoAndCheckerMaker {
}
// namespace paddle
namespace
ops
=
paddle
::
operators
;
REGISTER_OPERATOR
(
create_random_reader
,
ops
::
CreateRandomReaderOp
<
float
>
,
REGISTER_OPERATOR
(
create_random_data_generator
,
ops
::
CreateRandomDataGeneratorOp
<
float
>
,
ops
::
CreateFileReaderInferShape
,
ops
::
CreateRandom
Reade
rOpMaker
,
ops
::
CreateRandom
DataGenerato
rOpMaker
,
paddle
::
framework
::
EmptyGradOpMaker
,
ops
::
CreateReaderInferVarType
);
REGISTER_OPERATOR
(
create_shuffle_reader
,
ops
::
CreateShuffleReaderOp
,
...
...
paddle/operators/read_op.cc
浏览文件 @
c1349d98
...
...
@@ -59,7 +59,10 @@ class ReadOp : public framework::OperatorBase {
framework
::
ReaderHolder
*
reader
=
scope
.
FindVar
(
Input
(
"Reader"
))
->
GetMutable
<
framework
::
ReaderHolder
>
();
if
(
!
reader
->
HasNext
())
{
return
;
reader
->
ReInit
();
PADDLE_ENFORCE
(
reader
->
HasNext
(),
"Reader can not read the next data even it has been re-initialized."
);
}
std
::
vector
<
std
::
string
>
out_arg_names
=
Outputs
(
"Out"
);
std
::
vector
<
framework
::
LoDTensor
>
ins
;
...
...
python/paddle/v2/fluid/tests/test_cpp_reader.py
浏览文件 @
c1349d98
...
...
@@ -20,11 +20,11 @@ prog = fluid.framework.Program()
block
=
prog
.
current_block
()
random_reader
=
block
.
create_var
(
type
=
fluid
.
core
.
VarDesc
.
VarType
.
READER
,
name
=
"Random
Reade
r"
)
type
=
fluid
.
core
.
VarDesc
.
VarType
.
READER
,
name
=
"Random
DataGenerato
r"
)
random_reader
.
desc
.
set_lod_levels
([
0
,
0
])
create_random_
reade
r_op
=
block
.
append_op
(
type
=
"create_random_
reade
r"
,
create_random_
data_generato
r_op
=
block
.
append_op
(
type
=
"create_random_
data_generato
r"
,
outputs
=
{
"Out"
:
random_reader
},
attrs
=
{
"shape_concat"
:
[
1
,
2
,
1
,
1
],
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录