Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
机器未来
Paddle
提交
93cab641
P
Paddle
项目概览
机器未来
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
93cab641
编写于
2月 01, 2018
作者:
F
fengjiayi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Complete CreateRandomReaderOp
上级
d8cc21da
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
111 addition
and
16 deletion
+111
-16
paddle/framework/reader.h
paddle/framework/reader.h
+21
-16
paddle/operators/create_reader_op.cc
paddle/operators/create_reader_op.cc
+90
-0
未找到文件。
paddle/framework/reader.h
浏览文件 @
93cab641
...
...
@@ -33,8 +33,6 @@ class ReaderBase {
class
FileReader
:
public
ReaderBase
{
public:
explicit
FileReader
(
const
std
::
vector
<
DDim
>&
shapes
)
:
shapes_
(
shapes
)
{}
DDim
shape
(
size_t
idx
)
const
override
;
std
::
vector
<
DDim
>
shapes
()
const
override
{
return
shapes_
;
}
...
...
@@ -44,8 +42,6 @@ class FileReader : public ReaderBase {
class
ReaderDecorator
:
public
ReaderBase
{
public:
explicit
ReaderDecorator
(
ReaderBase
*
reader
)
:
reader_
(
reader
)
{}
bool
HasNext
()
const
override
{
return
reader_
->
HasNext
();
}
DDim
shape
(
size_t
idx
)
const
override
{
return
reader_
->
shape
(
idx
);
}
...
...
@@ -60,19 +56,19 @@ class ReaderDecorator : public ReaderBase {
template
<
typename
T
>
class
RandomReader
:
public
FileReader
{
public:
RandomReader
(
const
std
::
vector
<
DDim
>&
shapes
,
float
min
,
float
max
)
:
FileReader
(
shapes
),
min_
(
min
),
max_
(
max
)
{
void
Initialize
(
const
std
::
vector
<
DDim
>&
shapes
,
float
min
,
float
max
)
{
PADDLE_ENFORCE_LE
(
min
,
max
,
"'min' should be less than or equal to 'max'.(%f vs %f)"
,
min
,
max
);
shapes_
=
shapes
;
min_
=
min
;
max_
=
max
;
unsigned
int
seed
=
std
::
random_device
()();
engine_
.
seed
(
seed
);
dist_
=
std
::
uniform_real_distribution
<
float
>
(
min_
,
max_
);
}
std
::
vector
<
LoDTensor
>
ReadNext
()
override
{
std
::
minstd_rand
engine
;
unsigned
int
seed
=
std
::
random_device
()();
engine
.
seed
(
seed
);
std
::
uniform_real_distribution
<
float
>
dist
(
min_
,
max_
);
std
::
vector
<
LoDTensor
>
res
;
res
.
reserve
(
shapes_
.
size
());
for
(
const
DDim
&
shape
:
shapes_
)
{
...
...
@@ -85,7 +81,7 @@ class RandomReader : public FileReader {
T
*
data
=
out
.
mutable_data
<
T
>
(
platform
::
CPUPlace
());
int64_t
numel
=
product
(
shape
);
for
(
int64_t
i
=
0
;
i
<
numel
;
++
i
)
{
data
[
i
]
=
dist
(
engine
);
data
[
i
]
=
dist
_
(
engine_
);
}
res
.
push_back
(
out
);
}
...
...
@@ -97,16 +93,21 @@ class RandomReader : public FileReader {
private:
float
min_
;
float
max_
;
std
::
minstd_rand
engine_
;
std
::
uniform_real_distribution
<
float
>
dist_
;
};
// decorators
class
ShuffleReader
:
public
ReaderDecorator
{
public:
ShuffleReader
(
ReaderBase
*
reader
,
int
buffer_size
)
:
ReaderDecorator
(
reader
),
buffer_size_
(
buffer_size
),
iteration_pos_
(
0
)
{
void
Initialize
(
ReaderBase
*
reader
,
int
buffer_size
)
{
reader_
=
reader
;
buffer_size_
=
buffer_size
;
iteration_pos_
=
0
;
buffer_
.
reserve
(
buffer_size
);
}
std
::
vector
<
LoDTensor
>
ReadNext
()
override
;
private:
...
...
@@ -117,8 +118,12 @@ class ShuffleReader : public ReaderDecorator {
class
BatchReader
:
public
ReaderDecorator
{
public:
BatchReader
(
ReaderBase
*
reader
,
int
batch_size
)
:
ReaderDecorator
(
reader
),
batch_size_
(
batch_size
)
{}
void
Initialize
(
ReaderBase
*
reader
,
int
batch_size
)
{
reader_
=
reader
;
batch_size_
=
batch_size
;
buffer_
.
reserve
(
batch_size_
);
}
std
::
vector
<
LoDTensor
>
ReadNext
()
override
;
private:
...
...
paddle/operators/create_reader_op.cc
0 → 100644
浏览文件 @
93cab641
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/framework/op_registry.h"
#include "paddle/framework/reader.h"
namespace
paddle
{
namespace
operators
{
// general infershape
class
CreateReaderInferShape
:
public
framework
::
InferShapeBase
{
public:
void
operator
()(
framework
::
InferShapeContext
*
ctx
)
const
override
{
PADDLE_ENFORCE
(
ctx
->
HasOutput
(
"Out"
),
"Output(Out) of CreateReaderOp should not be null."
);
}
};
template
<
typename
T
>
class
CreateRandomReaderOp
:
public
framework
::
OperatorBase
{
public:
using
framework
::
OperatorBase
::
OperatorBase
;
void
Run
(
const
framework
::
Scope
&
scope
,
const
platform
::
Place
&
dev_place
)
const
override
{
const
auto
&
shape_concat
=
Attr
<
std
::
vector
<
int
>>
(
"shape_concat"
);
const
auto
&
ranks
=
Attr
<
std
::
vector
<
int
>>
(
"ranks"
);
PADDLE_ENFORCE_EQ
(
std
::
accumulate
(
ranks
.
begin
(),
ranks
.
end
(),
0
),
int
(
shape_concat
.
size
()),
"The accumulate of all ranks should be equal to the "
"shape concat's length."
);
std
::
vector
<
framework
::
DDim
>
shapes
;
int
offset
=
0
;
for
(
int
len
:
ranks
)
{
auto
start_it
=
shape_concat
.
begin
()
+
offset
;
auto
end_it
=
start_it
+
len
;
shapes
.
push_back
(
framework
::
make_ddim
(
std
::
vector
<
int
>
(
start_it
,
end_it
)));
offset
+=
len
;
}
auto
*
out
=
scope
.
FindVar
(
Output
(
"Out"
))
->
template
GetMutable
<
framework
::
RandomReader
<
T
>
>
();
out
->
Initialize
(
shapes
,
Attr
<
float
>
(
"min"
),
Attr
<
float
>
(
"max"
));
}
};
class
CreateRandomReaderOpMaker
:
public
framework
::
OpProtoAndCheckerMaker
{
public:
CreateRandomReaderOpMaker
(
OpProto
*
op_proto
,
OpAttrChecker
*
op_checker
)
:
OpProtoAndCheckerMaker
(
op_proto
,
op_checker
)
{
AddOutput
(
"Out"
,
"(RandomReader) The created random reader."
);
AddAttr
<
std
::
vector
<
int
>>
(
"shape_concat"
,
"The concat of all data's shapes."
);
AddAttr
<
std
::
vector
<
int
>>
(
"ranks"
,
"The ranks of each data."
"e.g."
"shape_concat = [2,3,4,5,6]"
"ranks = [3,2]"
"It means the reader will generate two data each time,"
"whose shapes are [2,3,4] and [5,6] respectively."
);
AddAttr
<
float
>
(
"min"
,
"The lower bound of reader's uniform distribution."
);
AddAttr
<
float
>
(
"max"
,
"The upper bound of reader's uniform distribution."
);
AddComment
(
R"DOC(
CreateRandomReader Operator
This Op creates a random reader.
The reader generates random data instead of really reading from files.
Generated data follow an uniform distribution between 'min' and 'max'.
)DOC"
);
}
};
}
// namespace operators
}
// namespace paddle
namespace
ops
=
paddle
::
operators
;
REGISTER_OPERATOR
(
create_random_reader
,
ops
::
CreateRandomReaderOp
<
float
>
,
ops
::
CreateReaderInferShape
,
ops
::
CreateRandomReaderOpMaker
,
paddle
::
framework
::
EmptyGradOpMaker
);
\ No newline at end of file
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录