Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
DS小龙哥
Paddle
提交
c2436f2c
P
Paddle
项目概览
DS小龙哥
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
c2436f2c
编写于
5月 23, 2018
作者:
F
fengjiayi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
expose random_data_generator
上级
8655904b
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
84 addition
and
16 deletion
+84
-16
paddle/fluid/operators/reader/create_random_data_generator_op.cc
...fluid/operators/reader/create_random_data_generator_op.cc
+14
-13
python/paddle/fluid/layers/io.py
python/paddle/fluid/layers/io.py
+68
-1
python/paddle/fluid/tests/test_cpp_reader.py
python/paddle/fluid/tests/test_cpp_reader.py
+2
-2
未找到文件。
paddle/fluid/operators/reader/create_random_data_generator_op.cc
浏览文件 @
c2436f2c
...
...
@@ -21,14 +21,15 @@ namespace reader {
template
<
typename
T
>
class
RandomDataGenerator
:
public
framework
::
ReaderBase
{
public:
RandomDataGenerator
(
const
std
::
vector
<
framework
::
DDim
>&
shapes
,
float
min
,
float
max
)
:
framework
::
ReaderBase
(),
min_
(
min
),
max_
(
max
),
shapes_
(
shapes
)
{
PADDLE_ENFORCE_LE
(
min
,
max
,
"'min' shouldn't be greater than 'max'.(%f vs %f)"
,
min
,
max
);
RandomDataGenerator
(
const
std
::
vector
<
framework
::
DDim
>&
shapes
,
float
low
,
float
high
)
:
framework
::
ReaderBase
(),
low_
(
low
),
high_
(
high
),
shapes_
(
shapes
)
{
PADDLE_ENFORCE_LE
(
low
,
high
,
"'low' shouldn't be greater than 'high'.(%f vs %f)"
,
low
,
high
);
unsigned
int
seed
=
std
::
random_device
()();
engine_
.
seed
(
seed
);
dist_
=
std
::
uniform_real_distribution
<
float
>
(
min_
,
max
_
);
dist_
=
std
::
uniform_real_distribution
<
float
>
(
low_
,
high
_
);
}
void
ReadNext
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
{
...
...
@@ -53,8 +54,8 @@ class RandomDataGenerator : public framework::ReaderBase {
void
ReInit
()
override
{
return
;
}
private:
float
min
_
;
float
max
_
;
float
low
_
;
float
high
_
;
std
::
minstd_rand
engine_
;
std
::
uniform_real_distribution
<
float
>
dist_
;
std
::
vector
<
framework
::
DDim
>
shapes_
;
...
...
@@ -78,22 +79,22 @@ class CreateRandomDataGeneratorOp : public framework::OperatorBase {
std
::
vector
<
framework
::
DDim
>
shapes
=
RestoreShapes
(
shape_concat
,
ranks
);
auto
*
out
=
scope
.
FindVar
(
Output
(
"Out"
))
->
template
GetMutable
<
framework
::
ReaderHolder
>();
out
->
Reset
(
new
RandomDataGenerator
<
T
>
(
shapes
,
Attr
<
float
>
(
"
min
"
),
Attr
<
float
>
(
"
max
"
)));
out
->
Reset
(
new
RandomDataGenerator
<
T
>
(
shapes
,
Attr
<
float
>
(
"
low
"
),
Attr
<
float
>
(
"
high
"
)));
}
};
class
CreateRandomDataGeneratorOpMaker
:
public
FileReaderMakerBase
{
protected:
void
Apply
()
override
{
AddAttr
<
float
>
(
"
min
"
,
"The lower bound of reader's uniform distribution."
);
AddAttr
<
float
>
(
"
max
"
,
"The upper bound of reader's uniform distribution."
);
AddAttr
<
float
>
(
"
low
"
,
"The lower bound of reader's uniform distribution."
);
AddAttr
<
float
>
(
"
high
"
,
"The upper bound of reader's uniform distribution."
);
AddComment
(
R"DOC(
CreateRandomDataGenerator Operator
This Op creates a random reader.
The reader generates random data instead of really reading from files.
Generated data follow an uniform distribution between '
min' and 'max
'.
Generated data follow an uniform distribution between '
low' and 'high
'.
)DOC"
);
}
};
...
...
python/paddle/fluid/layers/io.py
浏览文件 @
c2436f2c
...
...
@@ -321,7 +321,7 @@ def open_recordio_file(filename,
dtypes=['float32', 'int64'])
# Via the reader, we can use 'read_file' layer to get data:
image, label = fluid.layers.read_file(reader)
image, label = fluid.layers.
io.
read_file(reader)
"""
dtypes
=
[
convert_np_dtype_to_dtype_
(
dt
)
for
dt
in
dtypes
]
shape_concat
=
[]
...
...
@@ -359,6 +359,73 @@ def open_recordio_file(filename,
return
monkey_patch_reader_methods
(
main_prog_var
)
def
random_data_generator
(
low
,
high
,
shapes
,
lod_levels
,
for_parallel
=
True
):
"""
Create a uniform random data generator
This layer returns a Reader Variable.
Instead of opening a file and reading data from it, this
Reader Variable generates float uniform random data by itself.
It can be used as a dummy reader to test a network without
opening a real file.
Args:
low(float): The lower bound of data's uniform distribution.
high(float): The upper bound of data's uniform distribution.
shapes(list): List of tuples which declaring data shapes.
lod_levels(list): List of ints which declaring data lod_level.
for_parallel(Bool): Set it as True if you are going to run
subsequent operators in parallel.
Returns:
Variable: A Reader Variable from which we can get random data.
Examples:
.. code-block:: python
reader = fluid.layers.io.random_data_generator(
low=0.0,
high=1.0,
shapes=[(3,224,224), (1)],
lod_levels=[0, 0])
# Via the reader, we can use 'read_file' layer to get data:
image, label = fluid.layers.io.read_file(reader)
"""
dtypes
=
[
core
.
VarDesc
.
VarType
.
FP32
]
*
len
(
shapes
)
shape_concat
=
[]
ranks
=
[]
for
shape
in
shapes
:
shape_concat
.
extend
(
shape
)
ranks
.
append
(
len
(
shape
))
var_name
=
unique_name
(
'random_data_generator'
)
startup_blk
=
default_startup_program
().
current_block
()
startup_var
=
startup_blk
.
create_var
(
name
=
var_name
)
startup_blk
.
append_op
(
type
=
'create_random_data_generator'
,
outputs
=
{
'Out'
:
[
startup_var
]},
attrs
=
{
'low'
:
low
,
'high'
:
high
,
'shape_concat'
:
shape_concat
,
'lod_levels'
:
lod_levels
,
'ranks'
:
ranks
})
startup_var
.
desc
.
set_dtypes
(
dtypes
)
startup_var
.
persistable
=
True
main_prog_var
=
_copy_reader_var_
(
default_main_program
().
current_block
(),
startup_var
)
if
for_parallel
:
main_prog_var
=
parallel
(
reader
=
main_prog_var
)
return
monkey_patch_reader_methods
(
main_prog_var
)
def
open_files
(
filenames
,
shapes
,
lod_levels
,
...
...
python/paddle/fluid/tests/test_cpp_reader.py
浏览文件 @
c2436f2c
...
...
@@ -44,8 +44,8 @@ create_random_data_generator_op = startup_block.append_op(
attrs
=
{
"shape_concat"
:
[
1
,
2
,
1
,
1
],
"ranks"
:
[
2
,
2
],
"
min
"
:
0.0
,
"
max
"
:
1.0
,
"
low
"
:
0.0
,
"
high
"
:
1.0
,
'lod_levels'
:
[
0
,
0
]
})
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录