Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
wk1234123
Paddle
提交
f32ca636
P
Paddle
项目概览
wk1234123
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
4
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
f32ca636
编写于
1月 31, 2018
作者:
F
fengjiayi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
draft of Reader classes
上级
1acad21b
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
159 addition
and
33 deletion
+159
-33
paddle/framework/CMakeLists.txt
paddle/framework/CMakeLists.txt
+2
-0
paddle/framework/reader.cc
paddle/framework/reader.cc
+86
-21
paddle/framework/reader.h
paddle/framework/reader.h
+71
-12
未找到文件。
paddle/framework/CMakeLists.txt
浏览文件 @
f32ca636
...
@@ -24,6 +24,8 @@ cc_library(lod_tensor SRCS lod_tensor.cc DEPS ddim place tensor framework_proto)
...
@@ -24,6 +24,8 @@ cc_library(lod_tensor SRCS lod_tensor.cc DEPS ddim place tensor framework_proto)
cc_test
(
lod_tensor_test SRCS lod_tensor_test.cc DEPS lod_tensor paddle_memory
)
cc_test
(
lod_tensor_test SRCS lod_tensor_test.cc DEPS lod_tensor paddle_memory
)
nv_test
(
lod_tensor_gpu_test SRCS lod_tensor_test.cu DEPS lod_tensor
)
nv_test
(
lod_tensor_gpu_test SRCS lod_tensor_test.cu DEPS lod_tensor
)
cc_library
(
reader SRCS reader.cc DEPS lod_tensor ddim
)
cc_test
(
variable_test SRCS variable_test.cc
)
cc_test
(
variable_test SRCS variable_test.cc
)
cc_library
(
threadpool SRCS threadpool.cc DEPS enforce
)
cc_library
(
threadpool SRCS threadpool.cc DEPS enforce
)
...
...
paddle/framework/reader.cc
浏览文件 @
f32ca636
...
@@ -17,35 +17,100 @@
...
@@ -17,35 +17,100 @@
namespace
paddle
{
namespace
paddle
{
namespace
framework
{
namespace
framework
{
DDim
Reader
::
shape
(
in
t
idx
)
const
{
DDim
Reader
::
shape
(
size_
t
idx
)
const
{
PADDLE_ENFORCE_LT
(
PADDLE_ENFORCE_LT
(
idx
,
shapes_
.
size
(),
idx
,
shapes_
.
size
(),
"Cannot get the %d'th shape, 'shapes_' only has %d elements."
,
idx
,
"Cannot get the %d'th shape, 'shapes_' only has %d elements."
,
idx
,
shapes_
.
size
());
shapes_
.
size
());
return
shapes_
[
idx
];
}
}
int
RandomReader
::
ReadNext
(
std
::
vector
<
LoDTensor
>*
outs
)
{
std
::
vector
<
LoDTensor
>
ShuffleReader
::
ReadNext
()
{
PADDLE_ENFORCE_EQ
(
if
(
iteration_pos_
>=
buffer_
.
size
())
{
shapes_
.
size
(),
outs
.
size
(),
// Reload buffer with new data
"shapes_.size() is %d, while outs.size() is %d. They are not equal."
,
buffer_
.
clear
();
shapes_
.
size
(),
outs
.
size
());
for
(
int
i
=
0
;
i
<
buffer_size_
;
++
i
)
{
std
::
minstd_rand
engine
;
if
(
reader_
->
HasNext
())
{
unsigned
int
seed
=
std
::
random_device
()();
buffer_
.
push_back
(
reader_
->
ReadNext
());
engine
.
seed
(
seed
);
}
else
{
std
::
uniform_real_distribution
<
float
>
dist
(
min_
,
max_
);
break
;
for
(
int
idx
=
0
;
idx
<
shapes_
.
size
();
++
idx
)
{
}
DDim
shape
=
shapes_
[
idx
];
}
LoDTensor
*
out
=
outs
[
idx
];
std
::
random_shuffle
(
buffer_
.
begin
(),
buffer_
.
end
());
int64_t
numel
=
out
->
numel
();
iteration_pos_
=
0
;
PADDLE_ENFORCE_EQ
(
product
(
shape
),
numel
,
}
"The product of %d'th shape is %lld, while the "
if
(
buffer_
.
empty
())
{
"corresponding out's numel is %lld. They are not equal."
,
std
::
vector
<
LoDTensor
>
empty_res
;
idx
,
product
(
shape
),
numel
);
return
empty_res
;
for
(
int64_t
i
=
0
;
i
<
numel
,
++
i
)
{
}
out
[
i
]
=
dist
(
engine
);
return
buffer_
[
iteration_pos_
++
];
}
}
}
return
0
;
std
::
vector
<
LoDTensor
>
BatchReader
::
ReadNext
()
{
buffer_
.
clear
();
for
(
int
i
=
0
;
i
<
batch_size_
;
++
i
)
{
if
(
reader_
->
HasNext
())
{
buffer_
.
push_back
(
reader_
->
ReadNext
());
}
else
{
break
;
}
}
// Concat instances
std
::
vector
<
LoDTensor
>
res
;
if
(
buffer_
.
empty
())
{
return
res
;
}
int
out_num
=
buffer_
[
0
].
size
();
res
.
reserve
(
out_num
);
for
(
int
j
=
0
;
j
<
out_num
;
++
j
)
{
// Merge shape and check date type
std
::
type_index
batch_type
=
buffer_
[
0
][
j
].
type
();
DDim
batch_shape
=
buffer_
[
0
][
j
].
dims
();
for
(
size_t
i
=
1
;
i
<
buffer_
.
size
();
++
i
)
{
std
::
type_index
ins_type
=
buffer_
[
i
][
j
].
type
();
DDim
ins_shape
=
buffer_
[
i
][
j
].
dims
();
PADDLE_ENFORCE_EQ
(
batch_type
,
ins_type
);
PADDLE_ENFORCE_EQ
(
slice_ddim
(
batch_shape
,
1
,
batch_shape
.
size
()),
slice_ddim
(
ins_shape
,
1
,
ins_shape
.
size
()));
PADDLE_ENFORCE_GT
(
ins_shape
[
0
],
0
);
batch_shape
[
0
]
+=
ins_shape
[
0
];
}
LoDTensor
out
;
out
.
Resize
(
batch_shape
);
out
.
mutable_data
(
platform
::
CPUPlace
(),
batch_type
);
int64_t
dst_offset
=
0
;
// Merge lod and data
LoD
batch_lod
;
std
::
vector
<
size_t
>
top_level_lod
({
0
});
for
(
size_t
i
=
0
;
i
<
buffer_
.
size
();
++
i
)
{
DDim
ins_shape
=
buffer_
[
i
][
j
].
dims
();
LoD
ins_lod
=
buffer_
[
i
][
j
].
lod
();
if
(
i
==
0
)
{
batch_lod
=
ins_lod
;
}
else
{
PADDLE_ENFORCE_EQ
(
batch_lod
.
size
(),
ins_lod
.
size
());
for
(
size_t
level_idx
=
0
;
level_idx
<
batch_lod
.
size
();
++
level_idx
)
{
auto
&
lod_level
=
batch_lod
[
level_idx
];
for
(
size_t
k
=
1
;
k
<
ins_lod
[
level_idx
].
size
();
++
k
)
{
lod_level
.
push_back
(
ins_lod
[
level_idx
][
k
]
+
lod_level
.
back
());
}
}
}
top_level_lod
.
push_back
(
top_level_lod
.
back
()
+
(
ins_lod
.
empty
()
?
ins_shape
[
0
]
:
(
ins_lod
[
0
].
size
()
-
1
)));
Tensor
dst
=
out
.
Slice
(
dst_offset
,
dst_offset
+
ins_shape
[
0
]);
Copy
(
buffer_
[
i
][
j
],
platform
::
CPUPlace
(),
&
dst
);
dst_offset
+=
ins_shape
[
0
];
}
batch_lod
.
insert
(
batch_lod
.
begin
(),
top_level_lod
);
out
.
set_lod
(
batch_lod
);
res
.
push_back
(
out
);
}
return
res
;
}
}
}
// namespace framework
}
// namespace framework
}
// namespace paddle
}
// namespace paddle
paddle/framework/reader.h
浏览文件 @
f32ca636
...
@@ -22,20 +22,61 @@ namespace framework {
...
@@ -22,20 +22,61 @@ namespace framework {
class
Reader
{
class
Reader
{
public:
public:
virtual
int
ReadNext
(
std
::
vector
<
LoDTensor
>*
outs
)
=
0
;
Reader
()
{}
DDim
shape
(
int
idx
)
const
;
explicit
Reader
(
const
std
::
vector
<
DDim
>&
shapes
)
:
shapes_
(
shapes
)
{}
virtual
std
::
vector
<
LoDTensor
>
ReadNext
()
=
0
;
virtual
bool
HasNext
()
const
=
0
;
virtual
DDim
shape
(
size_t
idx
)
const
;
virtual
std
::
vector
<
DDim
>
shapes
()
const
{
return
shapes_
;
}
virtual
~
Reader
()
{}
private:
private:
// set private to prevent directly access in decorators
// a decorator should access its underlying reader_'s shape, not its own.
std
::
vector
<
DDim
>
shapes_
;
std
::
vector
<
DDim
>
shapes_
;
};
};
// file readers
// file readers
template
<
typename
T
>
class
RandomReader
:
public
Reader
{
class
RandomReader
:
public
Reader
{
public:
public:
RandomReader
(
const
std
::
vector
<
DDim
>&
shapes
,
float
min
,
float
max
)
RandomReader
(
const
std
::
vector
<
DDim
>&
shapes
,
float
min
,
float
max
)
:
shapes_
(
shapes
),
min_
(
min
),
max_
(
max
)
{}
:
Reader
(
shapes
),
min_
(
min
),
max_
(
max
)
{
int
ReadNext
(
std
::
vector
<
LoDTensor
>*
outs
)
override
;
PADDLE_ENFORCE_LE
(
min
,
max
,
"'min' should be less than or equal to 'max'.(%f vs %f)"
,
min
,
max
);
}
std
::
vector
<
LoDTensor
>
ReadNext
()
override
{
std
::
minstd_rand
engine
;
unsigned
int
seed
=
std
::
random_device
()();
engine
.
seed
(
seed
);
std
::
uniform_real_distribution
<
float
>
dist
(
min_
,
max_
);
std
::
vector
<
LoDTensor
>
res
;
res
.
reserve
(
shapes
().
size
());
for
(
const
DDim
&
shape
:
shapes
())
{
PADDLE_ENFORCE_GE
(
shape
.
size
(),
2
,
"The rank of input data should be 2 at least.(Now it's %d)"
,
shape
.
size
());
LoDTensor
out
;
out
.
Resize
(
shape
);
T
*
data
=
out
.
mutable_data
<
T
>
(
platform
::
CPUPlace
());
int64_t
numel
=
product
(
shape
);
for
(
int64_t
i
=
0
;
i
<
numel
;
++
i
)
{
data
[
i
]
=
dist
(
engine
);
}
res
.
push_back
(
out
);
}
return
res
;
}
bool
HasNext
()
const
override
{
return
true
;
}
private:
private:
float
min_
;
float
min_
;
...
@@ -44,22 +85,40 @@ class RandomReader : public Reader {
...
@@ -44,22 +85,40 @@ class RandomReader : public Reader {
// decorators
// decorators
class
Batch
Reader
:
public
Reader
{
class
Shuffle
Reader
:
public
Reader
{
public:
public:
BatchReader
(
const
Reader
*
reader
)
:
reader_
(
reader
)
{}
ShuffleReader
(
Reader
*
reader
,
int
buffer_size
)
int
ReadNext
(
std
::
vector
<
LoDTensor
>*
outs
)
override
;
:
reader_
(
reader
),
buffer_size_
(
buffer_size
),
iteration_pos_
(
0
)
{
buffer_
.
reserve
(
buffer_size
);
}
std
::
vector
<
LoDTensor
>
ReadNext
()
override
;
bool
HasNext
()
const
override
{
return
reader_
->
HasNext
();
}
DDim
shape
(
size_t
idx
)
const
override
{
return
reader_
->
shape
(
idx
);
}
std
::
vector
<
DDim
>
shapes
()
const
override
{
return
reader_
->
shapes
();
}
private:
private:
const
Reader
*
reader_
;
Reader
*
reader_
;
int
buffer_size_
;
std
::
vector
<
std
::
vector
<
LoDTensor
>>
buffer_
;
size_t
iteration_pos_
;
};
};
class
Shuffle
Reader
:
public
Reader
{
class
Batch
Reader
:
public
Reader
{
public:
public:
ShuffleReader
(
const
Reader
*
reader
)
:
reader_
(
reader
)
{}
BatchReader
(
Reader
*
reader
,
int
batch_size
)
int
ReadNext
(
std
::
vector
<
LoDTensor
>*
outs
)
override
;
:
reader_
(
reader
),
batch_size_
(
batch_size
)
{}
std
::
vector
<
LoDTensor
>
ReadNext
()
override
;
bool
HasNext
()
const
override
{
return
reader_
->
HasNext
();
};
DDim
shape
(
size_t
idx
)
const
override
{
return
reader_
->
shape
(
idx
);
}
std
::
vector
<
DDim
>
shapes
()
const
override
{
return
reader_
->
shapes
();
}
private:
private:
const
Reader
*
reader_
;
Reader
*
reader_
;
int
batch_size_
;
std
::
vector
<
std
::
vector
<
LoDTensor
>>
buffer_
;
};
};
}
// namespace framework
}
// namespace framework
}
// namespace paddle
}
// namespace paddle
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录