Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
wzpei9
Paddle
提交
72b78154
P
Paddle
项目概览
wzpei9
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
72b78154
编写于
7月 17, 2018
作者:
Y
yuyang18
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Polish reader speed
上级
e576345f
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
28 addition
and
20 deletion
+28
-20
paddle/fluid/framework/lod_tensor.cc
paddle/fluid/framework/lod_tensor.cc
+15
-12
paddle/fluid/framework/lod_tensor.h
paddle/fluid/framework/lod_tensor.h
+3
-2
paddle/fluid/framework/lod_tensor_test.cc
paddle/fluid/framework/lod_tensor_test.cc
+3
-2
paddle/fluid/operators/reader/create_recordio_file_reader_op.cc
.../fluid/operators/reader/create_recordio_file_reader_op.cc
+7
-4
未找到文件。
paddle/fluid/framework/lod_tensor.cc
浏览文件 @
72b78154
...
...
@@ -312,19 +312,22 @@ void WriteToRecordIO(recordio::Writer *writer,
writer
->
Write
(
buffer
.
str
());
}
std
::
vector
<
LoDTensor
>
ReadFromRecordIO
(
recordio
::
Scanner
*
scanner
,
const
platform
::
DeviceContext
&
dev_ctx
)
{
std
::
vector
<
LoDTensor
>
result
;
if
(
scanner
->
HasNext
())
{
std
::
istringstream
sin
(
scanner
->
Next
());
uint32_t
sz
;
sin
.
read
(
reinterpret_cast
<
char
*>
(
&
sz
),
sizeof
(
uint32_t
));
result
.
resize
(
sz
);
for
(
uint32_t
i
=
0
;
i
<
sz
;
++
i
)
{
DeserializeFromStream
(
sin
,
&
result
[
i
],
dev_ctx
);
}
bool
ReadFromRecordIO
(
recordio
::
Scanner
*
scanner
,
const
platform
::
DeviceContext
&
dev_ctx
,
std
::
vector
<
LoDTensor
>
*
result_ptr
)
{
if
(
!
scanner
->
HasNext
())
{
return
false
;
}
return
result
;
std
::
istringstream
sin
(
scanner
->
Next
());
uint32_t
sz
;
sin
.
read
(
reinterpret_cast
<
char
*>
(
&
sz
),
sizeof
(
uint32_t
));
auto
&
result
=
*
result_ptr
;
result
.
resize
(
sz
);
for
(
uint32_t
i
=
0
;
i
<
sz
;
++
i
)
{
DeserializeFromStream
(
sin
,
&
result
[
i
],
dev_ctx
);
}
return
true
;
}
std
::
vector
<
LoDTensor
>
LoDTensor
::
SplitLoDTensor
(
...
...
paddle/fluid/framework/lod_tensor.h
浏览文件 @
72b78154
...
...
@@ -223,8 +223,9 @@ extern void WriteToRecordIO(recordio::Writer* writer,
const
std
::
vector
<
LoDTensor
>&
tensor
,
const
platform
::
DeviceContext
&
dev_ctx
);
extern
std
::
vector
<
LoDTensor
>
ReadFromRecordIO
(
recordio
::
Scanner
*
scanner
,
const
platform
::
DeviceContext
&
dev_ctx
);
extern
bool
ReadFromRecordIO
(
recordio
::
Scanner
*
scanner
,
const
platform
::
DeviceContext
&
dev_ctx
,
std
::
vector
<
LoDTensor
>*
result_ptr
);
/*
* Convert between length-based LoD and offset-based LoD.
...
...
paddle/fluid/framework/lod_tensor_test.cc
浏览文件 @
72b78154
...
...
@@ -301,11 +301,12 @@ static void TestRecordIO() {
{
std
::
unique_ptr
<
std
::
istream
>
stream_ptr
(
stream
);
recordio
::
Scanner
scanner
(
std
::
move
(
stream_ptr
));
auto
tensors
=
ReadFromRecordIO
(
&
scanner
,
ctx
);
std
::
vector
<
framework
::
LoDTensor
>
tensors
;
ASSERT_TRUE
(
ReadFromRecordIO
(
&
scanner
,
ctx
,
&
tensors
));
ASSERT_EQ
(
tensors
.
size
(),
static_cast
<
size_t
>
(
2
));
assert_tensor_ok
(
tensors
[
0
]);
assert_tensor_ok
(
tensors
[
1
]);
tensors
=
ReadFromRecordIO
(
&
scanner
,
ctx
);
ASSERT_TRUE
(
ReadFromRecordIO
(
&
scanner
,
ctx
,
&
tensors
)
);
ASSERT_EQ
(
tensors
.
size
(),
static_cast
<
size_t
>
(
2
));
assert_tensor_ok
(
tensors
[
0
]);
assert_tensor_ok
(
tensors
[
1
]);
...
...
paddle/fluid/operators/reader/create_recordio_file_reader_op.cc
浏览文件 @
72b78154
...
...
@@ -33,11 +33,14 @@ class RecordIOFileReader : public framework::FileReader {
protected:
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
{
std
::
unique_ptr
<
std
::
lock_guard
<
std
::
mutex
>>
guard
;
if
(
ThreadSafe
)
{
std
::
lock_guard
<
std
::
mutex
>
guard
(
*
mutex_
);
*
out
=
framework
::
ReadFromRecordIO
(
&
scanner_
,
dev_ctx_
);
}
else
{
*
out
=
framework
::
ReadFromRecordIO
(
&
scanner_
,
dev_ctx_
);
guard
.
reset
(
new
std
::
lock_guard
<
std
::
mutex
>
(
*
mutex_
));
}
bool
ok
=
framework
::
ReadFromRecordIO
(
&
scanner_
,
dev_ctx_
,
out
);
if
(
!
ok
)
{
out
->
clear
();
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录