Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
Paddle
提交
dc34effd
P
Paddle
项目概览
BaiXuePrincess
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
dc34effd
编写于
7月 14, 2018
作者:
Y
yuyang18
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Extract buffered reader
上级
ab1a6cb8
变更
4
显示空白变更内容
内联
并排
Showing
4 changed file
with
131 addition
and
73 deletion
+131
-73
paddle/fluid/operators/reader/CMakeLists.txt
paddle/fluid/operators/reader/CMakeLists.txt
+2
-1
paddle/fluid/operators/reader/buffered_reader.cc
paddle/fluid/operators/reader/buffered_reader.cc
+73
-0
paddle/fluid/operators/reader/buffered_reader.h
paddle/fluid/operators/reader/buffered_reader.h
+55
-0
paddle/fluid/operators/reader/create_double_buffer_reader_op.cc
.../fluid/operators/reader/create_double_buffer_reader_op.cc
+1
-72
未找到文件。
paddle/fluid/operators/reader/CMakeLists.txt
浏览文件 @
dc34effd
...
@@ -15,12 +15,13 @@ function(reader_library TARGET_NAME)
...
@@ -15,12 +15,13 @@ function(reader_library TARGET_NAME)
PARENT_SCOPE
)
PARENT_SCOPE
)
endfunction
()
endfunction
()
cc_library
(
buffered_reader SRCS buffered_reader.cc DEPS reader simple_threadpool
)
reader_library
(
open_files_op SRCS open_files_op.cc
)
reader_library
(
open_files_op SRCS open_files_op.cc
)
reader_library
(
create_random_data_generator_op SRCS create_random_data_generator_op.cc
)
reader_library
(
create_random_data_generator_op SRCS create_random_data_generator_op.cc
)
reader_library
(
create_shuffle_reader_op SRCS create_shuffle_reader_op.cc
)
reader_library
(
create_shuffle_reader_op SRCS create_shuffle_reader_op.cc
)
reader_library
(
create_batch_reader_op SRCS create_batch_reader_op.cc
)
reader_library
(
create_batch_reader_op SRCS create_batch_reader_op.cc
)
reader_library
(
create_recordio_file_reader_op SRCS create_recordio_file_reader_op.cc
)
reader_library
(
create_recordio_file_reader_op SRCS create_recordio_file_reader_op.cc
)
reader_library
(
create_double_buffer_reader_op SRCS create_double_buffer_reader_op.cc
)
reader_library
(
create_double_buffer_reader_op SRCS create_double_buffer_reader_op.cc
DEPS buffered_reader
)
reader_library
(
create_multi_pass_reader_op SRCS create_multi_pass_reader_op.cc
)
reader_library
(
create_multi_pass_reader_op SRCS create_multi_pass_reader_op.cc
)
reader_library
(
create_custom_reader_op SRCS create_custom_reader_op.cc
)
reader_library
(
create_custom_reader_op SRCS create_custom_reader_op.cc
)
reader_library
(
create_py_reader_op SRCS create_py_reader_op.cc
)
reader_library
(
create_py_reader_op SRCS create_py_reader_op.cc
)
...
...
paddle/fluid/operators/reader/buffered_reader.cc
0 → 100644
浏览文件 @
dc34effd
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/operators/reader/buffered_reader.h"
#include <vector>
namespace
paddle
{
namespace
operators
{
namespace
reader
{
BufferedReader
::~
BufferedReader
()
{
reader_
->
Shutdown
();
buffer_
.
clear
();
}
BufferedReader
::
BufferedReader
(
const
std
::
shared_ptr
<
framework
::
ReaderBase
>
&
reader
,
const
platform
::
Place
&
place
,
size_t
buffer_size
)
:
framework
::
DecoratedReader
(
reader
),
thread_pool_
(
1
),
place_
(
place
),
buffer_size_
(
buffer_size
)
{
AppendFutureToBatchSize
();
}
void
BufferedReader
::
AppendFutureToBatchSize
()
{
while
(
buffer_
.
size
()
<
buffer_size_
)
{
AppendFuture
();
}
}
void
BufferedReader
::
AppendFuture
()
{
buffer_
.
emplace_back
(
thread_pool_
.
enqueue
([
this
]
{
TensorVec
cpu_buffer
;
reader_
->
ReadNext
(
&
cpu_buffer
);
if
(
platform
::
is_gpu_place
(
place_
))
{
TensorVec
gpu_buffer
;
for
(
size_t
i
=
0
;
i
<
cpu_buffer
.
size
();
++
i
)
{
gpu_buffer
.
emplace_back
();
framework
::
TensorCopySync
(
cpu_buffer
[
i
],
place_
,
&
gpu_buffer
.
back
());
}
cpu_buffer
=
gpu_buffer
;
}
return
cpu_buffer
;
}));
}
void
BufferedReader
::
ShutdownImpl
()
{
reader_
->
Shutdown
();
buffer_
.
clear
();
}
void
BufferedReader
::
StartImpl
()
{
reader_
->
Start
();
AppendFutureToBatchSize
();
}
void
BufferedReader
::
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>
*
out
)
{
PADDLE_ENFORCE_EQ
(
buffer_
.
size
(),
buffer_size_
);
*
out
=
buffer_
.
front
().
get
();
buffer_
.
pop_front
();
AppendFuture
();
}
}
// namespace reader
}
// namespace operators
}
// namespace paddle
paddle/fluid/operators/reader/buffered_reader.h
0 → 100644
浏览文件 @
dc34effd
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <list>
#include <vector>
#include "ThreadPool.h"
#include "paddle/fluid/framework/reader.h"
namespace
paddle
{
namespace
operators
{
namespace
reader
{
class
BufferedReader
:
public
framework
::
DecoratedReader
{
using
TensorVec
=
std
::
vector
<
framework
::
LoDTensor
>
;
using
VecFuture
=
std
::
future
<
TensorVec
>
;
public:
BufferedReader
(
const
std
::
shared_ptr
<
framework
::
ReaderBase
>&
reader
,
const
platform
::
Place
&
place
,
size_t
buffer_size
);
~
BufferedReader
()
override
;
private:
void
AppendFutureToBatchSize
();
void
AppendFuture
();
protected:
void
ShutdownImpl
()
override
;
void
StartImpl
()
override
;
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
;
private:
ThreadPool
thread_pool_
;
platform
::
Place
place_
;
const
size_t
buffer_size_
;
std
::
list
<
VecFuture
>
buffer_
;
};
}
// namespace reader
}
// namespace operators
}
// namespace paddle
paddle/fluid/operators/reader/create_double_buffer_reader_op.cc
浏览文件 @
dc34effd
...
@@ -12,83 +12,12 @@
...
@@ -12,83 +12,12 @@
// See the License for the specific language governing permissions and
// See the License for the specific language governing permissions and
// limitations under the License.
// limitations under the License.
#include <thread> // NOLINT
#include "paddle/fluid/operators/reader/buffered_reader.h"
#include "ThreadPool.h"
#include "paddle/fluid/operators/reader/blocking_queue.h"
#include "paddle/fluid/operators/reader/reader_op_registry.h"
#include "paddle/fluid/operators/reader/reader_op_registry.h"
namespace
paddle
{
namespace
paddle
{
namespace
operators
{
namespace
operators
{
namespace
reader
{
namespace
reader
{
class
BufferedReader
:
public
framework
::
DecoratedReader
{
using
TensorVec
=
std
::
vector
<
framework
::
LoDTensor
>
;
using
VecFuture
=
std
::
future
<
TensorVec
>
;
public:
BufferedReader
(
const
std
::
shared_ptr
<
framework
::
ReaderBase
>&
reader
,
const
platform
::
Place
&
place
,
size_t
buffer_size
)
:
framework
::
DecoratedReader
(
reader
),
thread_pool_
(
1
),
place_
(
place
),
buffer_size_
(
buffer_size
)
{
AppendFutureToBatchSize
();
}
~
BufferedReader
()
override
{
reader_
->
Shutdown
();
buffer_
.
clear
();
}
private:
void
AppendFutureToBatchSize
()
{
while
(
buffer_
.
size
()
<
buffer_size_
)
{
AppendFuture
();
}
}
void
AppendFuture
()
{
buffer_
.
emplace_back
(
thread_pool_
.
enqueue
([
this
]
{
TensorVec
cpu_buffer
;
reader_
->
ReadNext
(
&
cpu_buffer
);
if
(
platform
::
is_gpu_place
(
place_
))
{
TensorVec
gpu_buffer
;
for
(
size_t
i
=
0
;
i
<
cpu_buffer
.
size
();
++
i
)
{
gpu_buffer
.
emplace_back
();
framework
::
TensorCopySync
(
cpu_buffer
[
i
],
place_
,
&
gpu_buffer
.
back
());
}
cpu_buffer
=
gpu_buffer
;
}
return
cpu_buffer
;
}));
}
protected:
void
ShutdownImpl
()
override
{
reader_
->
Shutdown
();
buffer_
.
clear
();
}
void
StartImpl
()
override
{
reader_
->
Start
();
AppendFutureToBatchSize
();
}
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
{
std
::
cerr
<<
"Read"
<<
std
::
endl
;
PADDLE_ENFORCE_EQ
(
buffer_
.
size
(),
buffer_size_
);
*
out
=
buffer_
.
front
().
get
();
buffer_
.
pop_front
();
AppendFuture
();
}
private:
ThreadPool
thread_pool_
;
platform
::
Place
place_
;
const
size_t
buffer_size_
;
std
::
list
<
VecFuture
>
buffer_
;
};
class
CreateDoubleBufferReaderOp
:
public
framework
::
OperatorBase
{
class
CreateDoubleBufferReaderOp
:
public
framework
::
OperatorBase
{
public:
public:
using
framework
::
OperatorBase
::
OperatorBase
;
using
framework
::
OperatorBase
::
OperatorBase
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录