Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
Crayon鑫
Paddle
提交
b4f0e579
P
Paddle
项目概览
Crayon鑫
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
b4f0e579
编写于
7月 08, 2018
作者:
F
fengjiayi
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix errors
上级
6fc6cc2f
变更
10
隐藏空白更改
内联
并排
Showing
10 changed file
with
18 addition
and
22 deletion
+18
-22
paddle/fluid/framework/reader.h
paddle/fluid/framework/reader.h
+2
-1
paddle/fluid/operators/reader/create_batch_reader_op.cc
paddle/fluid/operators/reader/create_batch_reader_op.cc
+0
-1
paddle/fluid/operators/reader/create_custom_reader_op.cc
paddle/fluid/operators/reader/create_custom_reader_op.cc
+1
-3
paddle/fluid/operators/reader/create_double_buffer_reader_op.cc
.../fluid/operators/reader/create_double_buffer_reader_op.cc
+2
-2
paddle/fluid/operators/reader/create_multi_pass_reader_op.cc
paddle/fluid/operators/reader/create_multi_pass_reader_op.cc
+1
-3
paddle/fluid/operators/reader/create_random_data_generator_op.cc
...fluid/operators/reader/create_random_data_generator_op.cc
+0
-1
paddle/fluid/operators/reader/create_recordio_file_reader_op.cc
.../fluid/operators/reader/create_recordio_file_reader_op.cc
+0
-1
paddle/fluid/operators/reader/create_shuffle_reader_op.cc
paddle/fluid/operators/reader/create_shuffle_reader_op.cc
+1
-1
paddle/fluid/operators/reader/create_threaded_reader_op.cc
paddle/fluid/operators/reader/create_threaded_reader_op.cc
+1
-3
paddle/fluid/operators/reader/open_files_op.cc
paddle/fluid/operators/reader/open_files_op.cc
+10
-6
未找到文件。
paddle/fluid/framework/reader.h
浏览文件 @
b4f0e579
...
...
@@ -14,6 +14,7 @@
#pragma once
#include <atomic>
#include <memory>
#include <vector>
...
...
@@ -43,7 +44,7 @@ class ReaderBase {
virtual
void
StartImpl
()
=
0
;
std
::
atomic
<
ReaderStatus
>
status_
{
k
Stopped
};
std
::
atomic
<
ReaderStatus
>
status_
{
k
Running
};
};
class
DecoratedReader
:
public
ReaderBase
{
...
...
paddle/fluid/operators/reader/create_batch_reader_op.cc
浏览文件 @
b4f0e579
...
...
@@ -26,7 +26,6 @@ class BatchReader : public framework::DecoratedReader {
batch_size_
(
batch_size
),
discard_leftover_
(
discard_leftover
)
{
buffer_
.
reserve
(
batch_size_
);
Start
();
}
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
;
...
...
paddle/fluid/operators/reader/create_custom_reader_op.cc
浏览文件 @
b4f0e579
...
...
@@ -31,9 +31,7 @@ class CustomReader : public framework::DecoratedReader {
sub_block_id_
(
sub_block
.
ID
()),
exe_
(
framework
::
Executor
(
platform
::
CPUPlace
())),
source_var_names_
(
source_var_names
),
sink_var_names_
(
sink_var_names
)
{
Start
();
}
sink_var_names_
(
sink_var_names
)
{}
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
;
...
...
paddle/fluid/operators/reader/create_double_buffer_reader_op.cc
浏览文件 @
b4f0e579
...
...
@@ -47,12 +47,12 @@ class DoubleBufferReader : public framework::DecoratedReader {
}
}
#endif
Start
();
Start
Prefetcher
();
}
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
;
~
DoubleBufferReader
()
{
Shutdown
();
}
~
DoubleBufferReader
()
{
EndPrefetcher
();
}
private:
void
ShutdownImpl
()
override
{
...
...
paddle/fluid/operators/reader/create_multi_pass_reader_op.cc
浏览文件 @
b4f0e579
...
...
@@ -22,9 +22,7 @@ namespace reader {
class
MultiPassReader
:
public
framework
::
DecoratedReader
{
public:
MultiPassReader
(
const
std
::
shared_ptr
<
ReaderBase
>&
reader
,
int
pass_num
)
:
DecoratedReader
(
reader
),
pass_num_
(
pass_num
)
{
Start
();
}
:
DecoratedReader
(
reader
),
pass_num_
(
pass_num
),
pass_count_
(
0
)
{}
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
{
reader_
->
ReadNext
(
out
);
...
...
paddle/fluid/operators/reader/create_random_data_generator_op.cc
浏览文件 @
b4f0e579
...
...
@@ -30,7 +30,6 @@ class RandomDataGenerator : public framework::FileReader {
unsigned
int
seed
=
std
::
random_device
()();
engine_
.
seed
(
seed
);
dist_
=
std
::
uniform_real_distribution
<
float
>
(
low_
,
high_
);
Start
();
}
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
{
...
...
paddle/fluid/operators/reader/create_recordio_file_reader_op.cc
浏览文件 @
b4f0e579
...
...
@@ -30,7 +30,6 @@ class RecordIOFileReader : public framework::FileReader {
mutex_
.
reset
(
new
std
::
mutex
());
}
LOG
(
INFO
)
<<
"Creating file reader"
<<
filename
;
Start
();
}
protected:
...
...
paddle/fluid/operators/reader/create_shuffle_reader_op.cc
浏览文件 @
b4f0e579
...
...
@@ -31,7 +31,7 @@ class ShuffleReader : public framework::DecoratedReader {
std
::
random_device
device
;
seed_
=
device
();
}
Start
();
ReloadBuffer
();
}
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
{
...
...
paddle/fluid/operators/reader/create_threaded_reader_op.cc
浏览文件 @
b4f0e579
...
...
@@ -22,9 +22,7 @@ namespace reader {
class
ThreadedReader
:
public
framework
::
DecoratedReader
{
public:
explicit
ThreadedReader
(
const
std
::
shared_ptr
<
ReaderBase
>&
reader
)
:
DecoratedReader
(
reader
)
{
Start
();
}
:
DecoratedReader
(
reader
)
{}
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
{
std
::
lock_guard
<
std
::
mutex
>
lock
(
mutex_
);
...
...
paddle/fluid/operators/reader/open_files_op.cc
浏览文件 @
b4f0e579
...
...
@@ -31,16 +31,20 @@ class MultiFileReader : public framework::ReaderBase {
readers_
.
emplace_back
(
CreateReaderByFileName
(
f_name
));
}
prefetchers_
.
resize
(
thread_num
);
Start
();
Start
NewScheduler
();
}
void
ReadNextImpl
(
std
::
vector
<
framework
::
LoDTensor
>*
out
)
override
;
~
MultiFileReader
()
{
Shutdown
();
}
~
MultiFileReader
()
{
EndScheduler
();
}
private:
void
StartImpl
()
override
;
void
ShutdownImpl
()
override
;
void
ShutdownImpl
()
override
{
EndScheduler
();
}
void
StartImpl
()
override
{
StartNewScheduler
();
}
void
StartNewScheduler
();
void
EndScheduler
();
void
ScheduleThreadFunc
();
void
PrefetchThreadFunc
(
size_t
reader_idx
,
size_t
thread_idx
);
...
...
@@ -59,7 +63,7 @@ void MultiFileReader::ReadNextImpl(std::vector<framework::LoDTensor>* out) {
}
}
void
MultiFileReader
::
Start
Impl
()
{
void
MultiFileReader
::
Start
NewScheduler
()
{
size_t
thread_num
=
prefetchers_
.
size
();
waiting_reader_idx_
=
new
reader
::
BlockingQueue
<
size_t
>
(
readers_
.
size
());
available_thread_idx_
=
new
reader
::
BlockingQueue
<
size_t
>
(
thread_num
);
...
...
@@ -77,7 +81,7 @@ void MultiFileReader::StartImpl() {
scheduler_
=
std
::
thread
([
this
]
{
ScheduleThreadFunc
();
});
}
void
MultiFileReader
::
ShutdownImpl
()
{
void
MultiFileReader
::
EndScheduler
()
{
available_thread_idx_
->
Close
();
buffer_
->
Close
();
waiting_reader_idx_
->
Close
();
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录