未验证 提交 b048ddf0 编写于 作者: Y yuyang18

Merge error

上级 b8975d68
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
#include "ThreadPool.h" #include "ThreadPool.h"
#include "paddle/fluid/framework/blocking_queue.h" #include "paddle/fluid/framework/blocking_queue.h"
#include "paddle/fluid/operators/reader/blocking_queue.h" #include "paddle/fluid/operators/reader/blocking_queue.h"
#include "paddle/fluid/operators/reader/buffered_reader.h"
#include "paddle/fluid/operators/reader/reader_op_registry.h" #include "paddle/fluid/operators/reader/reader_op_registry.h"
namespace paddle { namespace paddle {
...@@ -232,12 +233,17 @@ class OpenFilesOp : public framework::OperatorBase { ...@@ -232,12 +233,17 @@ class OpenFilesOp : public framework::OperatorBase {
container.reset(new OrderedReaderContainer()); container.reset(new OrderedReaderContainer());
} else { } else {
container.reset(new PreemptiveReaderContainer( container.reset(new PreemptiveReaderContainer(
std::min(file_names.size(), static_cast<size_t>(Attr<int>("thread_num"))));
static_cast<size_t>(std::thread::hardware_concurrency()))));
} }
out->Reset( std::shared_ptr<framework::ReaderBase> reader(
std::make_shared<MultiFileReader>(file_names, std::move(container))); new MultiFileReader(file_names, std::move(container)));
auto buffer_size = Attr<int>("buffer_size");
if (buffer_size > 1) {
reader = framework::MakeDecoratedReader<BufferedReader>(
reader, platform::CPUPlace(), buffer_size);
}
out->Reset(reader);
} }
}; };
...@@ -253,6 +259,8 @@ class OpenFilesOpMaker : public FileReaderMakerBase { ...@@ -253,6 +259,8 @@ class OpenFilesOpMaker : public FileReaderMakerBase {
An OpenFilesOp creates a MultiFileReader, which is able to An OpenFilesOp creates a MultiFileReader, which is able to
read data multi-threaded from multiple files. read data multi-threaded from multiple files.
)DOC"); )DOC");
AddAttr<int>("thread_num", "Number of thread to read files.");
AddAttr<int>("buffer_size", "The reading buffer of these files.");
} }
}; };
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册