未验证 提交 fc038da7 编写于 作者: J jiaqi 提交者: GitHub

fix QueueDataset queue size (#19016)

* fix QueueDataset queue size,set queue size = batch size * 100, to avoid too many instances in channel when training is much slower than reading data.
上级 8f537354
...@@ -106,6 +106,7 @@ void PrivateQueueDataFeed<T>::SetQueueSize(int queue_size) { ...@@ -106,6 +106,7 @@ void PrivateQueueDataFeed<T>::SetQueueSize(int queue_size) {
PADDLE_ENFORCE(queue_size > 0, "Illegal queue size: %d.", queue_size); PADDLE_ENFORCE(queue_size > 0, "Illegal queue size: %d.", queue_size);
queue_size_ = queue_size; queue_size_ = queue_size;
queue_ = paddle::framework::MakeChannel<T>(); queue_ = paddle::framework::MakeChannel<T>();
queue_->SetCapacity(queue_size);
} }
template <typename T> template <typename T>
...@@ -301,7 +302,8 @@ void MultiSlotDataFeed::Init( ...@@ -301,7 +302,8 @@ void MultiSlotDataFeed::Init(
paddle::framework::MultiSlotDesc multi_slot_desc = paddle::framework::MultiSlotDesc multi_slot_desc =
data_feed_desc.multi_slot_desc(); data_feed_desc.multi_slot_desc();
SetBatchSize(data_feed_desc.batch_size()); SetBatchSize(data_feed_desc.batch_size());
SetQueueSize(data_feed_desc.batch_size()); // temporarily set queue size = batch size * 100
SetQueueSize(data_feed_desc.batch_size() * 100);
size_t all_slot_num = multi_slot_desc.slots_size(); size_t all_slot_num = multi_slot_desc.slots_size();
all_slots_.resize(all_slot_num); all_slots_.resize(all_slot_num);
all_slots_type_.resize(all_slot_num); all_slots_type_.resize(all_slot_num);
......
...@@ -194,7 +194,8 @@ std::shared_ptr<FILE> shell_popen(const std::string& cmd, ...@@ -194,7 +194,8 @@ std::shared_ptr<FILE> shell_popen(const std::string& cmd,
<< ", err_no[" << *err_no << "]"; << ", err_no[" << *err_no << "]";
} }
if (wstatus == -1 && errno == ECHILD) { if (wstatus == -1 && errno == ECHILD) {
LOG(WARNING) << "errno is ECHILD"; // temporarily remove this warning
// LOG(WARNING) << "errno is ECHILD";
} }
}}; }};
#endif #endif
...@@ -285,7 +286,8 @@ std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open( ...@@ -285,7 +286,8 @@ std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open(
<< "status[" << wstatus << "], cmd[" << cmd << "]"; << "status[" << wstatus << "], cmd[" << cmd << "]";
if (wstatus == -1 && errno == ECHILD) { if (wstatus == -1 && errno == ECHILD) {
LOG(WARNING) << "errno is ECHILD"; // temporarily remove this warning
// LOG(WARNING) << "errno is ECHILD";
} }
}}; }};
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册