未验证 提交 fc038da7 编写于 作者: J jiaqi 提交者: GitHub

fix QueueDataset queue size (#19016)

* fix QueueDataset queue size,set queue size = batch size * 100, to avoid too many instances in channel when training is much slower than reading data.
上级 8f537354
......@@ -106,6 +106,7 @@ void PrivateQueueDataFeed<T>::SetQueueSize(int queue_size) {
PADDLE_ENFORCE(queue_size > 0, "Illegal queue size: %d.", queue_size);
queue_size_ = queue_size;
queue_ = paddle::framework::MakeChannel<T>();
queue_->SetCapacity(queue_size);
}
template <typename T>
......@@ -301,7 +302,8 @@ void MultiSlotDataFeed::Init(
paddle::framework::MultiSlotDesc multi_slot_desc =
data_feed_desc.multi_slot_desc();
SetBatchSize(data_feed_desc.batch_size());
SetQueueSize(data_feed_desc.batch_size());
// temporarily set queue size = batch size * 100
SetQueueSize(data_feed_desc.batch_size() * 100);
size_t all_slot_num = multi_slot_desc.slots_size();
all_slots_.resize(all_slot_num);
all_slots_type_.resize(all_slot_num);
......
......@@ -194,7 +194,8 @@ std::shared_ptr<FILE> shell_popen(const std::string& cmd,
<< ", err_no[" << *err_no << "]";
}
if (wstatus == -1 && errno == ECHILD) {
LOG(WARNING) << "errno is ECHILD";
// temporarily remove this warning
// LOG(WARNING) << "errno is ECHILD";
}
}};
#endif
......@@ -285,7 +286,8 @@ std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open(
<< "status[" << wstatus << "], cmd[" << cmd << "]";
if (wstatus == -1 && errno == ECHILD) {
LOG(WARNING) << "errno is ECHILD";
// temporarily remove this warning
// LOG(WARNING) << "errno is ECHILD";
}
}};
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册