From fc038da749a1134f3bf27f5a08675ca884cbbfad Mon Sep 17 00:00:00 2001
From: jiaqi <173596896@qq.com>
Date: Thu, 8 Aug 2019 11:28:57 +0800
Subject: [PATCH] fix QueueDataset queue size (#19016)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* fix QueueDataset queue size，set queue size = batch size * 100, to avoid too many instances in channel when training is much slower than reading data.
---
 paddle/fluid/framework/data_feed.cc | 4 +++-
 paddle/fluid/framework/io/shell.cc  | 6 ++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/paddle/fluid/framework/data_feed.cc b/paddle/fluid/framework/data_feed.cc
index ed94e30e5cd..1ed472de6c1 100644
--- a/paddle/fluid/framework/data_feed.cc
+++ b/paddle/fluid/framework/data_feed.cc
@@ -106,6 +106,7 @@ void PrivateQueueDataFeed<T>::SetQueueSize(int queue_size) {
   PADDLE_ENFORCE(queue_size > 0, "Illegal queue size: %d.", queue_size);
   queue_size_ = queue_size;
   queue_ = paddle::framework::MakeChannel<T>();
+  queue_->SetCapacity(queue_size);
 }
 
 template <typename T>
@@ -301,7 +302,8 @@ void MultiSlotDataFeed::Init(
   paddle::framework::MultiSlotDesc multi_slot_desc =
       data_feed_desc.multi_slot_desc();
   SetBatchSize(data_feed_desc.batch_size());
-  SetQueueSize(data_feed_desc.batch_size());
+  // temporarily set queue size = batch size * 100
+  SetQueueSize(data_feed_desc.batch_size() * 100);
   size_t all_slot_num = multi_slot_desc.slots_size();
   all_slots_.resize(all_slot_num);
   all_slots_type_.resize(all_slot_num);
diff --git a/paddle/fluid/framework/io/shell.cc b/paddle/fluid/framework/io/shell.cc
index ab671cb5690..d63fee7f6e8 100644
--- a/paddle/fluid/framework/io/shell.cc
+++ b/paddle/fluid/framework/io/shell.cc
@@ -194,7 +194,8 @@ std::shared_ptr<FILE> shell_popen(const std::string& cmd,
                            << ", err_no[" << *err_no << "]";
             }
             if (wstatus == -1 && errno == ECHILD) {
-              LOG(WARNING) << "errno is ECHILD";
+              // temporarily remove this warning
+              // LOG(WARNING) << "errno is ECHILD";
             }
           }};
 #endif
@@ -285,7 +286,8 @@ std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open(
             << "status[" << wstatus << "], cmd[" << cmd << "]";
 
         if (wstatus == -1 && errno == ECHILD) {
-          LOG(WARNING) << "errno is ECHILD";
+          // temporarily remove this warning
+          // LOG(WARNING) << "errno is ECHILD";
         }
       }};
 
-- 
GitLab