From d25389fefdb988b667a50931e0d8847176d54315 Mon Sep 17 00:00:00 2001
From: xujiaqi01 <xujiaqi01@baidu.com>
Date: Thu, 14 Mar 2019 12:20:10 +0800
Subject: [PATCH] add some log && fix error

---
 paddle/fluid/framework/data_feed.cc           | 18 +++++++++++++-----
 paddle/fluid/framework/data_set.cc            |  1 -
 paddle/fluid/framework/data_set.h             |  4 ----
 paddle/fluid/framework/executor.cc            |  2 +-
 paddle/fluid/framework/executor.h             |  2 +-
 paddle/fluid/framework/fleet/fleet_wrapper.cc |  3 +++
 6 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/paddle/fluid/framework/data_feed.cc b/paddle/fluid/framework/data_feed.cc
index 14daf9448..62f35f205 100644
--- a/paddle/fluid/framework/data_feed.cc
+++ b/paddle/fluid/framework/data_feed.cc
@@ -177,6 +177,9 @@ int InMemoryDataFeed<T>::Next() {
   }
   CHECK(in_channel != nullptr);
   CHECK(out_channel != nullptr);
+  VLOG(3) << "in_channel size=" << in_channel->Size()
+          << ", out_channel size=" << out_channel->Size()
+          << ", thread_id=" << thread_id_;
   int index = 0;
   T instance;
   T ins_vec;
@@ -259,14 +262,19 @@ void InMemoryDataFeed<T>::FillChannelToMemoryData() {
     channel = shuffled_ins_out_;
   }
   CHECK(channel != nullptr);
-  local_vec.reserve(channel->Size());
+  local_vec.resize(channel->Size());
   for (int64_t i = 0; i < channel->Size(); ++i) {
     channel->Pop(local_vec[i]);
   }
-  std::unique_lock<std::mutex> lock(*mutex_for_update_memory_data_);
-  lock.lock();
-  memory_data_->insert(memory_data_->end(), local_vec.begin(), local_vec.end());
-  lock.unlock();
+  VLOG(3) << "local_vec size=" << local_vec.size() <<", thread_id=" << thread_id_;
+  {
+    std::lock_guard<std::mutex> g(*mutex_for_update_memory_data_);
+    VLOG(3) << "before insert, memory_data_ size=" << memory_data_->size()
+            << ", thread_id=" << thread_id_;
+    memory_data_->insert(memory_data_->end(), local_vec.begin(), local_vec.end());
+    VLOG(3) << "after insert memory_data_ size=" << memory_data_->size()
+            << ", thread_id=" << thread_id_;
+  }
   std::vector<T>().swap(local_vec);
 }
 
diff --git a/paddle/fluid/framework/data_set.cc b/paddle/fluid/framework/data_set.cc
index 28cfbed4f..1d2a018be 100644
--- a/paddle/fluid/framework/data_set.cc
+++ b/paddle/fluid/framework/data_set.cc
@@ -176,7 +176,6 @@ void DatasetImpl<T>::DestroyReaders() {
   for (std::thread& t : fill_threads) {
     t.join();
   }
-  std::vector<std::string>().swap(filelist_);
   std::vector<std::shared_ptr<paddle::framework::DataFeed>>().swap(readers_);
 }
 
diff --git a/paddle/fluid/framework/data_set.h b/paddle/fluid/framework/data_set.h
index 334fceb69..41aa636c6 100644
--- a/paddle/fluid/framework/data_set.h
+++ b/paddle/fluid/framework/data_set.h
@@ -83,10 +83,6 @@ class DatasetImpl : public Dataset {
   std::vector<std::shared_ptr<paddle::framework::DataFeed>> readers_;
   std::vector<T> memory_data_;
   std::mutex mutex_for_update_memory_data_;
-  std::vector<std::shared_ptr<paddle::framework::BlockingQueue<T>>>
-      shuffled_ins_vec_;
-  std::vector<std::shared_ptr<paddle::framework::BlockingQueue<T>>>
-      shuffled_ins_out_vec_;
   int thread_num_;
   paddle::framework::DataFeedDesc data_feed_desc_;
   std::vector<std::string> filelist_;
diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc
index e4fd00628..501480876 100644
--- a/paddle/fluid/framework/executor.cc
+++ b/paddle/fluid/framework/executor.cc
@@ -118,7 +118,7 @@ void Executor::CreateVariables(const ProgramDesc& pdesc, Scope* scope,
 }
 
 void Executor::RunFromDataset(const ProgramDesc& main_program, Scope* scope,
-                              MultiSlotDataset* dataset,
+                              Dataset* dataset,
                               const std::string& trainer_desc_str) {
   VLOG(3) << "Start to RunFromDataset in executor";
   TrainerDesc trainer_desc;
diff --git a/paddle/fluid/framework/executor.h b/paddle/fluid/framework/executor.h
index b351b924b..d0bd3a4c7 100644
--- a/paddle/fluid/framework/executor.h
+++ b/paddle/fluid/framework/executor.h
@@ -113,7 +113,7 @@ class Executor {
   void EnableMKLDNN(const ProgramDesc& program);
 
   void RunFromDataset(const ProgramDesc& main_program, Scope* scope,
-                      MultiSlotDataset* dataset,
+                      Dataset* dataset,
                       const std::string& trainer_desc_str);
 
  private:
diff --git a/paddle/fluid/framework/fleet/fleet_wrapper.cc b/paddle/fluid/framework/fleet/fleet_wrapper.cc
index 954920df6..92b762946 100644
--- a/paddle/fluid/framework/fleet/fleet_wrapper.cc
+++ b/paddle/fluid/framework/fleet/fleet_wrapper.cc
@@ -297,6 +297,9 @@ void FleetWrapper::PushSparseVarsWithLabelAsync(
 int FleetWrapper::RegisterClientToClientMsgHandler(
     int msg_type, MsgHandlerFunc handler) {
 #ifdef PADDLE_WITH_PSLIB
+  VLOG(3) << "calling FleetWrapper::RegisterClientToClientMsgHandler";
+  VLOG(3) << "pslib_ptr_=" << pslib_ptr_;
+  VLOG(3) << "_worker_ptr=" << pslib_ptr_->_worker_ptr;
   pslib_ptr_->_worker_ptr->registe_client2client_msg_handler(
       msg_type, handler);
 #else
-- 
GitLab