tmp

paddle/fluid/operators/distributed/communicator.cc

@@ -458,7 +458,7 @@ void GeoCommunicator::Send(const std::vector<std::string> &var_names,
   //     var_tables.size(), 1,
   //     platform::errors::InvalidArgument("var_tables.size() == 1 is
   //     permitted"));
-
+  auto before_send = GetCurrentUS();
   std::shared_ptr<SparseIdsMap> ids_table = std::make_shared<SparseIdsMap>();
   for (size_t i = 0; i < var_tables.size(); i++) {
     auto table_name = var_tables[i];
@@ -494,7 +494,8 @@ void GeoCommunicator::Send(const std::vector<std::string> &var_names,
     }
   }
   need_push_queue_->Push(ids_table);
-  VLOG(1) << "run send_op finish.";
+  auto after_send = GetCurrentUS();
+  VLOG(0) << "run send_op finish. using " << (after_send - before_send);
 }
 
 void GeoCommunicator::MainThread() {
@@ -506,25 +507,12 @@ void GeoCommunicator::MainThread() {
   }
 
   while (running_) {
-    // int meet = Meet();
-
-    // VLOG(1) << "async_meet: " << meet;
-
-    //  SendGlobalStep(meet);
-    auto before = GetCurrentUS();
-    SendByCommunicator(0);
-    auto after = GetCurrentUS();
-    VLOG(0) << "finish one SendByCommunicator using " << (after - before);
-  }
-  VLOG(1) << "geo-communicator stopped, send thread exit";
-}
-
-void GeoCommunicator::SendByCommunicator(int batches) {
     std::vector<std::future<void>> tasks;
     tasks.reserve(send_var_nums_);
 
     auto before_send_by_communicator = GetCurrentUS();
     size_t wait_times = 0;
+
     while (ids_send_vec_.size() < static_cast<size_t>(max_merge_var_num_)) {
       VLOG(1) << "ids_send_vec_ Size: " << ids_send_vec_.size();
       if (need_push_queue_->Size() > 0) {
@@ -545,7 +533,8 @@ void GeoCommunicator::SendByCommunicator(int batches) {
     if (ids_send_vec_.size() >= static_cast<size_t>(max_merge_var_num_)) {
       auto before_send_global_step = GetCurrentUS();
       VLOG(0) << "finish ins_send_vec using time "
-            << (before_send_global_step - before_send_by_communicator);
+              << (before_send_global_step - before_send_by_communicator)
+              << "; send_var_nums_ = " << send_var_nums_;
       SendGlobalStep(max_merge_var_num_);
       auto after_send_global_step = GetCurrentUS();
       VLOG(0) << "finish send global_step using "
@@ -574,8 +563,9 @@ void GeoCommunicator::SendByCommunicator(int batches) {
               VLOG(0)
                   << "send recv "
                   << send_varname_to_ctx_.at(var_name).splited_varnames[ep_idx]
-                << " finish, using " << (after_send_sparse - before_send_sparse)
-                << " and " << (after_recv_sparse - after_send_sparse)
+                  << " finish, using "
+                  << (after_send_sparse - before_send_sparse) << " and "
+                  << (after_recv_sparse - after_send_sparse)
                   << "; total = " << (after_recv_sparse - before_send_sparse);
             };
             tasks.emplace_back(
@@ -605,7 +595,10 @@ void GeoCommunicator::SendByCommunicator(int batches) {
       }
 
       ids_send_vec_.clear();
-    VLOG(1) << "Finish SendByCommunicator";
+      auto finish_one_comm = GetCurrentUS();
+      VLOG(0) << "Finish SendByCommunicator "
+              << (finish_one_comm - after_send_global_step);
+    }
   }
 }
 

paddle/fluid/operators/distributed/communicator.h

@@ -424,7 +424,7 @@ class GeoCommunicator : public AsyncCommunicator {
             const std::vector<std::string> &var_tables,
             const framework::Scope &scope) override;
 
-  void SendByCommunicator(int batches) override;
+  // void SendByCommunicator(int batches) override;
 
   void SendSparse(const std::string &varname, int ep_idx);