From 13e8b5bf8962eea9aafe0e6c32f761e386767cea Mon Sep 17 00:00:00 2001
From: Qiao Longfei <qiaolongfei@baidu.com>
Date: Mon, 4 Mar 2019 15:31:56 +0800
Subject: [PATCH] clear gradient before merge

---
 paddle/fluid/operators/distributed/communicator.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/paddle/fluid/operators/distributed/communicator.cc b/paddle/fluid/operators/distributed/communicator.cc
index 403fcf4b16..a88b764474 100644
--- a/paddle/fluid/operators/distributed/communicator.cc
+++ b/paddle/fluid/operators/distributed/communicator.cc
@@ -47,6 +47,8 @@ static inline void MergeVars(const std::string &var_name,
     }
   } else if (var0->IsType<framework::SelectedRows>()) {
     auto *out_slr = out_var->GetMutable<framework::SelectedRows>();
+    out_slr->mutable_rows()->clear();
+    out_slr->mutable_value()->mutable_data<float>({{}}, cpu_place);
     std::vector<const paddle::framework::SelectedRows *> inputs;
     inputs.reserve(vars.size());
     for (auto &var : vars) {
@@ -71,6 +73,7 @@ void Communicator::SendThread() {
         VLOG(3) << "merge var " << var_name << " and send";
         auto &var_queue = iter.second;
         std::vector<std::shared_ptr<Variable>> vars;
+        // TODO(qiao): need to be configurable
         const size_t max_merge_var_num = 20;
         size_t merged_var_num = 0;
         while (var_queue->Size() > 0 && merged_var_num < max_merge_var_num) {
-- 
GitLab