From f8f1a963d9508cbdbd37c61554e8ffac9bf4a6ab Mon Sep 17 00:00:00 2001
From: Yu Yang <yuyang18@baidu.com>
Date: Tue, 20 Mar 2018 16:52:20 +0800
Subject: [PATCH] Add debug code

---
 paddle/fluid/framework/parallel_executor.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index 2a1652f749d..d1652a3030b 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -365,6 +365,7 @@ struct NCCLAllReduceOpHandle : public OpHandle {
         auto &p = static_cast<VarHandle *>(in)->place_;
         in->generated_op_->Wait(dev_ctx_[p]);
       }
+      PADDLE_ENFORCE(cudaDeviceSynchronize());
 
       auto &var_name = static_cast<VarHandle *>(this->inputs_[0])->name_;
       int dtype = -1;
@@ -393,6 +394,8 @@ struct NCCLAllReduceOpHandle : public OpHandle {
             nccl_ctx.comm, nccl_ctx.stream());
       }
       platform::dynload::ncclGroupEnd();
+
+      PADDLE_ENFORCE(cudaDeviceSynchronize());
     }
   }
 };
-- 
GitLab