diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc
index c3a90149a1f0daa6c5240ba2e65da41ad6f135de..67e7078fbc769dc271cf6f91bbccf59e5777eee3 100644
--- a/paddle/fluid/framework/parallel_executor.cc
+++ b/paddle/fluid/framework/parallel_executor.cc
@@ -155,7 +155,7 @@ struct ScaleLossGradOpHandle : public OpHandle {
       VLOG(3) << "2";
       memory::Copy(boost::get<platform::CUDAPlace>(place_), tmp,
                    platform::CPUPlace(), &coeff_, sizeof(float), stream);
-      PADDLE_ENFORCE(cudaGetLastError());
+      PADDLE_ENFORCE(cudaDeviceSynchronize());
       VLOG(3) << "3";
       PADDLE_ENFORCE(cudaEventRecord(ev_, stream));
       VLOG(3) << "4";