diff --git a/paddle/fluid/framework/details/multi_devices_graph_builder.cc b/paddle/fluid/framework/details/multi_devices_graph_builder.cc
index 882d438934920f6c3c51f73b41d5fe30b61043e7..d2b6a35a5d5c260b023c68ec4684da95a5b79e81 100644
--- a/paddle/fluid/framework/details/multi_devices_graph_builder.cc
+++ b/paddle/fluid/framework/details/multi_devices_graph_builder.cc
@@ -143,10 +143,8 @@ void MultiDevSSAGraphBuilder::InsertNCCLAllReduceOp(
   for (size_t i = 0; i < places_.size(); ++i) {
     auto &p = places_[i];
     auto &vars = result->vars_[i][og];
-    if (vars.empty()) {  // This device has no data. continue.
-      continue;
-    }
-    auto &prev_grad = vars[vars.size() - 1];
+    PADDLE_ENFORCE(!vars.empty());
+    auto &prev_grad = vars.back();
     op_handle->AddInput(prev_grad.get());
 
     auto var = new VarHandle(vars.size() - 1, i, og, p);