diff --git a/paddle/fluid/framework/details/multi_devices_graph_builder.cc b/paddle/fluid/framework/details/multi_devices_graph_builder.cc
index 50bfd42581367b827f36333d07628ea707f31d27..35d23d68c0dd26a05544a72316d5764129aa8d40 100644
--- a/paddle/fluid/framework/details/multi_devices_graph_builder.cc
+++ b/paddle/fluid/framework/details/multi_devices_graph_builder.cc
@@ -170,7 +170,7 @@ std::unique_ptr<SSAGraph> MultiDevSSAGraphBuilder::Build(
 
             PADDLE_ENFORCE_EQ(backward_vars.size() % 2, 0);
 
-            for (size_t i = 0; i < backward_vars.size(); ++i) {
+            for (size_t i = 0; i < backward_vars.size(); i += 2) {
               auto &p_name = backward_vars[i];
               auto &g_name = backward_vars[i + 1];
               VLOG(10) << "Bcast " << g_name << " for parameter " << p_name;