diff --git a/paddle/fluid/imperative/basic_engine.cc b/paddle/fluid/imperative/basic_engine.cc
index 376391c568b2627f7fa04ed58d66ec71e068dfc0..990937647ac8809f55a9f5ac0f32d0bf944c4595 100644
--- a/paddle/fluid/imperative/basic_engine.cc
+++ b/paddle/fluid/imperative/basic_engine.cc
@@ -337,9 +337,13 @@ void BasicEngine::Execute() {
                     "Cannot find gradient of variable %s", var->Name()));
           }
 
-          // leaf_accumulators_ : hooks and accumulate-grad for leaf tensor
+          // leaf_accumulators_ : hooks and accumulate-grad for leaf tensor,
+          // it should be orderly and not reapeated.
           if (var->IsLeafGrad()) {
-            leaf_accumulators_.insert(iter->second.get());
+            if (std::find(leaf_accumulators_.begin(), leaf_accumulators_.end(),
+                          iter->second.get()) == leaf_accumulators_.end()) {
+              leaf_accumulators_.push_back(iter->second.get());
+            }
 
             if (iter->second->HasInnerVar()) {
               var = iter->second->InnerVar();
diff --git a/paddle/fluid/imperative/basic_engine.h b/paddle/fluid/imperative/basic_engine.h
index 87c4ea380f3c09d29f779691e6c1b143a9614d49..a2ad8b5f8aa61e438e5afdc72681a96ee21c996b 100644
--- a/paddle/fluid/imperative/basic_engine.h
+++ b/paddle/fluid/imperative/basic_engine.h
@@ -69,7 +69,9 @@ class BasicEngine : public Engine {
   std::vector<std::pair<GradientAccumulator*, std::shared_ptr<VariableWrapper>>>
       need_accu_var_list_;
   // leaf_accumulators_ is only for leaf tensor(hooks/accumulate grad)
-  std::unordered_set<GradientAccumulator*> leaf_accumulators_;
+  // It should be orderly and not repeated, because multiple cards must ensure
+  // that the order of vars is the same.
+  std::vector<GradientAccumulator*> leaf_accumulators_;
 
   bool retain_graph_;
 };