diff --git a/paddle/fluid/operators/distributed/async_sparse_param_update_recorder.h b/paddle/fluid/operators/distributed/async_sparse_param_update_recorder.h
index 49e7c07e5d33d620777c40f98813486c312bdfba..0b9351a67635ed2bf49eeb60b5348fd1a4f62ade 100644
--- a/paddle/fluid/operators/distributed/async_sparse_param_update_recorder.h
+++ b/paddle/fluid/operators/distributed/async_sparse_param_update_recorder.h
@@ -120,9 +120,13 @@ class AsyncSparseParamUpdateRecorder {
     auto& param_name = grad_to_param_.at(grad_name);
     auto& trainer_to_rows = param_to_updated_rows_.at(param_name);
 
+    std::vector<std::future<void>> fs;
     for (auto& set : trainer_to_rows) {
+      fs.push_back(set->Update(update_rows));
+    }
+    for (auto& f : fs) {
       // no need to wait here because GetAndClear will wait.
-      set->Update(update_rows);
+      // f.wait();
     }
   }