diff --git a/paddle/fluid/operators/distributed/grpc_client.cc b/paddle/fluid/operators/distributed/grpc_client.cc
index 07ac20797ddab54296a45e99915588a40cc6f3c7..e22bc552f85b85c75f06b4158f2abac2d3843256 100644
--- a/paddle/fluid/operators/distributed/grpc_client.cc
+++ b/paddle/fluid/operators/distributed/grpc_client.cc
@@ -290,12 +290,18 @@ void GRPCClient::Proceed() {
       c->Finish(false);
     }
 
-    delete c;
+    bool notify = false;
     {
       std::lock_guard<std::mutex> lk(sync_mutex_);
       req_count_--;
+      notify = (req_count_ <= 0 || !c->status_.ok());
+    }
+
+    delete c;
+
+    if (notify) {
+      sync_cond_.notify_all();
     }
-    sync_cond_.notify_all();
   }
   VLOG(3) << "GRPCClient Proceed end";
 }