diff --git a/paddle/fluid/operators/distributed/heart_beat_monitor.h b/paddle/fluid/operators/distributed/heart_beat_monitor.h
index 639785ba513e6ec8286d47f04da6f09ef168ca39..211e40757fc752ecd8846a45fae73db143f481e3 100644
--- a/paddle/fluid/operators/distributed/heart_beat_monitor.h
+++ b/paddle/fluid/operators/distributed/heart_beat_monitor.h
@@ -84,14 +84,7 @@ class HeartBeatMonitor {
                    be_monitored_var);
   }
 
-  static HeartBeatMonitor* GetInstance() {
-    if (monitor_ == nullptr) {
-      PADDLE_THROW(
-          "HeartBeatMonitor is not inited, call "
-          "HeartBeatMonitor::Init first");
-    }
-    return monitor_.get();
-  }
+  static HeartBeatMonitor* GetInstance() { return monitor_.get(); }
 
   void Stop() {
     running_ = false;
diff --git a/paddle/fluid/operators/distributed/request_handler_impl.cc b/paddle/fluid/operators/distributed/request_handler_impl.cc
index 96098a4e22632ca57c2efbee5374f0c8806bfbe7..9a7da5f8f924ba37e3bf77b673a71ebf23519275 100644
--- a/paddle/fluid/operators/distributed/request_handler_impl.cc
+++ b/paddle/fluid/operators/distributed/request_handler_impl.cc
@@ -53,7 +53,11 @@ bool RequestSendHandler::Handle(const std::string& varname,
     rpc_server_->IncreaseBatchBarrier(kRequestSend);
   } else if (varname == COMPLETE_MESSAGE) {
     VLOG(3) << "sync: recv complete message";
-    HeartBeatMonitor::GetInstance()->Update(trainer_id, "", COMPLETED);
+
+    if (HeartBeatMonitor::GetInstance() != nullptr) {
+      HeartBeatMonitor::GetInstance()->Update(trainer_id, "", COMPLETED);
+    }
+
     rpc_server_->Complete();
   } else {
     // Async