diff --git a/python/paddle/distributed/fleet/base/role_maker.py b/python/paddle/distributed/fleet/base/role_maker.py
index b6be992ad1e92f81a3b758fea6164cf6322418a8..276d56ea12df15eaf86a630a93b8e9abc91b2ef0 100644
--- a/python/paddle/distributed/fleet/base/role_maker.py
+++ b/python/paddle/distributed/fleet/base/role_maker.py
@@ -175,7 +175,8 @@ class Gloo(object):
             http_server = KVServer(port, size_d)
             http_server.start()
             wait_seconds = 5
-            while http_server_d.get("running", False):
+            while http_server_d.get("running",
+                                    False) or not http_server.should_stop():
                 time.sleep(wait_seconds)
             http_server.stop()