diff --git a/python/paddle/distributed/parallel.py b/python/paddle/distributed/parallel.py
index 2f951d6aa92f5d925ff942725ffe27991c10fdcf..4d60db6f06dddad9990e0c4a66846b29ec8bad76 100644
--- a/python/paddle/distributed/parallel.py
+++ b/python/paddle/distributed/parallel.py
@@ -49,7 +49,7 @@ def _start_kv_server(port, http_server_d):
     http_server = KVServer(int(port))
     http_server.start()
     wait_seconds = 5
-    while http_server_d.get("running", False):
+    while http_server_d.get("running", False) or not http_server.should_stop():
         time.sleep(wait_seconds)
     http_server.stop()