diff --git a/python/paddle/distributed/collective.py b/python/paddle/distributed/collective.py
index 77258f7036cbd9a163310fabd8b1abd35c76db37..6825dae045f150850340d47bcfcf80684f64426f 100644
--- a/python/paddle/distributed/collective.py
+++ b/python/paddle/distributed/collective.py
@@ -409,11 +409,8 @@ def new_group(ranks=None, backend=None, timeout=_default_timeout):
         # TODO(shenliang03): This is a temporary solution to solve the problem of
         # hang caused by tcp
         paddle.distributed.barrier(group=group)
-        # NOTE(liyurui): All processors should hang and wait using tcp store, in case master exit before sub-group is created.
-        if backend != 'heter':
-            _barrier_by_tcp_store(group_name, _default_store, timeout)
-        else:
-            print("Warning: store barrier is not supported for heter backend.")
+        if paddle.distributed.get_world_size() > 1:
+            paddle.distributed.barrier()
         return group
 
     if not backend: