[Cherry-pick] Disable gloo by default #29559 #29805 (#29601)

* update, test=develop (#29559)

* Disable gloo by default (#29805)

* update, test=develop

* update, test=develop

paddle/fluid/framework/fleet/gloo_wrapper.cc

@@ -272,8 +272,7 @@ void GlooWrapper::Init() {
   attr.iface = iface_;
   std::shared_ptr<gloo::rendezvous::HdfsStore> file_store = nullptr;
   std::shared_ptr<gloo::rendezvous::HTTPStore> http_store = nullptr;
-  auto context =
-      std::make_shared<gloo::rendezvous::ParallelConnectContext>(rank_, size_);
+  auto context = std::make_shared<gloo::rendezvous::Context>(rank_, size_);
   context->setTimeout(run_timeout_);
   auto dev = gloo::transport::tcp::CreateDevice(attr);
   switch (store_type_) {
@@ -295,6 +294,7 @@ void GlooWrapper::Init() {
       http_store->SetTimeoutSeconds(init_timeout_.count());
       context->connectFullMesh(*http_store, dev);
       http_store->Finalize();
+      VLOG(3) << "after calling http_store->Finalize.";
       break;
     }
     default:
@@ -304,6 +304,7 @@ void GlooWrapper::Init() {
   context_ = std::move(context);
 #endif
   is_initialized_ = true;
+  VLOG(3) << "gloo initialized done.";
 }
 
 template std::vector<int64_t> GlooWrapper::AllReduce<int64_t>(

python/paddle/distributed/fleet/base/role_maker.py

@@ -220,15 +220,8 @@ class Gloo(object):
             rank, nodes = self._get_rank_nodes(Role.WORKER)
             gloo = init(rank, nodes, "WORKER")
             self._worker_comm = gloo
-        else:
-            rank, nodes = self._get_rank_nodes(Role.SERVER)
-            gloo = init(rank, nodes, "SERVER")
-            self._server_comm = gloo
+        # TODO (sandyhouse): initialize gloo for server and all
 
-        if self._need_init_all:
-            rank, nodes = self._get_rank_nodes(Role.ALL)
-            gloo = init(rank, nodes, "ALL")
-            self._nodes_comm = gloo
         if start_http_server:
             http_server_d["running"] = False
             http_server.join()

python/paddle/distributed/fleet/launch.py

@@ -219,7 +219,7 @@ def launch_collective(args):
     global_envs = copy.copy(os.environ.copy())
     gloo_rendezvous_dir = tempfile.mkdtemp()
     # add gloo env
-    global_envs["PADDLE_WITH_GLOO"] = str(os.getenv("PADDLE_WITH_GLOO", "1"))
+    global_envs["PADDLE_WITH_GLOO"] = str(os.getenv("PADDLE_WITH_GLOO", "0"))
     global_envs["PADDLE_GLOO_RENDEZVOUS"] = "3"
     global_envs["PADDLE_GLOO_FS_PATH"] = gloo_rendezvous_dir
 

python/paddle/distributed/fleet/launch_utils.py

@@ -955,7 +955,7 @@ class ParameterServerLauncher(object):
                 "TRAINING_ROLE": "PSERVER",
                 "PADDLE_TRAINERS_NUM": str(self.worker_num),
                 "POD_IP": cur_server.endpoint.split(":")[0],
-                "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "1")),
+                "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")),
                 "PADDLE_GLOO_RENDEZVOUS": "3",
                 "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir,
                 "PADDLE_GLOO_HTTP_ENDPOINT": self.http_port
@@ -1019,7 +1019,7 @@ class ParameterServerLauncher(object):
                 self.heter_worker_endpoints,
                 "TRAINING_ROLE": "TRAINER",
                 "PADDLE_TRAINER_ID": str(cur_worker.rank),
-                "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "1")),
+                "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")),
                 "PADDLE_GLOO_RENDEZVOUS": "3",
                 "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir,
                 "FLAGS_selected_gpus": "0",
@@ -1089,7 +1089,7 @@ class ParameterServerLauncher(object):
                 "TRAINING_ROLE": "HETER_TRAINER",
                 "PADDLE_TRAINERS_NUM": str(self.worker_num),
                 "POD_IP": cur_heter_worker.endpoint.split(":")[0],
-                "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "1")),
+                "PADDLE_WITH_GLOO": str(os.getenv("PADDLE_WITH_GLOO", "0")),
                 "PADDLE_GLOO_RENDEZVOUS": "3",
                 "PADDLE_GLOO_FS_PATH": self.gloo_rendezvous_dir,
                 "FLAGS_selected_gpus": "0",

python/paddle/distributed/parallel.py

@@ -142,6 +142,8 @@ def init_parallel_env():
     _check_var_exists("PADDLE_TRAINER_ENDPOINTS")
 
     # 3: init gloo context (step 1: httpsever start)
+    init_gloo = int(os.getenv("PADDLE_WITH_GLOO", "0"))
+    if init_gloo:
         ep_rank_0 = parallel_env.trainer_endpoints[0].split(":")
         ep_rank = parallel_env.trainer_endpoints[parallel_env.rank].split(":")
         manager = Manager()
@@ -185,6 +187,7 @@ def init_parallel_env():
     # dividing init_gloo into two part beacause nccl and gloo
     # are separately looking for free ports which sometimes
     # leads to port-conflict.
+    if init_gloo:
         wait_server_ready([parallel_env.trainer_endpoints[0]])
 
         gloo_strategy = core.GlooParallelStrategy()

python/paddle/fluid/tests/unittests/test_collective_api_base.py

@@ -169,6 +169,7 @@ class TestDistBase(unittest.TestCase):
                          path_id="0",
                          check_error_log=False,
                          need_envs={}):
+        with_gloo = '0' if backend == "nccl" else '1'
         required_envs = {
             "FLAGS_fraction_of_gpu_memory_to_use": "0.15",
             "FLAGS_eager_delete_tensor_gb": "0.0",
@@ -178,6 +179,7 @@ class TestDistBase(unittest.TestCase):
             "LD_PRELOAD": os.getenv("LD_PRELOAD", ""),
             "GLOG_v": "0",
             "NCCL_P2P_DISABLE": "1",
+            "PADDLE_WITH_GLOO": with_gloo,
             "BACKEND": backend,
             "PATH_ID": path_id
         }