Fix a dead lock bug for dyload/nccl.h when nccl lib cannot be loaded (#5533)

It caused by a bug of std::call_once described in https://stackoverflow.com/questions/41717579/stdcall-once-hangs-on-second-call-after-callable-threw-on-first-call. It is likely caused by a deeper bug of pthread_once, which is discussed in https://patchwork.ozlabs.org/patch/482350/

paddle/operators/nccl/nccl_gpu_common.h

@@ -35,6 +35,7 @@ constexpr int kInvalidGPUId = -1;
 struct Communicator {
   std::vector<ncclComm_t> comms_;
   std::unordered_map<int, int> comm_id_map_;
+  bool inited_;
 
   Communicator() {}
 
@@ -42,19 +43,23 @@ struct Communicator {
 
   void InitAll(const std::vector<int>& gpus) {
     comms_.resize(gpus.size());
+    inited_ = false;
     for (size_t i = 0; i < gpus.size(); ++i) {
       comm_id_map_[gpus[i]] = i;
     }
     PADDLE_ENFORCE(
         dynload::ncclCommInitAll(comms_.data(), gpus.size(), gpus.data()));
+    inited_ = true;
   }
 
   ~Communicator() {
+    if (inited_) {
       for (size_t i = 0; i < comms_.size(); ++i) {
         // FIXME(dzh) : PADDLE_ENFORCE return void
         dynload::ncclCommDestroy(comms_[i]);
       }
     }
+  }
 
   DISABLE_COPY_AND_ASSIGN(Communicator);
 };

paddle/platform/call_once.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserve.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License. */
+
+#pragma once
+
+#include <mutex>
+
+namespace paddle {
+namespace platform {
+
+/*
+ The current implementation of std::call_once has a bug described in
+ https://stackoverflow.com/questions/41717579/stdcall-once-hangs-on-second-call-after-callable-threw-on-first-call.
+ This is likely caused by a deeper bug of pthread_once, which is discussed in
+ https://patchwork.ozlabs.org/patch/482350/
+
+ This wrap is a hack to avoid this bug.
+*/
+template <class Callable, class... Args>
+inline void call_once(std::once_flag& flag, Callable&& f, Args&&... args) {
+  bool good = false;
+  std::exception ex;
+  std::call_once(flag, [&]() {
+    try {
+      f(args...);
+      good = true;
+    } catch (const std::exception& e) {
+      ex = e;
+    } catch (...) {
+      ex = std::runtime_error("excption caught in call_once");
+    }
+  });
+  if (!good) {
+    throw std::exception(ex);
+  }
+}
+
+}  // namespace platform
+}  // namespace paddle

paddle/platform/dynload/nccl.h

@@ -17,6 +17,7 @@
 #include <dlfcn.h>
 #include <nccl.h>
 #include <mutex>
+#include "paddle/platform/call_once.h"
 #include "paddle/platform/dynload/dynamic_loader.h"
 
 namespace paddle {
@@ -32,7 +33,7 @@ extern void* nccl_dso_handle;
     template <typename... Args>                                        \
     auto operator()(Args... args) -> decltype(__name(args...)) {       \
       using nccl_func = decltype(__name(args...)) (*)(Args...);        \
-      std::call_once(nccl_dso_flag,                               \
+      platform::call_once(nccl_dso_flag,                               \
                           paddle::platform::dynload::GetNCCLDsoHandle, \
                           &nccl_dso_handle);                           \
       void* p_##__name = dlsym(nccl_dso_handle, #__name);              \