[GPUPS]FleetWrapper initialize (#44441)

* fix FleetWrapper initialize

paddle/fluid/framework/fleet/fleet_wrapper.cc

@@ -37,6 +37,7 @@ namespace framework {
 const uint32_t MAX_FEASIGN_NUM = 1024 * 100 * 100;
 std::shared_ptr<FleetWrapper> FleetWrapper::s_instance_ = NULL;
 bool FleetWrapper::is_initialized_ = false;
+std::mutex FleetWrapper::ins_mutex;
 
 #ifdef PADDLE_WITH_PSLIB
 std::shared_ptr<paddle::distributed::PSlib> FleetWrapper::pslib_ptr_ = NULL;

paddle/fluid/framework/fleet/fleet_wrapper.h

@@ -24,6 +24,7 @@ limitations under the License. */
 #include <atomic>
 #include <ctime>
 #include <map>
+#include <mutex>
 #include <random>
 #include <string>
 #include <unordered_map>
@@ -381,9 +382,12 @@ class FleetWrapper {
   void Revert();
   // FleetWrapper singleton
   static std::shared_ptr<FleetWrapper> GetInstance() {
+    {
+      std::lock_guard<std::mutex> lk(ins_mutex);
       if (NULL == s_instance_) {
         s_instance_.reset(new paddle::framework::FleetWrapper());
       }
+    }
     return s_instance_;
   }
   // this performs better than rand_r, especially large data
@@ -397,6 +401,7 @@ class FleetWrapper {
 
  private:
   static std::shared_ptr<FleetWrapper> s_instance_;
+  static std::mutex ins_mutex;
 #ifdef PADDLE_WITH_PSLIB
   std::map<uint64_t, std::vector<paddle::ps::Region>> _regions;
 #endif

paddle/fluid/pybind/fleet_wrapper_py.cc

@@ -41,8 +41,9 @@ namespace py = pybind11;
 namespace paddle {
 namespace pybind {
 void BindFleetWrapper(py::module* m) {
-  py::class_<framework::FleetWrapper>(*m, "Fleet")
-      .def(py::init())
+  py::class_<framework::FleetWrapper, std::shared_ptr<framework::FleetWrapper>>(
+      *m, "Fleet")
+      .def(py::init([]() { return framework::FleetWrapper::GetInstance(); }))
       .def("push_dense", &framework::FleetWrapper::PushDenseVarsSync)
       .def("pull_dense", &framework::FleetWrapper::PullDenseVarsSync)
       .def("init_server", &framework::FleetWrapper::InitServer)

python/paddle/fluid/incubate/fleet/parameter_server/distribute_transpiler/__init__.py

@@ -88,6 +88,7 @@ class FleetTranspiler(Fleet):
         if role_maker is None:
             role_maker = MPISymetricRoleMaker()
         super(FleetTranspiler, self).init(role_maker)
+        if self._fleet_ptr is None:
             self._fleet_ptr = core.Fleet()
 
     def _init_transpiler_worker(self):