[CustomDevice] Improve device initialize & release (#57099)

paddle/phi/backends/device_manager.cc

@@ -30,10 +30,17 @@
 namespace phi {
 
 void Device::CheckInitialized() {
-  std::call_once(initialized_, [&]() { this->impl_->InitDevice(dev_id_); });
+  std::call_once(initialized_once_flag_, [&]() {
+    this->impl_->InitDevice(dev_id_);
+    this->initialized_ = true;
+  });
 }
 
-Device::~Device() { impl_->DeInitDevice(dev_id_); }
+Device::~Device() {
+  if (initialized_) {
+    impl_->DeInitDevice(dev_id_);
+  }
+}
 
 void Device::CreateStream(stream::Stream* stream,
                           const stream::Stream::Priority& priority,

paddle/phi/backends/device_manager.h

@@ -127,7 +127,8 @@ class Device final {
  private:
   size_t dev_id_;
   DeviceInterface* impl_;
-  std::once_flag initialized_;
+  std::once_flag initialized_once_flag_;
+  bool initialized_{false};
 };
 
 class DeviceManager {