diff --git a/paddle/phi/backends/device_manager.cc b/paddle/phi/backends/device_manager.cc
index d95cb12646f41291ca2d2c524f61c99ec3cd2bdf..24ad5087769de5daf0b5d9076e4554a488a1c525 100644
--- a/paddle/phi/backends/device_manager.cc
+++ b/paddle/phi/backends/device_manager.cc
@@ -30,10 +30,17 @@
 namespace phi {
 
 void Device::CheckInitialized() {
-  std::call_once(initialized_, [&]() { this->impl_->InitDevice(dev_id_); });
+  std::call_once(initialized_once_flag_, [&]() {
+    this->impl_->InitDevice(dev_id_);
+    this->initialized_ = true;
+  });
 }
 
-Device::~Device() { impl_->DeInitDevice(dev_id_); }
+Device::~Device() {
+  if (initialized_) {
+    impl_->DeInitDevice(dev_id_);
+  }
+}
 
 void Device::CreateStream(stream::Stream* stream,
                           const stream::Stream::Priority& priority,
diff --git a/paddle/phi/backends/device_manager.h b/paddle/phi/backends/device_manager.h
index 62c85aeb52674b569e5aec330924e23790fd0470..58a9e6ebe7ab80fe85a7b60e9cc205047ce4db9a 100644
--- a/paddle/phi/backends/device_manager.h
+++ b/paddle/phi/backends/device_manager.h
@@ -127,7 +127,8 @@ class Device final {
  private:
   size_t dev_id_;
   DeviceInterface* impl_;
-  std::once_flag initialized_;
+  std::once_flag initialized_once_flag_;
+  bool initialized_{false};
 };
 
 class DeviceManager {