diff --git a/paddle/framework/executor.cc b/paddle/framework/executor.cc
index 7fda2332b872280c836aaf0367219b25b6cf43ab..b38d6be16f2aa99027cc1eab3d8509bbf3247a04 100644
--- a/paddle/framework/executor.cc
+++ b/paddle/framework/executor.cc
@@ -80,26 +80,34 @@ class ExecutorImpl : public Executor {
 
 template <typename T, typename... Args>
 std::unique_ptr<T> make_unique(Args&&... args) {
-  return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
+      return std::unique_ptr<T>(new T(std::forward<Args>(args)...));
 }
 
-static std::unique_ptr<platform::CPUDeviceContext> g_cpu_device_context =
-    make_unique<platform::CPUDeviceContext>(platform::CPUPlace());
+platform::CPUDeviceContext* GetCPUDeviceContext() {
+  static std::unique_ptr<platform::CPUDeviceContext> g_cpu_device_context =
+        make_unique<platform::CPUDeviceContext>(platform::CPUPlace());
+  return g_cpu_device_context.get();
+}
 
 #ifndef PADDLE_ONLY_CPU
-static std::unique_ptr<platform::CUDADeviceContext> g_cuda_device_context =
-    make_unique<platform::CUDADeviceContext>(platform::GPUPlace(0));
+platform::CUDADeviceContext* GetCUDADeviceContext() {
+  static std::unique_ptr<platform::CUDADeviceContext> g_cuda_device_context =
+          make_unique<platform::CUDADeviceContext>(platform::GPUPlace(0));
+  return g_cuda_device_context.get();
+}
 #endif
 
 Executor* NewLocalExecutor(const platform::Place& place,
                            const ProgramDesc& pdesc, bool is_linear) {
   platform::DeviceContext* device_context = nullptr;
   if (platform::is_cpu_place(place)) {
-    device_context = g_cpu_device_context.get();
-  }
+    device_context = GetCPUDeviceContext();
+  } else if (platform::is_gpu_place(place)) {
 #ifndef PADDLE_ONLY_CPU
-  else if {
-    device_context = g_cuda_device_context.get();
+    device_context = GetCUDADeviceContext();
+  }
+#else
+    PADDLE_THROW("'GPUPlace' is not supported in CPU only device.");
   }
 #endif
   return new ExecutorImpl(device_context, &pdesc, is_linear);
diff --git a/paddle/framework/executor_test.cc b/paddle/framework/executor_test.cc
index c046ae315826918028139d1c7ddb8d1061064022..6f8ca3876842c68515e22bb24006674ae9ae4725 100644
--- a/paddle/framework/executor_test.cc
+++ b/paddle/framework/executor_test.cc
@@ -23,4 +23,5 @@ TEST(Executor, Init) {
   CPUPlace cpu_place;
   Executor* executor = NewLocalExecutor(cpu_place, pdesc, true);
   executor->Run();
+  delete executor;
 }
\ No newline at end of file