diff --git a/paddle/fluid/platform/device/xpu/CMakeLists.txt b/paddle/fluid/platform/device/xpu/CMakeLists.txt
index b7134070be2db77b76d00aaf4752a92008d1d2e8..31ac51050b87bdb2f27886061c5583b48687416f 100644
--- a/paddle/fluid/platform/device/xpu/CMakeLists.txt
+++ b/paddle/fluid/platform/device/xpu/CMakeLists.txt
@@ -24,7 +24,13 @@ cc_library(
 cc_library(
   xpu_op_list
   SRCS xpu_op_list.cc
-  DEPS gflags glog enforce xpulib device_context op_kernel_type)
+  DEPS gflags
+       glog
+       enforce
+       xpulib
+       device_context
+       op_kernel_type
+       phi_xpu_op_list)
 cc_library(
   xpu_resource_pool
   SRCS xpu_resource_pool.cc
diff --git a/paddle/fluid/platform/device/xpu/xpu_op_list.cc b/paddle/fluid/platform/device/xpu/xpu_op_list.cc
index 2911b8f949ae5de9968a8c0d56ca9ed58c93853e..c5363e6d6c960d65c0a155f316e52c3690cd5dbf 100644
--- a/paddle/fluid/platform/device/xpu/xpu_op_list.cc
+++ b/paddle/fluid/platform/device/xpu/xpu_op_list.cc
@@ -49,31 +49,6 @@ static void tokenize(const std::string& ops,
   op_set->insert(ops.substr(beg));
 }
 
-bool is_in_xpu_black_list(const std::string& op_name) {
-  static bool inited = false;
-  static std::unordered_set<std::string> xpu_black_list;
-  static std::mutex s_mtx;
-  if (!inited) {
-    std::lock_guard<std::mutex> guard(s_mtx);
-    if (!inited) {
-      if (std::getenv("XPU_BLACK_LIST") != nullptr) {
-        std::string ops(std::getenv("XPU_BLACK_LIST"));
-        tokenize(ops, ',', &xpu_black_list);
-      }
-      inited = true;
-      VLOG(3) << "XPU Black List: ";
-      for (auto iter = xpu_black_list.begin(); iter != xpu_black_list.end();
-           ++iter) {
-        VLOG(3) << *iter << " ";
-      }
-    }
-  }
-  if (xpu_black_list.find(op_name) != xpu_black_list.end()) {
-    return true;
-  }
-  return false;
-}
-
 #ifdef PADDLE_WITH_XPU_KP
 bool is_xpu_kp_support_op(const std::string& op_name,
                           const pOpKernelType& type) {
diff --git a/paddle/fluid/platform/device/xpu/xpu_op_list.h b/paddle/fluid/platform/device/xpu/xpu_op_list.h
index 60926dd9a5660ee13be7d61eb453740207994029..e008e9d111c53f4690afc31c8a935a9b69180a32 100644
--- a/paddle/fluid/platform/device/xpu/xpu_op_list.h
+++ b/paddle/fluid/platform/device/xpu/xpu_op_list.h
@@ -15,6 +15,7 @@ limitations under the License. */
 #include <unordered_map>
 
 #include "paddle/fluid/framework/op_kernel_type.h"
+#include "paddle/phi/backends/xpu/xpu_op_list.h"
 
 namespace paddle {
 namespace platform {
@@ -25,7 +26,7 @@ using XPUOpListMap =
     std::unordered_map<std::string, std::vector<vartype::Type>>;
 
 bool is_xpu_support_op(const std::string& op_name, const pOpKernelType& type);
-bool is_in_xpu_black_list(const std::string& op_name);
+using phi::backends::xpu::is_in_xpu_black_list;
 
 #ifdef PADDLE_WITH_XPU_KP
 bool is_xpu_kp_support_op(const std::string& op_name,
diff --git a/paddle/phi/backends/CMakeLists.txt b/paddle/phi/backends/CMakeLists.txt
index b2095f7983f5a22efdd29e14addce679b5cb97e4..a4c76ab0e68a0cd3fc8e6826ecbae669c2139c89 100644
--- a/paddle/phi/backends/CMakeLists.txt
+++ b/paddle/phi/backends/CMakeLists.txt
@@ -16,6 +16,7 @@ if(WITH_GPU OR WITH_ROCM)
 endif()
 
 if(WITH_XPU)
+  add_subdirectory(xpu)
   list(APPEND BACKENDS_SRCS xpu/xpu_context.cc xpu/xpu_info.cc)
 endif()
 
diff --git a/paddle/phi/backends/xpu/CMakeLists.txt b/paddle/phi/backends/xpu/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..0e4fa36ea162b6e07ce1f6331552784b28d27504
--- /dev/null
+++ b/paddle/phi/backends/xpu/CMakeLists.txt
@@ -0,0 +1,4 @@
+cc_library(
+  phi_xpu_op_list
+  SRCS xpu_op_list.cc
+  DEPS glog)
diff --git a/paddle/phi/backends/xpu/xpu_op_list.cc b/paddle/phi/backends/xpu/xpu_op_list.cc
new file mode 100644
index 0000000000000000000000000000000000000000..9ec6f67c6a5ec7d9390fee941ba7b9db27e91e10
--- /dev/null
+++ b/paddle/phi/backends/xpu/xpu_op_list.cc
@@ -0,0 +1,66 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#ifdef PADDLE_WITH_XPU
+#include "paddle/phi/backends/xpu/xpu_op_list.h"
+
+#include <glog/logging.h>
+#include <mutex>
+#include <string>
+#include <unordered_set>
+
+namespace phi {
+namespace backends {
+namespace xpu {
+
+// ops_string contains op_list(e.g., 'mul,mul_grad'), parse the op string and
+// insert op to op set
+static void tokenize(const std::string& ops,
+                     char delim,
+                     std::unordered_set<std::string>* op_set) {
+  std::string::size_type beg = 0;
+  for (uint64_t end = 0; (end = ops.find(delim, end)) != std::string::npos;
+       ++end) {
+    op_set->insert(ops.substr(beg, end - beg));
+    beg = end + 1;
+  }
+
+  op_set->insert(ops.substr(beg));
+}
+
+bool is_in_xpu_black_list(const std::string& op_name) {
+  static bool inited = false;
+  static std::unordered_set<std::string> xpu_black_list;
+  static std::mutex s_mtx;
+  if (!inited) {
+    std::lock_guard<std::mutex> guard(s_mtx);
+    if (!inited) {
+      if (std::getenv("XPU_BLACK_LIST") != nullptr) {
+        std::string ops(std::getenv("XPU_BLACK_LIST"));
+        tokenize(ops, ',', &xpu_black_list);
+      }
+      inited = true;
+      VLOG(3) << "XPU Black List: ";
+      for (auto iter = xpu_black_list.begin(); iter != xpu_black_list.end();
+           ++iter) {
+        VLOG(3) << *iter << " ";
+      }
+    }
+  }
+  if (xpu_black_list.find(op_name) != xpu_black_list.end()) {
+    return true;
+  }
+  return false;
+}
+
+}  // namespace xpu
+}  // namespace backends
+}  // namespace phi
+#endif
diff --git a/paddle/phi/backends/xpu/xpu_op_list.h b/paddle/phi/backends/xpu/xpu_op_list.h
new file mode 100644
index 0000000000000000000000000000000000000000..de322f025c3aa95ec0e9ecc67336cff457016b90
--- /dev/null
+++ b/paddle/phi/backends/xpu/xpu_op_list.h
@@ -0,0 +1,25 @@
+/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+#pragma once
+
+#ifdef PADDLE_WITH_XPU
+#include <string>
+
+namespace phi {
+namespace backends {
+namespace xpu {
+
+bool is_in_xpu_black_list(const std::string& op_name);
+
+}  // namespace xpu
+}  // namespace backends
+}  // namespace phi
+#endif
diff --git a/paddle/phi/core/CMakeLists.txt b/paddle/phi/core/CMakeLists.txt
index 38fd02192810431f191b7a5d12baa1aa702abac6..8911da82b5480e478625462572f1abbac8851710 100644
--- a/paddle/phi/core/CMakeLists.txt
+++ b/paddle/phi/core/CMakeLists.txt
@@ -19,7 +19,7 @@ if(WITH_XPU)
   cc_library(
     kernel_factory
     SRCS kernel_factory.cc
-    DEPS phi_enforce convert_utils xpu_op_list)
+    DEPS phi_enforce convert_utils phi_xpu_op_list)
 else()
   cc_library(
     kernel_factory
diff --git a/paddle/phi/core/kernel_factory.cc b/paddle/phi/core/kernel_factory.cc
index 0d43d3189028b0fd13a6c1defadcc49bfa55b323..3e714d7fe37cc38ebeef00a57ab51cb799a66754 100644
--- a/paddle/phi/core/kernel_factory.cc
+++ b/paddle/phi/core/kernel_factory.cc
@@ -17,7 +17,7 @@
 #include "glog/logging.h"
 #include "paddle/phi/core/enforce.h"
 #if defined(PADDLE_WITH_XPU) && !defined(PADDLE_WITH_XPU_KP)
-#include "paddle/fluid/platform/device/xpu/xpu_op_list.h"
+#include "paddle/phi/backends/xpu/xpu_op_list.h"
 #include "paddle/phi/core/compat/convert_utils.h"
 #endif
 #include "paddle/phi/core/compat/op_utils.h"
@@ -151,7 +151,7 @@ KernelResult KernelFactory::SelectKernelOrThrowError(
 #if defined(PADDLE_WITH_XPU) && !defined(PADDLE_WITH_XPU_KP)
   VLOG(6) << "fluid_op_name: " << TransToFluidOpName(kernel_name);
   if ((FLAGS_enable_api_kernel_fallback && kernel_iter == iter->second.end()) ||
-      paddle::platform::is_in_xpu_black_list(TransToFluidOpName(kernel_name))
+      phi::backends::xpu::is_in_xpu_black_list(TransToFluidOpName(kernel_name))
 #else
   if ((FLAGS_enable_api_kernel_fallback && kernel_iter == iter->second.end())
 #endif
diff --git a/paddle/phi/kernels/xpu/cast_kernel.cc b/paddle/phi/kernels/xpu/cast_kernel.cc
index 86633eac09a5cf0fc519d2f372dbebd5250973be..06e2a1e268a934a6b0f8b66bd11c56553992be8f 100644
--- a/paddle/phi/kernels/xpu/cast_kernel.cc
+++ b/paddle/phi/kernels/xpu/cast_kernel.cc
@@ -16,13 +16,11 @@
 
 #include "paddle/phi/backends/xpu/enforce_xpu.h"
 #include "paddle/phi/backends/xpu/xpu_context.h"
+#include "paddle/phi/backends/xpu/xpu_header.h"
 #include "paddle/phi/common/float16.h"
 #include "paddle/phi/core/enforce.h"
 #include "paddle/phi/core/kernel_registry.h"
 
-// See Note [ Why still include the fluid headers? ]
-#include "paddle/fluid/platform/device/xpu/xpu_header.h"
-
 namespace phi {
 
 template <typename T, typename Context>
diff --git a/paddle/phi/kernels/xpu/nonzero_kernel.cc b/paddle/phi/kernels/xpu/nonzero_kernel.cc
index cf936f659f3fc1696b1e38533b41c3311d50f70a..d5587ac13f8ca62e41826b6f29611f054385e2e7 100644
--- a/paddle/phi/kernels/xpu/nonzero_kernel.cc
+++ b/paddle/phi/kernels/xpu/nonzero_kernel.cc
@@ -15,8 +15,8 @@
 #include "paddle/phi/kernels/nonzero_kernel.h"
 
 #include "paddle/fluid/memory/memcpy.h"
-#include "paddle/fluid/platform/device/xpu/xpu_header.h"
 #include "paddle/phi/backends/xpu/xpu_context.h"
+#include "paddle/phi/backends/xpu/xpu_header.h"
 #include "paddle/phi/core/kernel_registry.h"
 
 namespace phi {