diff --git a/paddle/phi/core/CMakeLists.txt b/paddle/phi/core/CMakeLists.txt
index 669ca6c63c41e4d5498a885374b9c01d473285b2..d34f5f658b87b27d5c8ef54034536b687f674d4c 100644
--- a/paddle/phi/core/CMakeLists.txt
+++ b/paddle/phi/core/CMakeLists.txt
@@ -11,10 +11,17 @@ cc_library(
   SRCS enforce.cc
   DEPS ${phi_enforce_deps})
 
-cc_library(
-  kernel_factory
-  SRCS kernel_factory.cc
-  DEPS phi_enforce fluid_convert_utils)
+if(WITH_XPU)
+  cc_library(
+    kernel_factory
+    SRCS kernel_factory.cc
+    DEPS phi_enforce fluid_convert_utils convert_utils xpu_op_list)
+else()
+  cc_library(
+    kernel_factory
+    SRCS kernel_factory.cc
+    DEPS phi_enforce fluid_convert_utils)
+endif()
 cc_library(
   kernel_context
   SRCS kernel_context.cc
diff --git a/paddle/phi/core/kernel_factory.cc b/paddle/phi/core/kernel_factory.cc
index 6e16029ee40b551f1d35f0f034233ec5a3fd10cd..71256bdabaa67582625a8de04fdc4c86c0ca2c50 100644
--- a/paddle/phi/core/kernel_factory.cc
+++ b/paddle/phi/core/kernel_factory.cc
@@ -16,6 +16,10 @@
 
 #include "glog/logging.h"
 #include "paddle/phi/core/enforce.h"
+#if defined(PADDLE_WITH_XPU) && !defined(PADDLE_WITH_XPU_KP)
+#include "paddle/fluid/platform/device/xpu/xpu_op_list.h"
+#include "paddle/phi/core/compat/convert_utils.h"
+#endif
 
 DECLARE_bool(enable_api_kernel_fallback);
 
@@ -112,7 +116,6 @@ KernelResult KernelFactory::SelectKernelOrThrowError(
                  << "] is not registered.";
   }
 #endif
-
   auto kernel_iter = iter->second.find(kernel_key);
   // TODO(chenweihang): polish refind impl here
   if (kernel_iter == iter->second.end() &&
@@ -130,7 +133,12 @@ KernelResult KernelFactory::SelectKernelOrThrowError(
           kernel_key,
           kernel_name));
 
-  if (FLAGS_enable_api_kernel_fallback && kernel_iter == iter->second.end()) {
+  if ((FLAGS_enable_api_kernel_fallback && kernel_iter == iter->second.end())
+#if defined(PADDLE_WITH_XPU) && !defined(PADDLE_WITH_XPU_KP)
+      || paddle::platform::is_in_xpu_black_list(TransToFluidOpName(kernel_name))
+
+#endif
+  ) {
     // Fallback CPU backend
     phi::KernelKey cpu_kernel_key(
         phi::Backend::CPU, kernel_key.layout(), kernel_key.dtype());