diff --git a/paddle/pten/kernels/CMakeLists.txt b/paddle/pten/kernels/CMakeLists.txt
index f838c4d424c2de98c7486cd11fd9b25f5a964e96..999f72a7e6b657613067cd9311a774b6d3a69b8d 100644
--- a/paddle/pten/kernels/CMakeLists.txt
+++ b/paddle/pten/kernels/CMakeLists.txt
@@ -2,23 +2,9 @@ set(kernel_declare_file ${PADDLE_BINARY_DIR}/paddle/pten/kernels/declarations.h.
 set(kernel_declare_file_final ${PADDLE_BINARY_DIR}/paddle/pten/kernels/declarations.h)
 file(WRITE ${kernel_declare_file} "// Generated by the paddle/pten/kernels/CMakeLists.txt.  DO NOT EDIT!\n\n#pragma once\n\n")
 
-# kernel primitive api
-add_subdirectory(primitive)
 # pten functors and functions called by kernels
 add_subdirectory(funcs)
 
-add_subdirectory(cpu)
-if(WITH_GPU OR WITH_ROCM)
-  add_subdirectory(gpu)
-endif()
-if(WITH_MKLDNN)
-  # mkldnn will be deprecated and use the new name dnnl
-  add_subdirectory(dnnl)
-endif()
-if(WITH_XPU)
-  add_subdirectory(xpu)
-endif()
-
 # pten depends all pten kernel targets
 set_property(GLOBAL PROPERTY PTEN_KERNELS "")
 
diff --git a/paddle/pten/kernels/cpu/CMakeLists.txt b/paddle/pten/kernels/cpu/CMakeLists.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/paddle/pten/kernels/dnnl/CMakeLists.txt b/paddle/pten/kernels/dnnl/CMakeLists.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/paddle/pten/kernels/gpu/CMakeLists.txt b/paddle/pten/kernels/gpu/CMakeLists.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/paddle/pten/kernels/primitive/CMakeLists.txt b/paddle/pten/kernels/primitive/CMakeLists.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/paddle/pten/kernels/xpu/CMakeLists.txt b/paddle/pten/kernels/xpu/CMakeLists.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000