remove useless cmake list (#39141)

paddle/pten/kernels/CMakeLists.txt

@@ -2,23 +2,9 @@ set(kernel_declare_file ${PADDLE_BINARY_DIR}/paddle/pten/kernels/declarations.h.
 set(kernel_declare_file_final ${PADDLE_BINARY_DIR}/paddle/pten/kernels/declarations.h)
 file(WRITE ${kernel_declare_file} "// Generated by the paddle/pten/kernels/CMakeLists.txt.  DO NOT EDIT!\n\n#pragma once\n\n")
 
-# kernel primitive api
-add_subdirectory(primitive)
 # pten functors and functions called by kernels
 add_subdirectory(funcs)
 
-add_subdirectory(cpu)
-if(WITH_GPU OR WITH_ROCM)
-  add_subdirectory(gpu)
-endif()
-if(WITH_MKLDNN)
-  # mkldnn will be deprecated and use the new name dnnl
-  add_subdirectory(dnnl)
-endif()
-if(WITH_XPU)
-  add_subdirectory(xpu)
-endif()
-
 # pten depends all pten kernel targets
 set_property(GLOBAL PROPERTY PTEN_KERNELS "")