diff --git a/cmake/phi.cmake b/cmake/phi.cmake
index 82d04f0c4695d6f1cbeb8a905040f7474e353364..9f716969dcdec8cfdddfbbca3d1c012d590ead31 100644
--- a/cmake/phi.cmake
+++ b/cmake/phi.cmake
@@ -112,354 +112,6 @@ function(kernel_declare TARGET_LIST)
   endforeach()
 endfunction()
 
-function(kernel_library TARGET)
-  return()
-  set(common_srcs)
-  set(cpu_srcs)
-  set(gpu_srcs)
-  set(xpu_srcs)
-  set(gpudnn_srcs)
-  set(kps_srcs)
-  # parse and save the deps kerenl targets
-  set(all_srcs)
-  set(kernel_deps)
-
-  set(oneValueArgs SUB_DIR)
-  set(multiValueArgs SRCS DEPS)
-  set(target_build_flag 1)
-
-  cmake_parse_arguments(kernel_library "${options}" "${oneValueArgs}"
-                        "${multiValueArgs}" ${ARGN})
-
-  # used for cc_library selected_rows dir target
-  set(target_suffix "")
-  if("${kernel_library_SUB_DIR}" STREQUAL "selected_rows")
-    set(target_suffix "_sr")
-  endif()
-  if("${kernel_library_SUB_DIR}" STREQUAL "sparse")
-    set(target_suffix "_sp")
-  endif()
-
-  list(LENGTH kernel_library_SRCS kernel_library_SRCS_len)
-  # one kernel only match one impl file in each backend
-  if(${kernel_library_SRCS_len} EQUAL 0)
-    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cc)
-      list(APPEND common_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.cc)
-    endif()
-    if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/cpu/${TARGET}.cc)
-      list(APPEND cpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/cpu/${TARGET}.cc)
-    endif()
-    if(WITH_GPU OR WITH_ROCM)
-      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gpu/${TARGET}.cu)
-        list(APPEND gpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/gpu/${TARGET}.cu)
-      endif()
-      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gpu/${TARGET}.cu.cc)
-        list(APPEND gpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/gpu/${TARGET}.cu.cc)
-      endif()
-      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/kps/${TARGET}.cu)
-        list(APPEND gpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/kps/${TARGET}.cu)
-      endif()
-      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/gpudnn/${TARGET}.cu)
-        list(APPEND gpudnn_srcs ${CMAKE_CURRENT_SOURCE_DIR}/gpudnn/${TARGET}.cu)
-      endif()
-    endif()
-    if(WITH_XPU)
-      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/xpu/${TARGET}.cc)
-        list(APPEND xpu_srcs ${CMAKE_CURRENT_SOURCE_DIR}/xpu/${TARGET}.cc)
-      endif()
-    endif()
-    if(WITH_XPU_KP)
-      if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/kps/${TARGET}.cu)
-        # Change XPU2 file suffix
-        # NOTE(chenweihang): If we can be sure that the *.kps suffix is no longer used, it can be copied directly to *.xpu
-        file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/kps/${TARGET}.cu
-             DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/kps)
-        file(RENAME ${CMAKE_CURRENT_BINARY_DIR}/kps/${TARGET}.cu
-             ${CMAKE_CURRENT_BINARY_DIR}/kps/${TARGET}.kps)
-        list(APPEND kps_srcs ${CMAKE_CURRENT_BINARY_DIR}/kps/${TARGET}.kps)
-      endif()
-    endif()
-  else()
-    # TODO(chenweihang): impl compile by source later
-  endif()
-
-  list(APPEND all_srcs ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET}.h)
-  if(EXISTS ${CMAKE_CURRENT_SOURCE_DIR}/impl/${TARGET}_impl.h)
-    list(APPEND all_srcs ${CMAKE_CURRENT_SOURCE_DIR}/impl/${TARGET}_impl.h)
-  endif()
-  list(APPEND all_srcs ${common_srcs})
-  list(APPEND all_srcs ${cpu_srcs})
-  list(APPEND all_srcs ${gpu_srcs})
-  list(APPEND all_srcs ${xpu_srcs})
-  list(APPEND all_srcs ${gpudnn_srcs})
-  list(APPEND all_srcs ${kps_srcs})
-
-  set(all_include_kernels)
-  set(all_kernel_name)
-
-  foreach(src ${all_srcs})
-    file(READ ${src} target_content)
-    # "kernels/xxx"(DenseTensor Kernel) can only include each other, but can't include "SUB_DIR/xxx" (such as selected_rows Kernel)
-    string(REGEX MATCHALL
-                 "#include \"paddle\/phi\/kernels\/[a-z0-9_]+_kernel.h\""
-                 include_kernels ${target_content})
-    list(APPEND all_include_kernels ${include_kernels})
-
-    # "SUB_DIR/xxx" can include "kernels/xx" and "SUB_DIR/xxx"
-    if(NOT "${kernel_library_SUB_DIR}" STREQUAL "")
-      string(
-        REGEX
-          MATCHALL
-          "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/[a-z0-9_]+_kernel.h\""
-          include_kernels
-          ${target_content})
-      list(APPEND all_include_kernels ${include_kernels})
-    endif()
-
-    foreach(include_kernel ${all_include_kernels})
-      if("${kernel_library_SUB_DIR}" STREQUAL "")
-        string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/" "" kernel_name
-                             ${include_kernel})
-        string(REGEX REPLACE ".h\"" "" kernel_name ${kernel_name})
-        list(APPEND all_kernel_name ${kernel_name})
-      else()
-        # NOTE(dev): we should firstly match kernel_library_SUB_DIR.
-        if(${include_kernel} MATCHES
-           "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/")
-          string(
-            REGEX
-            REPLACE
-              "#include \"paddle\/phi\/kernels\/${kernel_library_SUB_DIR}\/" ""
-              kernel_name ${include_kernel})
-          # for selected_rows directory, add ${target_suffix}.
-          string(REGEX REPLACE ".h\"" "${target_suffix}" kernel_name
-                               ${kernel_name})
-          list(APPEND all_kernel_name ${kernel_name})
-        else()
-          string(REGEX REPLACE "#include \"paddle\/phi\/kernels\/" ""
-                               kernel_name ${include_kernel})
-          string(REGEX REPLACE ".h\"" "" kernel_name ${kernel_name})
-          list(APPEND all_kernel_name ${kernel_name})
-        endif()
-      endif()
-      list(APPEND kernel_deps ${all_kernel_name})
-    endforeach()
-  endforeach()
-  list(REMOVE_DUPLICATES kernel_deps)
-  list(REMOVE_ITEM kernel_deps ${TARGET}${target_suffix})
-
-  list(LENGTH common_srcs common_srcs_len)
-  list(LENGTH cpu_srcs cpu_srcs_len)
-  list(LENGTH gpu_srcs gpu_srcs_len)
-  list(LENGTH xpu_srcs xpu_srcs_len)
-  list(LENGTH gpudnn_srcs gpudnn_srcs_len)
-  list(LENGTH kps_srcs kps_srcs_len)
-
-  # kernel source file level
-  # level 1: base device kernel (if any device or dnn kernel exists, the cpu_kernel must be exists!!!)
-  # - cpu_srcs / gpu_srcs / xpu_srcs / kps_srcs
-  # = dnn srcs: gpudnn_srcs
-  # level 2: device-independent kernel
-  # - common_srcs
-
-  set(partial_build_flag 0)
-  set(base_build_flag 0)
-  if(${common_srcs_len} GREATER 0)
-    set(partial_build_flag 1)
-  endif()
-  if(${cpu_srcs_len} GREATER 0
-     OR ${gpu_srcs_len} GREATER 0
-     OR ${xpu_srcs_len} GREATER 0
-     OR ${kps_srcs_len} GREATER 0)
-    set(base_build_flag 1)
-  endif()
-
-  # gpudnn or mkldnn needs to be compiled separately
-  set(dnn_kernels)
-  if(${gpudnn_srcs_len} GREATER 0)
-    if(WITH_GPU)
-      nv_library(
-        ${TARGET}_gpudnn${target_suffix}
-        SRCS ${gpudnn_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-    elseif(WITH_ROCM)
-      hip_library(
-        ${TARGET}_gpudnn${target_suffix}
-        SRCS ${gpudnn_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-    endif()
-    list(APPEND dnn_kernels ${TARGET}_gpudnn${target_suffix})
-  endif()
-  list(LENGTH dnn_kernels dnn_kernels_len)
-
-  if(${partial_build_flag} EQUAL 0 AND ${base_build_flag} EQUAL 1)
-    if(WITH_GPU)
-      if(${dnn_kernels_len} GREATER 0)
-        nv_library(
-          ${TARGET}_base${target_suffix}
-          SRCS ${cpu_srcs} ${gpu_srcs}
-          DEPS ${kernel_library_DEPS} ${kernel_deps})
-        nv_library(${TARGET}${target_suffix} DEPS ${TARGET}_base${target_suffix}
-                                                  ${dnn_kernels})
-      else()
-        nv_library(
-          ${TARGET}${target_suffix}
-          SRCS ${cpu_srcs} ${gpu_srcs}
-          DEPS ${kernel_library_DEPS} ${kernel_deps})
-      endif()
-    elseif(WITH_ROCM)
-      if(${dnn_kernels_len} GREATER 0)
-        hip_library(
-          ${TARGET}_base${target_suffix}
-          SRCS ${cpu_srcs} ${gpu_srcs}
-          DEPS ${kernel_library_DEPS} ${kernel_deps})
-        hip_library(${TARGET}${target_suffix}
-                    DEPS ${TARGET}_base${target_suffix} ${dnn_kernels})
-      else()
-        hip_library(
-          ${TARGET}${target_suffix}
-          SRCS ${cpu_srcs} ${gpu_srcs}
-          DEPS ${kernel_library_DEPS} ${kernel_deps})
-      endif()
-    elseif(WITH_XPU_KP)
-      xpu_library(
-        ${TARGET}${target_suffix}
-        SRCS ${cpu_srcs} ${kps_srcs} ${xpu_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-    else()
-      cc_library(
-        ${TARGET}${target_suffix}
-        SRCS ${cpu_srcs} ${xpu_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-    endif()
-  elseif(${partial_build_flag} EQUAL 1 AND ${base_build_flag} EQUAL 1)
-    if(WITH_GPU)
-      nv_library(
-        ${TARGET}_base${target_suffix}
-        SRCS ${cpu_srcs} ${gpu_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-      nv_library(
-        ${TARGET}${target_suffix}
-        SRCS ${common_srcs}
-        DEPS ${TARGET}_base${target_suffix} ${dnn_kernels})
-    elseif(WITH_ROCM)
-      hip_library(
-        ${TARGET}_base${target_suffix}
-        SRCS ${cpu_srcs} ${gpu_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-      hip_library(
-        ${TARGET}${target_suffix}
-        SRCS ${common_srcs}
-        DEPS ${TARGET}_base${target_suffix} ${dnn_kernels})
-    elseif(WITH_XPU_KP)
-      xpu_library(
-        ${TARGET}_base${target_suffix}
-        SRCS ${cpu_srcs} ${kps_srcs} ${xpu_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-      xpu_library(
-        ${TARGET}${target_suffix}
-        SRCS ${common_srcs}
-        DEPS ${TARGET}_base${target_suffix})
-    else()
-      cc_library(
-        ${TARGET}_base${target_suffix}
-        SRCS ${cpu_srcs} ${xpu_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-      cc_library(
-        ${TARGET}${target_suffix}
-        SRCS ${common_srcs}
-        DEPS ${TARGET}_base${target_suffix})
-    endif()
-  elseif(${partial_build_flag} EQUAL 1 AND ${base_build_flag} EQUAL 0)
-    if(WITH_GPU)
-      nv_library(
-        ${TARGET}${target_suffix}
-        SRCS ${common_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-    elseif(WITH_ROCM)
-      hip_library(
-        ${TARGET}${target_suffix}
-        SRCS ${common_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-    elseif(WITH_XPU_KP)
-      xpu_library(
-        ${TARGET}${target_suffix}
-        SRCS ${common_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-    else()
-      cc_library(
-        ${TARGET}${target_suffix}
-        SRCS ${common_srcs}
-        DEPS ${kernel_library_DEPS} ${kernel_deps})
-    endif()
-  else()
-    set(target_build_flag 0)
-  endif()
-
-  if(${target_build_flag} EQUAL 1)
-    if(${common_srcs_len} GREATER 0
-       OR ${cpu_srcs_len} GREATER 0
-       OR ${gpu_srcs_len} GREATER 0
-       OR ${xpu_srcs_len} GREATER 0
-       OR ${kps_srcs_len} GREATER 0
-       OR ${gpudnn_srcs_len} GREATER 0)
-      # append target into PHI_KERNELS property
-      get_property(phi_kernels GLOBAL PROPERTY PHI_KERNELS)
-      set(phi_kernels ${phi_kernels} ${TARGET}${target_suffix})
-      set_property(GLOBAL PROPERTY PHI_KERNELS ${phi_kernels})
-    endif()
-
-    # parse kernel name and auto generate kernel declaration
-    # here, we don't need to check WITH_XXX, because if not WITH_XXX, the
-    # xxx_srcs_len will be equal to 0
-    if(${common_srcs_len} GREATER 0)
-      kernel_declare(${common_srcs})
-    endif()
-    if(${cpu_srcs_len} GREATER 0)
-      kernel_declare(${cpu_srcs})
-    endif()
-    if(${gpu_srcs_len} GREATER 0)
-      kernel_declare(${gpu_srcs})
-    endif()
-    if(${xpu_srcs_len} GREATER 0)
-      kernel_declare(${xpu_srcs})
-    endif()
-    if(${gpudnn_srcs_len} GREATER 0)
-      kernel_declare(${gpudnn_srcs})
-    endif()
-    if(${kps_srcs_len} GREATER 0)
-      kernel_declare(${kps_srcs})
-    endif()
-  endif()
-endfunction()
-
-function(register_kernels)
-  set(options "")
-  set(oneValueArgs SUB_DIR)
-  set(multiValueArgs EXCLUDES DEPS)
-  cmake_parse_arguments(register_kernels "${options}" "${oneValueArgs}"
-                        "${multiValueArgs}" ${ARGN})
-
-  file(
-    GLOB KERNELS
-    RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
-    "*_kernel.h")
-  string(REPLACE ".h" "" KERNELS "${KERNELS}")
-  list(LENGTH register_kernels_DEPS register_kernels_DEPS_len)
-
-  foreach(target ${KERNELS})
-    list(FIND register_kernels_EXCLUDES ${target} _index)
-    if(${_index} EQUAL -1)
-      if(${register_kernels_DEPS_len} GREATER 0)
-        kernel_library(${target} DEPS ${register_kernels_DEPS} SUB_DIR
-                       ${register_kernels_SUB_DIR})
-      else()
-        kernel_library(${target} SUB_DIR ${register_kernels_SUB_DIR})
-      endif()
-    endif()
-  endforeach()
-endfunction()
-
 function(append_op_util_declare TARGET)
   file(READ ${CMAKE_CURRENT_SOURCE_DIR}/${TARGET} target_content)
   string(
diff --git a/paddle/phi/kernels/CMakeLists.txt b/paddle/phi/kernels/CMakeLists.txt
index 7cddacb7e3928d4dd80b9a79e5e92bc1cdac05e0..ad71823e3c019a59d75ca926765d55861b877109 100644
--- a/paddle/phi/kernels/CMakeLists.txt
+++ b/paddle/phi/kernels/CMakeLists.txt
@@ -81,8 +81,6 @@ set(COMMON_KERNEL_DEPS
     utf8proc)
 
 copy_if_different(${kernel_declare_file} ${kernel_declare_file_final})
-# For strings kernels
-add_subdirectory(strings)
 
 file(GLOB kernel_h "*.h" "selected_rows/*.h" "sparse/*.h" "strings/*.h")
 file(GLOB kernel_impl_h "impl/*.h" "selected_rows/impl/*.h")
diff --git a/paddle/phi/kernels/selected_rows/CMakeLists.txt b/paddle/phi/kernels/selected_rows/CMakeLists.txt
deleted file mode 100644
index 520536d82352a697666ca5a9ff1c1640e7a95f49..0000000000000000000000000000000000000000
--- a/paddle/phi/kernels/selected_rows/CMakeLists.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-set(SELECTED_ROWS_KERNEL_DEPS
-    dense_tensor
-    selected_rows
-    selected_rows_functor
-    sparse_csr_tensor
-    kernel_context
-    kernel_factory
-    arg_map_context
-    convert_utils
-    lod_utils
-    math_function
-    custom_kernel)
-register_kernels(DEPS ${SELECTED_ROWS_KERNEL_DEPS} SUB_DIR "selected_rows")
diff --git a/paddle/phi/kernels/sparse/CMakeLists.txt b/paddle/phi/kernels/sparse/CMakeLists.txt
deleted file mode 100644
index 6c5e7dee4cb35547aa785d39fa76586a6f2b7289..0000000000000000000000000000000000000000
--- a/paddle/phi/kernels/sparse/CMakeLists.txt
+++ /dev/null
@@ -1,13 +0,0 @@
-set(SPARSE_KERNEL_DEPS
-    dense_tensor
-    sparse_coo_tensor
-    sparse_csr_tensor
-    kernel_context
-    kernel_factory
-    arg_map_context
-    convert_utils
-    lod_utils
-    math_function
-    custom_kernel
-    copy_kernel)
-register_kernels(DEPS ${SPARSE_KERNEL_DEPS} SUB_DIR "sparse")
diff --git a/paddle/phi/kernels/strings/CMakeLists.txt b/paddle/phi/kernels/strings/CMakeLists.txt
deleted file mode 100644
index 7cbba08e1618918490d7f79344b69b25e47e08da..0000000000000000000000000000000000000000
--- a/paddle/phi/kernels/strings/CMakeLists.txt
+++ /dev/null
@@ -1,29 +0,0 @@
-add_subdirectory(cpu)
-if(WITH_GPU OR WITH_ROCM)
-  add_subdirectory(gpu)
-endif()
-
-cc_library(
-  unicode
-  SRCS unicode.cc
-  DEPS utf8proc)
-set_property(GLOBAL PROPERTY STRING_KERNELS "")
-
-set(STRING_KERNEL_DEPS
-    dense_tensor
-    string_tensor
-    sparse_coo_tensor
-    sparse_csr_tensor
-    kernel_context
-    kernel_factory
-    arg_map_context
-    convert_utils
-    lod_utils
-    custom_kernel)
-set(STRING_KERNEL_DEPS ${STRING_KERNEL_DEPS} eigen_function blas math_function)
-# remove this dep after removing fluid deps on tensor creation
-set(STRING_KERNEL_DEPS ${STRING_KERNEL_DEPS} phi_api_utils)
-set(STRING_KERNEL_DEPS ${STRING_KERNEL_DEPS} string_infermeta)
-set(STRING_KERNEL_DEPS ${STRING_KERNEL_DEPS} unicode)
-
-register_kernels(DEPS ${STRING_KERNEL_DEPS} SUB_DIR "strings")
diff --git a/paddle/phi/kernels/strings/cpu/CMakeLists.txt b/paddle/phi/kernels/strings/cpu/CMakeLists.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000
diff --git a/paddle/phi/kernels/strings/gpu/CMakeLists.txt b/paddle/phi/kernels/strings/gpu/CMakeLists.txt
deleted file mode 100644
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000