compile phi/backends into one static library (#44373)

* compile into one static library

* fix xpu compile

* fix xpu compile

* fix inference compile

* fix inference compile

* add custom test

* revert one file

paddle/fluid/imperative/CMakeLists.txt

@@ -52,7 +52,7 @@ if(WITH_GPU)
   cc_library(
     layout_autotune
     SRCS layout_autotune.cc
-    DEPS op_info phi_gpu_info)
+    DEPS op_info phi_backends)
 else()
   cc_library(
     layout_autotune

paddle/fluid/memory/allocation/CMakeLists.txt

 include(ExternalProject)
 
-set(ALLOCATOR_DEPS place stats profiler)
+set(ALLOCATOR_DEPS place stats profiler phi_backends)
 set(ALLOCATOR_SRCS
     allocator.cc
     cpu_allocator.cc
@@ -54,7 +54,6 @@ endif()
 
 if(WITH_CUSTOM_DEVICE)
   list(APPEND ALLOCATOR_SRCS custom_allocator.cc)
-  list(APPEND ALLOCATOR_DEPS device_manager)
 endif()
 
 if(WITH_XPU)

paddle/fluid/platform/CMakeLists.txt

@@ -220,16 +220,9 @@ cc_library(
        ${XPU_CTX_DEPS}
        ${MLU_CTX_DEPS}
        eigen3
+       phi_backends
+       phi_device_context
        generator)
-if(WITH_XPU)
-  target_link_libraries(device_context xpu_context xpu_resource_pool)
-endif()
-
-if(WITH_MKLDNN)
-  target_link_libraries(device_context onednn_context)
-endif()
-
-target_link_libraries(device_context cpu_context)
 
 cc_library(
   collective_helper
@@ -244,22 +237,19 @@ if(WITH_CNCL)
 endif()
 
 if(WITH_GPU OR WITH_ROCM)
-  target_link_libraries(device_context gpu_info gpu_context phi_gpu_info)
   target_link_libraries(device_context gpu_resource_pool)
 endif()
-if(WITH_CUSTOM_DEVICE)
-  target_link_libraries(device_context custom_context)
-endif()
+
 if(WITH_ASCEND_CL)
   target_link_libraries(device_context npu_resource_pool)
 endif()
 
-if(WITH_MLU)
-  target_link_libraries(device_context mlu_resource_pool)
+if(WITH_XPU)
+  target_link_libraries(device_context xpu_resource_pool)
 endif()
 
-if(WITH_CUSTOM_DEVICE)
-  target_link_libraries(device_context custom_context)
+if(WITH_MLU)
+  target_link_libraries(device_context mlu_resource_pool)
 endif()
 
 cc_test(

paddle/fluid/platform/device/gpu/CMakeLists.txt

@@ -3,7 +3,7 @@ if(WITH_GPU)
   nv_library(
     gpu_info
     SRCS gpu_info.cc
-    DEPS phi_gpu_info gflags glog enforce monitor dynload_cuda)
+    DEPS phi_backends gflags glog enforce monitor dynload_cuda)
 
   nv_test(cuda_helper_test SRCS cuda_helper_test.cu)
   nv_test(
@@ -15,7 +15,7 @@ elseif(WITH_ROCM)
   hip_library(
     gpu_info
     SRCS gpu_info.cc
-    DEPS phi_gpu_info gflags glog enforce monitor dynload_cuda)
+    DEPS phi_backends gflags glog enforce monitor dynload_cuda)
 
   hip_test(cuda_helper_test SRCS cuda_helper_test.cu)
   hip_test(

paddle/fluid/platform/device/xpu/CMakeLists.txt

@@ -20,7 +20,7 @@ cc_library(
        xpulib
        device_context
        place
-       phi_xpu_info)
+       phi_backends)
 cc_library(
   xpu_op_list
   SRCS xpu_op_list.cc

paddle/phi/CMakeLists.txt

@@ -30,7 +30,7 @@ endif()
 set(PHI_DEPS
     convert_utils
     dense_tensor
-    phi_context
+    phi_backends
     kernel_factory
     kernel_context
     arg_map_context

paddle/phi/api/lib/CMakeLists.txt

@@ -351,12 +351,12 @@ cc_library(
 cc_library(
   context_pool
   SRCS context_pool.cc
-  DEPS phi_context phi_enforce place init)
+  DEPS phi_backends phi_enforce place init)
 
 cc_library(
   kernel_dispatch
   SRCS kernel_dispatch.cc
-  DEPS phi_tensor_raw phi_context kernel_factory context_pool)
+  DEPS phi_tensor_raw phi_backends kernel_factory context_pool)
 cc_library(
   api_gen_utils
   SRCS api_gen_utils.cc

paddle/phi/backends/CMakeLists.txt

 add_subdirectory(dynload)
 
-add_subdirectory(cpu)
-
-add_subdirectory(custom)
+set(BACKENDS_SRCS all_context.cc cpu/cpu_context.cc)
+set(BACKENDS_DEPS enforce place flags)
 
 if(WITH_GPU OR WITH_ROCM)
-  add_subdirectory(gpu)
+  list(APPEND BACKENDS_SRCS gpu/gpu_context.cc gpu/gpu_info.cc
+       gpu/gpu_resources.cc)
+  if(WITH_GPU)
+    list(APPEND BACKENDS_SRCS gpu/cuda/cuda_info.cc)
+  endif()
+  if(WITH_ROCM)
+    list(APPEND BACKENDS_SRCS gpu/rocm/rocm_info.cc)
+  endif()
+  list(APPEND BACKENDS_DEPS phi_dynload_cuda)
 endif()
 
 if(WITH_XPU)
-  add_subdirectory(xpu)
+  list(APPEND BACKENDS_SRCS xpu/xpu_context.cc xpu/xpu_info.cc)
 endif()
 
 if(WITH_MKLDNN)
-  add_subdirectory(onednn)
+  list(APPEND BACKENDS_SRCS onednn/onednn_context.cc)
+  list(APPEND BACKENDS_DEPS mkldnn)
 endif()
 
-cc_library(
-  phi_context
-  SRCS all_context.cc
-  DEPS device_context cpu_context)
-
-if(WITH_XPU)
-  add_dependencies(phi_context xpu_context)
+if(WITH_CUSTOM_DEVICE)
+  list(
+    APPEND
+    BACKENDS_SRCS
+    callback_manager.cc
+    device_guard.cc
+    stream.cc
+    event.cc
+    device_base.cc
+    device_manager.cc
+    custom/custom_context.cc
+    custom/custom_device.cc)
 endif()
 
-if(WITH_GPU)
-  add_dependencies(phi_context gpu_context)
-endif()
+add_library(phi_backends "${BACKENDS_SRCS}")
+target_link_libraries(phi_backends ${BACKENDS_DEPS})
+
+# for inference library
+get_property(phi_modules GLOBAL PROPERTY PHI_MODULES)
+set(phi_modules ${phi_modules} phi_backends)
+set_property(GLOBAL PROPERTY PHI_MODULES "${phi_modules}")
 
 if(WITH_CUSTOM_DEVICE)
-  add_dependencies(phi_context custom_context)
-  cc_library(
-    callback_manager
-    SRCS callback_manager.cc
-    DEPS enforce place)
-  cc_library(
-    device_guard
-    SRCS device_guard.cc
-    DEPS enforce place)
-  cc_library(
-    stream
-    SRCS stream.cc
-    DEPS callback_manager)
-  cc_library(
-    event
-    SRCS event.cc
-    DEPS enforce place)
-  cc_library(
-    device_base
-    SRCS device_base.cc
-    DEPS stream event callback_manager device_guard device_context flags)
-  cc_library(
-    device_manager
-    SRCS device_manager.cc
-    DEPS custom_device)
-  set(GLOB_DEV_LIB
-      device_manager custom_device
-      CACHE INTERNAL "Global DEV library")
+  cc_test(
+    custom_device_test
+    SRCS custom/custom_device_test.cc
+    DEPS phi_backends phi_device_context)
+  cc_test(
+    capi_test
+    SRCS custom/capi_test.cc
+    DEPS phi_capi)
 endif()

paddle/phi/backends/cpu/CMakeLists.txt

-if(WITH_MKLDNN)
-  # TODO(wilber): support mkldnn context.
-  cc_library(
-    cpu_context
-    SRCS cpu_context.cc
-    DEPS phi_device_context mkldnn eigen3)
-else()
-  cc_library(
-    cpu_context
-    SRCS cpu_context.cc
-    DEPS phi_device_context eigen3)
-endif()

paddle/phi/backends/gpu/CMakeLists.txt

-if(WITH_GPU)
-  add_subdirectory(cuda)
-  nv_library(
-    phi_gpu_info
-    SRCS gpu_info.cc
-    DEPS phi_cuda_info gflags glog enforce phi_dynload_cuda)
-elseif(WITH_ROCM)
-  add_subdirectory(rocm)
-  hip_library(
-    phi_gpu_info
-    SRCS gpu_info.cc
-    DEPS phi_rocm_info gflags glog enforce phi_dynload_cuda)
-endif()
-
-cc_library(
-  gpu_resources
-  SRCS gpu_resources.cc
-  DEPS phi_device_context phi_gpu_info)
-cc_library(
-  gpu_context
-  SRCS gpu_context.cc
-  DEPS phi_device_context phi_gpu_info eigen3 gpu_resources)

paddle/phi/backends/gpu/cuda/CMakeLists.txt

-nv_library(
-  phi_cuda_info
-  SRCS cuda_info.cc
-  DEPS gflags glog enforce phi_dynload_cuda)

paddle/phi/backends/gpu/rocm/CMakeLists.txt

-hip_library(
-  phi_rocm_info
-  SRCS rocm_info.cc
-  DEPS gflags glog enforce phi_dynload_cuda)

paddle/phi/backends/onednn/CMakeLists.txt

-if(WITH_MKLDNN)
-  cc_library(
-    onednn_context
-    SRCS onednn_context.cc
-    DEPS cpu_context mkldnn)
-endif()

paddle/phi/backends/xpu/CMakeLists.txt

-cc_library(
-  phi_xpu_info
-  SRCS xpu_info.cc
-  DEPS enforce xpulib phi_place)
-cc_library(
-  xpu_context
-  SRCS xpu_context.cc
-  DEPS phi_device_context phi_xpu_info)

paddle/phi/capi/lib/CMakeLists.txt

@@ -6,7 +6,7 @@ cc_library(
 cc_library(
   phi_c_device_context
   SRCS c_device_context.cc
-  DEPS phi_context)
+  DEPS phi_backends)
 
 cc_library(
   phi_c_int_array

paddle/phi/common/CMakeLists.txt

@@ -2,12 +2,12 @@ if(WITH_GPU)
   nv_library(
     phi_place
     SRCS place.cc
-    DEPS phi_gpu_info)
+    DEPS phi_backends)
 elseif(WITH_ROCM)
   hip_library(
     phi_place
     SRCS place.cc
-    DEPS phi_gpu_info)
+    DEPS phi_backends)
 else()
   cc_library(phi_place SRCS place.cc)
 endif()

paddle/phi/core/CMakeLists.txt

@@ -18,7 +18,7 @@ cc_library(
 cc_library(
   kernel_context
   SRCS kernel_context.cc
-  DEPS phi_enforce phi_context)
+  DEPS phi_enforce phi_backends)
 
 cc_library(
   ddim
@@ -86,38 +86,20 @@ if(WITH_GPU)
   nv_library(
     phi_tensor_utils
     SRCS tensor_utils.cc
-    DEPS cpu_context
-         gpu_context
-         dense_tensor
-         selected_rows
-         malloc
-         memcpy
-         device_context)
+    DEPS phi_backends dense_tensor selected_rows malloc memcpy device_context)
 elseif(WITH_ROCM)
   hip_library(
     phi_tensor_utils
     SRCS tensor_utils.cc
-    DEPS cpu_context
-         gpu_context
-         dense_tensor
-         selected_rows
-         malloc
-         memcpy
-         device_context)
+    DEPS phi_backends dense_tensor selected_rows malloc memcpy device_context)
 elseif(WITH_XPU_KP)
   xpu_library(
     phi_tensor_utils
     SRCS tensor_utils.cc
-    DEPS cpu_context
-         xpu_context
-         dense_tensor
-         selected_rows
-         malloc
-         memcpy
-         device_context)
+    DEPS phi_backends dense_tensor selected_rows malloc memcpy device_context)
 else()
   cc_library(
     phi_tensor_utils
     SRCS tensor_utils.cc
-    DEPS cpu_context dense_tensor selected_rows malloc memcpy device_context)
+    DEPS dense_tensor selected_rows malloc memcpy device_context phi_backends)
 endif()

paddle/phi/core/compat/CMakeLists.txt

@@ -7,18 +7,8 @@ cc_library(
   SRCS op_utils.cc
   DEPS arg_map_context enforce)
 
-set(convert_utils_deps data_type place op_utils)
+set(convert_utils_deps data_type place op_utils phi_backends)
 
-if(WITH_GPU)
-  set(convert_utils_deps ${convert_utils_deps} phi_gpu_info)
-elseif(WITH_ROCM)
-  set(convert_utils_deps ${convert_utils_deps} phi_gpu_info)
-elseif(WITH_XPU)
-  set(convert_utils_deps ${convert_utils_deps} phi_xpu_info)
-endif()
-if(WITH_CUSTOM_DEVICE)
-  set(convert_utils_deps ${convert_utils_deps} device_manager)
-endif()
 cc_library(
   convert_utils
   SRCS convert_utils.cc

paddle/phi/kernels/CMakeLists.txt

@@ -157,7 +157,7 @@ endif()
 if(WITH_MKLDNN)
   add_library(phi_onednn ${kernel_onednn})
   kernel_declare(${kernel_onednn})
-  set(COMMON_KERNEL_DEPS ${COMMON_KERNEL_DEPS} onednn_context)
+  set(COMMON_KERNEL_DEPS ${COMMON_KERNEL_DEPS} phi_backends)
   target_link_libraries(phi_onednn ${COMMON_KERNEL_DEPS})
   set(ADD_PHI_KERNELS ${ADD_PHI_KERNELS} phi_onednn)
 endif()