From 1047cb1726742656b1f7c32393340ac7e75aee33 Mon Sep 17 00:00:00 2001 From: Leo Chen Date: Tue, 19 Jul 2022 09:14:11 -0500 Subject: [PATCH] compile phi/backends into one static library (#44373) * compile into one static library * fix xpu compile * fix xpu compile * fix inference compile * fix inference compile * add custom test * revert one file --- paddle/fluid/imperative/CMakeLists.txt | 2 +- paddle/fluid/memory/allocation/CMakeLists.txt | 3 +- paddle/fluid/platform/CMakeLists.txt | 24 ++---- .../fluid/platform/device/gpu/CMakeLists.txt | 4 +- .../fluid/platform/device/xpu/CMakeLists.txt | 2 +- paddle/phi/CMakeLists.txt | 2 +- paddle/phi/api/lib/CMakeLists.txt | 4 +- paddle/phi/backends/CMakeLists.txt | 85 +++++++++---------- paddle/phi/backends/cpu/CMakeLists.txt | 12 --- paddle/phi/backends/gpu/CMakeLists.txt | 22 ----- paddle/phi/backends/gpu/cuda/CMakeLists.txt | 4 - paddle/phi/backends/gpu/rocm/CMakeLists.txt | 4 - paddle/phi/backends/onednn/CMakeLists.txt | 6 -- paddle/phi/backends/xpu/CMakeLists.txt | 8 -- paddle/phi/capi/lib/CMakeLists.txt | 2 +- paddle/phi/common/CMakeLists.txt | 4 +- paddle/phi/core/CMakeLists.txt | 28 ++---- paddle/phi/core/compat/CMakeLists.txt | 12 +-- paddle/phi/kernels/CMakeLists.txt | 2 +- 19 files changed, 66 insertions(+), 164 deletions(-) delete mode 100644 paddle/phi/backends/cpu/CMakeLists.txt delete mode 100644 paddle/phi/backends/gpu/CMakeLists.txt delete mode 100644 paddle/phi/backends/gpu/cuda/CMakeLists.txt delete mode 100644 paddle/phi/backends/gpu/rocm/CMakeLists.txt delete mode 100644 paddle/phi/backends/onednn/CMakeLists.txt delete mode 100644 paddle/phi/backends/xpu/CMakeLists.txt diff --git a/paddle/fluid/imperative/CMakeLists.txt b/paddle/fluid/imperative/CMakeLists.txt index 98ece2db96..16aa364736 100644 --- a/paddle/fluid/imperative/CMakeLists.txt +++ b/paddle/fluid/imperative/CMakeLists.txt @@ -52,7 +52,7 @@ if(WITH_GPU) cc_library( layout_autotune SRCS layout_autotune.cc - DEPS op_info phi_gpu_info) + DEPS op_info phi_backends) else() cc_library( layout_autotune diff --git a/paddle/fluid/memory/allocation/CMakeLists.txt b/paddle/fluid/memory/allocation/CMakeLists.txt index ec8391469f..042efa3349 100644 --- a/paddle/fluid/memory/allocation/CMakeLists.txt +++ b/paddle/fluid/memory/allocation/CMakeLists.txt @@ -1,6 +1,6 @@ include(ExternalProject) -set(ALLOCATOR_DEPS place stats profiler) +set(ALLOCATOR_DEPS place stats profiler phi_backends) set(ALLOCATOR_SRCS allocator.cc cpu_allocator.cc @@ -54,7 +54,6 @@ endif() if(WITH_CUSTOM_DEVICE) list(APPEND ALLOCATOR_SRCS custom_allocator.cc) - list(APPEND ALLOCATOR_DEPS device_manager) endif() if(WITH_XPU) diff --git a/paddle/fluid/platform/CMakeLists.txt b/paddle/fluid/platform/CMakeLists.txt index e872fb1625..5090e0d891 100644 --- a/paddle/fluid/platform/CMakeLists.txt +++ b/paddle/fluid/platform/CMakeLists.txt @@ -220,16 +220,9 @@ cc_library( ${XPU_CTX_DEPS} ${MLU_CTX_DEPS} eigen3 + phi_backends + phi_device_context generator) -if(WITH_XPU) - target_link_libraries(device_context xpu_context xpu_resource_pool) -endif() - -if(WITH_MKLDNN) - target_link_libraries(device_context onednn_context) -endif() - -target_link_libraries(device_context cpu_context) cc_library( collective_helper @@ -244,22 +237,19 @@ if(WITH_CNCL) endif() if(WITH_GPU OR WITH_ROCM) - target_link_libraries(device_context gpu_info gpu_context phi_gpu_info) target_link_libraries(device_context gpu_resource_pool) endif() -if(WITH_CUSTOM_DEVICE) - target_link_libraries(device_context custom_context) -endif() + if(WITH_ASCEND_CL) target_link_libraries(device_context npu_resource_pool) endif() -if(WITH_MLU) - target_link_libraries(device_context mlu_resource_pool) +if(WITH_XPU) + target_link_libraries(device_context xpu_resource_pool) endif() -if(WITH_CUSTOM_DEVICE) - target_link_libraries(device_context custom_context) +if(WITH_MLU) + target_link_libraries(device_context mlu_resource_pool) endif() cc_test( diff --git a/paddle/fluid/platform/device/gpu/CMakeLists.txt b/paddle/fluid/platform/device/gpu/CMakeLists.txt index 66120f55f7..dd2a7d1eb8 100644 --- a/paddle/fluid/platform/device/gpu/CMakeLists.txt +++ b/paddle/fluid/platform/device/gpu/CMakeLists.txt @@ -3,7 +3,7 @@ if(WITH_GPU) nv_library( gpu_info SRCS gpu_info.cc - DEPS phi_gpu_info gflags glog enforce monitor dynload_cuda) + DEPS phi_backends gflags glog enforce monitor dynload_cuda) nv_test(cuda_helper_test SRCS cuda_helper_test.cu) nv_test( @@ -15,7 +15,7 @@ elseif(WITH_ROCM) hip_library( gpu_info SRCS gpu_info.cc - DEPS phi_gpu_info gflags glog enforce monitor dynload_cuda) + DEPS phi_backends gflags glog enforce monitor dynload_cuda) hip_test(cuda_helper_test SRCS cuda_helper_test.cu) hip_test( diff --git a/paddle/fluid/platform/device/xpu/CMakeLists.txt b/paddle/fluid/platform/device/xpu/CMakeLists.txt index 19656bf1cc..b7134070be 100644 --- a/paddle/fluid/platform/device/xpu/CMakeLists.txt +++ b/paddle/fluid/platform/device/xpu/CMakeLists.txt @@ -20,7 +20,7 @@ cc_library( xpulib device_context place - phi_xpu_info) + phi_backends) cc_library( xpu_op_list SRCS xpu_op_list.cc diff --git a/paddle/phi/CMakeLists.txt b/paddle/phi/CMakeLists.txt index 9715fd7704..438b85cafe 100644 --- a/paddle/phi/CMakeLists.txt +++ b/paddle/phi/CMakeLists.txt @@ -30,7 +30,7 @@ endif() set(PHI_DEPS convert_utils dense_tensor - phi_context + phi_backends kernel_factory kernel_context arg_map_context diff --git a/paddle/phi/api/lib/CMakeLists.txt b/paddle/phi/api/lib/CMakeLists.txt index 5a5aa9638a..05d27571b8 100644 --- a/paddle/phi/api/lib/CMakeLists.txt +++ b/paddle/phi/api/lib/CMakeLists.txt @@ -351,12 +351,12 @@ cc_library( cc_library( context_pool SRCS context_pool.cc - DEPS phi_context phi_enforce place init) + DEPS phi_backends phi_enforce place init) cc_library( kernel_dispatch SRCS kernel_dispatch.cc - DEPS phi_tensor_raw phi_context kernel_factory context_pool) + DEPS phi_tensor_raw phi_backends kernel_factory context_pool) cc_library( api_gen_utils SRCS api_gen_utils.cc diff --git a/paddle/phi/backends/CMakeLists.txt b/paddle/phi/backends/CMakeLists.txt index 50367a32b0..a2d4b1deef 100644 --- a/paddle/phi/backends/CMakeLists.txt +++ b/paddle/phi/backends/CMakeLists.txt @@ -1,61 +1,58 @@ add_subdirectory(dynload) -add_subdirectory(cpu) - -add_subdirectory(custom) +set(BACKENDS_SRCS all_context.cc cpu/cpu_context.cc) +set(BACKENDS_DEPS enforce place flags) if(WITH_GPU OR WITH_ROCM) - add_subdirectory(gpu) + list(APPEND BACKENDS_SRCS gpu/gpu_context.cc gpu/gpu_info.cc + gpu/gpu_resources.cc) + if(WITH_GPU) + list(APPEND BACKENDS_SRCS gpu/cuda/cuda_info.cc) + endif() + if(WITH_ROCM) + list(APPEND BACKENDS_SRCS gpu/rocm/rocm_info.cc) + endif() + list(APPEND BACKENDS_DEPS phi_dynload_cuda) endif() if(WITH_XPU) - add_subdirectory(xpu) + list(APPEND BACKENDS_SRCS xpu/xpu_context.cc xpu/xpu_info.cc) endif() if(WITH_MKLDNN) - add_subdirectory(onednn) + list(APPEND BACKENDS_SRCS onednn/onednn_context.cc) + list(APPEND BACKENDS_DEPS mkldnn) endif() -cc_library( - phi_context - SRCS all_context.cc - DEPS device_context cpu_context) - -if(WITH_XPU) - add_dependencies(phi_context xpu_context) +if(WITH_CUSTOM_DEVICE) + list( + APPEND + BACKENDS_SRCS + callback_manager.cc + device_guard.cc + stream.cc + event.cc + device_base.cc + device_manager.cc + custom/custom_context.cc + custom/custom_device.cc) endif() -if(WITH_GPU) - add_dependencies(phi_context gpu_context) -endif() +add_library(phi_backends "${BACKENDS_SRCS}") +target_link_libraries(phi_backends ${BACKENDS_DEPS}) + +# for inference library +get_property(phi_modules GLOBAL PROPERTY PHI_MODULES) +set(phi_modules ${phi_modules} phi_backends) +set_property(GLOBAL PROPERTY PHI_MODULES "${phi_modules}") if(WITH_CUSTOM_DEVICE) - add_dependencies(phi_context custom_context) - cc_library( - callback_manager - SRCS callback_manager.cc - DEPS enforce place) - cc_library( - device_guard - SRCS device_guard.cc - DEPS enforce place) - cc_library( - stream - SRCS stream.cc - DEPS callback_manager) - cc_library( - event - SRCS event.cc - DEPS enforce place) - cc_library( - device_base - SRCS device_base.cc - DEPS stream event callback_manager device_guard device_context flags) - cc_library( - device_manager - SRCS device_manager.cc - DEPS custom_device) - set(GLOB_DEV_LIB - device_manager custom_device - CACHE INTERNAL "Global DEV library") + cc_test( + custom_device_test + SRCS custom/custom_device_test.cc + DEPS phi_backends phi_device_context) + cc_test( + capi_test + SRCS custom/capi_test.cc + DEPS phi_capi) endif() diff --git a/paddle/phi/backends/cpu/CMakeLists.txt b/paddle/phi/backends/cpu/CMakeLists.txt deleted file mode 100644 index e32aa17758..0000000000 --- a/paddle/phi/backends/cpu/CMakeLists.txt +++ /dev/null @@ -1,12 +0,0 @@ -if(WITH_MKLDNN) - # TODO(wilber): support mkldnn context. - cc_library( - cpu_context - SRCS cpu_context.cc - DEPS phi_device_context mkldnn eigen3) -else() - cc_library( - cpu_context - SRCS cpu_context.cc - DEPS phi_device_context eigen3) -endif() diff --git a/paddle/phi/backends/gpu/CMakeLists.txt b/paddle/phi/backends/gpu/CMakeLists.txt deleted file mode 100644 index 6d9f2de67d..0000000000 --- a/paddle/phi/backends/gpu/CMakeLists.txt +++ /dev/null @@ -1,22 +0,0 @@ -if(WITH_GPU) - add_subdirectory(cuda) - nv_library( - phi_gpu_info - SRCS gpu_info.cc - DEPS phi_cuda_info gflags glog enforce phi_dynload_cuda) -elseif(WITH_ROCM) - add_subdirectory(rocm) - hip_library( - phi_gpu_info - SRCS gpu_info.cc - DEPS phi_rocm_info gflags glog enforce phi_dynload_cuda) -endif() - -cc_library( - gpu_resources - SRCS gpu_resources.cc - DEPS phi_device_context phi_gpu_info) -cc_library( - gpu_context - SRCS gpu_context.cc - DEPS phi_device_context phi_gpu_info eigen3 gpu_resources) diff --git a/paddle/phi/backends/gpu/cuda/CMakeLists.txt b/paddle/phi/backends/gpu/cuda/CMakeLists.txt deleted file mode 100644 index 9765f5dc03..0000000000 --- a/paddle/phi/backends/gpu/cuda/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -nv_library( - phi_cuda_info - SRCS cuda_info.cc - DEPS gflags glog enforce phi_dynload_cuda) diff --git a/paddle/phi/backends/gpu/rocm/CMakeLists.txt b/paddle/phi/backends/gpu/rocm/CMakeLists.txt deleted file mode 100644 index 730aad5d2f..0000000000 --- a/paddle/phi/backends/gpu/rocm/CMakeLists.txt +++ /dev/null @@ -1,4 +0,0 @@ -hip_library( - phi_rocm_info - SRCS rocm_info.cc - DEPS gflags glog enforce phi_dynload_cuda) diff --git a/paddle/phi/backends/onednn/CMakeLists.txt b/paddle/phi/backends/onednn/CMakeLists.txt deleted file mode 100644 index a65d6b002f..0000000000 --- a/paddle/phi/backends/onednn/CMakeLists.txt +++ /dev/null @@ -1,6 +0,0 @@ -if(WITH_MKLDNN) - cc_library( - onednn_context - SRCS onednn_context.cc - DEPS cpu_context mkldnn) -endif() diff --git a/paddle/phi/backends/xpu/CMakeLists.txt b/paddle/phi/backends/xpu/CMakeLists.txt deleted file mode 100644 index 861b57956b..0000000000 --- a/paddle/phi/backends/xpu/CMakeLists.txt +++ /dev/null @@ -1,8 +0,0 @@ -cc_library( - phi_xpu_info - SRCS xpu_info.cc - DEPS enforce xpulib phi_place) -cc_library( - xpu_context - SRCS xpu_context.cc - DEPS phi_device_context phi_xpu_info) diff --git a/paddle/phi/capi/lib/CMakeLists.txt b/paddle/phi/capi/lib/CMakeLists.txt index de335bb668..60afb74a6d 100644 --- a/paddle/phi/capi/lib/CMakeLists.txt +++ b/paddle/phi/capi/lib/CMakeLists.txt @@ -6,7 +6,7 @@ cc_library( cc_library( phi_c_device_context SRCS c_device_context.cc - DEPS phi_context) + DEPS phi_backends) cc_library( phi_c_int_array diff --git a/paddle/phi/common/CMakeLists.txt b/paddle/phi/common/CMakeLists.txt index b32e0f4bef..d96cb3e895 100644 --- a/paddle/phi/common/CMakeLists.txt +++ b/paddle/phi/common/CMakeLists.txt @@ -2,12 +2,12 @@ if(WITH_GPU) nv_library( phi_place SRCS place.cc - DEPS phi_gpu_info) + DEPS phi_backends) elseif(WITH_ROCM) hip_library( phi_place SRCS place.cc - DEPS phi_gpu_info) + DEPS phi_backends) else() cc_library(phi_place SRCS place.cc) endif() diff --git a/paddle/phi/core/CMakeLists.txt b/paddle/phi/core/CMakeLists.txt index d7ffa1b82f..b119dd3c53 100644 --- a/paddle/phi/core/CMakeLists.txt +++ b/paddle/phi/core/CMakeLists.txt @@ -18,7 +18,7 @@ cc_library( cc_library( kernel_context SRCS kernel_context.cc - DEPS phi_enforce phi_context) + DEPS phi_enforce phi_backends) cc_library( ddim @@ -86,38 +86,20 @@ if(WITH_GPU) nv_library( phi_tensor_utils SRCS tensor_utils.cc - DEPS cpu_context - gpu_context - dense_tensor - selected_rows - malloc - memcpy - device_context) + DEPS phi_backends dense_tensor selected_rows malloc memcpy device_context) elseif(WITH_ROCM) hip_library( phi_tensor_utils SRCS tensor_utils.cc - DEPS cpu_context - gpu_context - dense_tensor - selected_rows - malloc - memcpy - device_context) + DEPS phi_backends dense_tensor selected_rows malloc memcpy device_context) elseif(WITH_XPU_KP) xpu_library( phi_tensor_utils SRCS tensor_utils.cc - DEPS cpu_context - xpu_context - dense_tensor - selected_rows - malloc - memcpy - device_context) + DEPS phi_backends dense_tensor selected_rows malloc memcpy device_context) else() cc_library( phi_tensor_utils SRCS tensor_utils.cc - DEPS cpu_context dense_tensor selected_rows malloc memcpy device_context) + DEPS dense_tensor selected_rows malloc memcpy device_context phi_backends) endif() diff --git a/paddle/phi/core/compat/CMakeLists.txt b/paddle/phi/core/compat/CMakeLists.txt index 3fd9b74255..31b1636d5b 100644 --- a/paddle/phi/core/compat/CMakeLists.txt +++ b/paddle/phi/core/compat/CMakeLists.txt @@ -7,18 +7,8 @@ cc_library( SRCS op_utils.cc DEPS arg_map_context enforce) -set(convert_utils_deps data_type place op_utils) +set(convert_utils_deps data_type place op_utils phi_backends) -if(WITH_GPU) - set(convert_utils_deps ${convert_utils_deps} phi_gpu_info) -elseif(WITH_ROCM) - set(convert_utils_deps ${convert_utils_deps} phi_gpu_info) -elseif(WITH_XPU) - set(convert_utils_deps ${convert_utils_deps} phi_xpu_info) -endif() -if(WITH_CUSTOM_DEVICE) - set(convert_utils_deps ${convert_utils_deps} device_manager) -endif() cc_library( convert_utils SRCS convert_utils.cc diff --git a/paddle/phi/kernels/CMakeLists.txt b/paddle/phi/kernels/CMakeLists.txt index 05abcbd0d1..98ea91ce5a 100644 --- a/paddle/phi/kernels/CMakeLists.txt +++ b/paddle/phi/kernels/CMakeLists.txt @@ -157,7 +157,7 @@ endif() if(WITH_MKLDNN) add_library(phi_onednn ${kernel_onednn}) kernel_declare(${kernel_onednn}) - set(COMMON_KERNEL_DEPS ${COMMON_KERNEL_DEPS} onednn_context) + set(COMMON_KERNEL_DEPS ${COMMON_KERNEL_DEPS} phi_backends) target_link_libraries(phi_onednn ${COMMON_KERNEL_DEPS}) set(ADD_PHI_KERNELS ${ADD_PHI_KERNELS} phi_onednn) endif() -- GitLab