From e89d72978358373eb677256131f18d610c8606c2 Mon Sep 17 00:00:00 2001 From: Xinger <34963409+Ningsir@users.noreply.github.com> Date: Wed, 19 Oct 2022 15:55:13 +0800 Subject: [PATCH] fix rpc compile bug (#47026) --- CMakeLists.txt | 49 +++++++++++++++++++ cmake/configure.cmake | 4 ++ cmake/third_party.cmake | 13 +++++ paddle/fluid/distributed/CMakeLists.txt | 7 +-- paddle/fluid/distributed/rpc/CMakeLists.txt | 20 +++++++- paddle/fluid/pybind/CMakeLists.txt | 14 +----- paddle/fluid/pybind/pybind.cc | 8 +-- .../fluid/tests/unittests/CMakeLists.txt | 6 ++- 8 files changed, 98 insertions(+), 23 deletions(-) mode change 100755 => 100644 cmake/configure.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 3cb05a98da..e125063f15 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -303,6 +303,7 @@ option(WITH_RECORD_BUILDTIME option(WITH_CUSTOM_DEVICE "Compile with custom device support" OFF) option(WITH_ARM_BRPC "Supprot Brpc in Arm" OFF) option(WITH_FLPS "FL PS mode" OFF) +option(WITH_RPC "Compile with rpc support" ${WITH_DISTRIBUTE}) if(WITH_RECORD_BUILDTIME) set_property( @@ -511,6 +512,54 @@ if(WITH_DISTRIBUTE) endif() endif() +if(WITH_RPC) + if(NOT LINUX) + message( + WARNING "Disable WITH_RPC when not compiled on Linux. Force WITH_RPC=OFF." + ) + set(WITH_RPC + OFF + CACHE BOOL "Disable WITH_RPC when not compiled on Linux" FORCE) + endif() + if(NOT WITH_DISTRIBUTE AND WITH_RPC) + message( + WARNING + "Disable WITH_RPC when not compiled with distribute. Force WITH_RPC=OFF." + ) + set(WITH_RPC + OFF + CACHE BOOL "Disable WITH_RPC when not compiled with distribute" FORCE) + endif() + if(WITH_ASCEND_CL AND WITH_RPC) + message( + WARNING "Disable WITH_RPC when compiling with NPU. Force WITH_RPC=OFF.") + set(WITH_RPC + OFF + CACHE BOOL "Disable WITH_RPC when compiling with NPU" FORCE) + endif() + if(WITH_ROCM AND WITH_RPC) + message( + WARNING "Disable WITH_RPC when compiling with ROCM. Force WITH_RPC=OFF.") + set(WITH_RPC + OFF + CACHE BOOL "Disable WITH_RPC when compiling with ROCM" FORCE) + endif() + if(WITH_XPU AND WITH_RPC) + message( + WARNING "Disable WITH_RPC when compiling with XPU. Force WITH_RPC=OFF.") + set(WITH_RPC + OFF + CACHE BOOL "Disable WITH_RPC when compiling with XPU" FORCE) + endif() + if(WITH_CINN AND WITH_RPC) + message( + WARNING "Disable WITH_RPC when compiling with CINN. Force WITH_RPC=OFF.") + set(WITH_RPC + OFF + CACHE BOOL "Disable WITH_RPC when compiling with CINN" FORCE) + endif() +endif() + if(WITH_MPI) include(mpi) endif() diff --git a/cmake/configure.cmake b/cmake/configure.cmake old mode 100755 new mode 100644 index 4bc00593c8..c49a879fa0 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -222,6 +222,10 @@ if(WITH_PSCORE) add_definitions(-DPADDLE_WITH_PSCORE) endif() +if(WITH_RPC) + add_definitions(-DPADDLE_WITH_RPC) +endif() + if(WITH_HETERPS) add_definitions(-DPADDLE_WITH_HETERPS) endif() diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake index bfba3dfbac..06ca0d16df 100755 --- a/cmake/third_party.cmake +++ b/cmake/third_party.cmake @@ -424,6 +424,19 @@ if(WITH_PSCORE) list(APPEND third_party_deps extern_rocksdb) endif() +if(WITH_RPC + AND NOT WITH_PSCORE + AND NOT WITH_PSLIB) + include(external/snappy) + list(APPEND third_party_deps extern_snappy) + + include(external/leveldb) + list(APPEND third_party_deps extern_leveldb) + + include(external/brpc) + list(APPEND third_party_deps extern_brpc) +endif() + if(WITH_XBYAK) include(external/xbyak) # download, build, install xbyak list(APPEND third_party_deps extern_xbyak) diff --git a/paddle/fluid/distributed/CMakeLists.txt b/paddle/fluid/distributed/CMakeLists.txt index ef76aa3960..ac4747adb8 100755 --- a/paddle/fluid/distributed/CMakeLists.txt +++ b/paddle/fluid/distributed/CMakeLists.txt @@ -28,6 +28,10 @@ if(WITH_PYTHON) endif() endif() +if(WITH_RPC) + add_subdirectory(rpc) +endif() + if(NOT WITH_PSCORE) add_subdirectory(fleet_executor) return() @@ -42,9 +46,6 @@ set(DISTRIBUTE_COMPILE_FLAGS if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new") endif() -if(LINUX) - add_subdirectory(rpc) -endif() add_subdirectory(common) add_subdirectory(ps) add_subdirectory(test) diff --git a/paddle/fluid/distributed/rpc/CMakeLists.txt b/paddle/fluid/distributed/rpc/CMakeLists.txt index 655a28d4f7..dae4efd6e8 100644 --- a/paddle/fluid/distributed/rpc/CMakeLists.txt +++ b/paddle/fluid/distributed/rpc/CMakeLists.txt @@ -1,13 +1,31 @@ set(PADDLE_RPC_SRCS python_rpc_handler.cc rpc_agent.cc) +set(DISTRIBUTE_COMPILE_FLAGS + "-Wno-error=unused-value -Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result" +) + +if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) + set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new") +endif() set_source_files_properties( python_rpc_handler.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) set_source_files_properties(rpc_agent.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -set(PADDLE_RPC_DEPS brpc protobuf glog pybind) +set(PADDLE_RPC_DEPS + brpc + ssl + crypto + protobuf + zlib + leveldb + snappy + gflags + glog + pybind) proto_library(paddle_rpc_proto SRCS rpc.proto) cc_library( paddle_rpc SRCS ${PADDLE_RPC_SRCS} DEPS ${PADDLE_RPC_DEPS} paddle_rpc_proto) +add_dependencies(paddle_rpc brpc) diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index 03a7b97af4..ee48cb31ca 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -49,12 +49,7 @@ if(WITH_PSCORE) set(PYBIND_DEPS ${PYBIND_DEPS} graph_gpu_wrapper) endif() endif() -if(WITH_DISTRIBUTE - AND LINUX - AND NOT WITH_ASCEND_CL - AND NOT WITH_XPU - AND NOT WITH_CINN - AND NOT WITH_ROCM) +if(WITH_RPC) set(PYBIND_DEPS ${PYBIND_DEPS} paddle_rpc) endif() if(WITH_GPU OR WITH_ROCM) @@ -226,12 +221,7 @@ if(WITH_PSCORE) set(PYBIND_SRCS fleet_py.cc ${PYBIND_SRCS}) endif() -if(WITH_DISTRIBUTE - AND LINUX - AND NOT WITH_ASCEND_CL - AND NOT WITH_XPU - AND NOT WITH_CINN - AND NOT WITH_ROCM) +if(WITH_RPC) if(WITH_ARM_BRPC) set(DISTRIBUTE_COMPILE_FLAGS "-faligned-new -Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result" diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index 71f1cf8184..c37839a7c6 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -182,9 +182,7 @@ limitations under the License. */ #include "paddle/fluid/framework/paddle2cinn/cinn_compiler.h" #endif -#if defined(__linux__) && !defined(PADDLE_WITH_XPU) && \ - !defined(PADDLE_WITH_ASCEND_CL) && !defined(PADDLE_WITH_CINN) && \ - !defined(PADDLE_WITH_HIP) +#if defined(PADDLE_WITH_RPC) #include "paddle/fluid/pybind/rpc.h" #endif @@ -2610,9 +2608,7 @@ All parameter, weight, gradient are variables in Paddle. BindGraphGpuWrapper(&m); #endif #endif -#if defined(__linux__) && !defined(PADDLE_WITH_XPU) && \ - !defined(PADDLE_WITH_ASCEND_CL) && !defined(PADDLE_WITH_CINN) && \ - !defined(PADDLE_WITH_HIP) +#if defined(PADDLE_WITH_RPC) BindWorkerInfo(&m); BindFuture(&m); InitAndSetAgentInstance(&m); diff --git a/python/paddle/fluid/tests/unittests/CMakeLists.txt b/python/paddle/fluid/tests/unittests/CMakeLists.txt index be3a534208..a206e8994e 100755 --- a/python/paddle/fluid/tests/unittests/CMakeLists.txt +++ b/python/paddle/fluid/tests/unittests/CMakeLists.txt @@ -604,12 +604,16 @@ set_tests_properties(test_conv2d_api PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE") set_tests_properties(test_conv_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE") set_tests_properties(test_norm_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE") set_tests_properties(test_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE") + +if(WITH_RPC) + add_subdirectory(rpc) +endif() + if(WITH_DISTRIBUTE) add_subdirectory(distributed_passes) add_subdirectory(ps) add_subdirectory(auto_parallel) add_subdirectory(collective) - add_subdirectory(rpc) # FIXME(typhoonzero): add these tests back list(REMOVE_ITEM DIST_TEST_OPS "test_dist_transformer") -- GitLab