fix rpc compile bug (#47026)

CMakeLists.txt

@@ -303,6 +303,7 @@ option(WITH_RECORD_BUILDTIME
 option(WITH_CUSTOM_DEVICE "Compile with custom device support" OFF)
 option(WITH_ARM_BRPC "Supprot Brpc in Arm" OFF)
 option(WITH_FLPS "FL PS mode" OFF)
+option(WITH_RPC "Compile with rpc support" ${WITH_DISTRIBUTE})
 
 if(WITH_RECORD_BUILDTIME)
   set_property(
@@ -511,6 +512,54 @@ if(WITH_DISTRIBUTE)
   endif()
 endif()
 
+if(WITH_RPC)
+  if(NOT LINUX)
+    message(
+      WARNING "Disable WITH_RPC when not compiled on Linux. Force WITH_RPC=OFF."
+    )
+    set(WITH_RPC
+        OFF
+        CACHE BOOL "Disable WITH_RPC when not compiled on Linux" FORCE)
+  endif()
+  if(NOT WITH_DISTRIBUTE AND WITH_RPC)
+    message(
+      WARNING
+        "Disable WITH_RPC when not compiled with distribute. Force WITH_RPC=OFF."
+    )
+    set(WITH_RPC
+        OFF
+        CACHE BOOL "Disable WITH_RPC when not compiled with distribute" FORCE)
+  endif()
+  if(WITH_ASCEND_CL AND WITH_RPC)
+    message(
+      WARNING "Disable WITH_RPC when compiling with NPU. Force WITH_RPC=OFF.")
+    set(WITH_RPC
+        OFF
+        CACHE BOOL "Disable WITH_RPC when compiling with NPU" FORCE)
+  endif()
+  if(WITH_ROCM AND WITH_RPC)
+    message(
+      WARNING "Disable WITH_RPC when compiling with ROCM. Force WITH_RPC=OFF.")
+    set(WITH_RPC
+        OFF
+        CACHE BOOL "Disable WITH_RPC when compiling with ROCM" FORCE)
+  endif()
+  if(WITH_XPU AND WITH_RPC)
+    message(
+      WARNING "Disable WITH_RPC when compiling with XPU. Force WITH_RPC=OFF.")
+    set(WITH_RPC
+        OFF
+        CACHE BOOL "Disable WITH_RPC when compiling with XPU" FORCE)
+  endif()
+  if(WITH_CINN AND WITH_RPC)
+    message(
+      WARNING "Disable WITH_RPC when compiling with CINN. Force WITH_RPC=OFF.")
+    set(WITH_RPC
+        OFF
+        CACHE BOOL "Disable WITH_RPC when compiling with CINN" FORCE)
+  endif()
+endif()
+
 if(WITH_MPI)
   include(mpi)
 endif()

cmake/configure.cmake

@@ -222,6 +222,10 @@ if(WITH_PSCORE)
   add_definitions(-DPADDLE_WITH_PSCORE)
 endif()
 
+if(WITH_RPC)
+  add_definitions(-DPADDLE_WITH_RPC)
+endif()
+
 if(WITH_HETERPS)
   add_definitions(-DPADDLE_WITH_HETERPS)
 endif()

cmake/third_party.cmake

@@ -424,6 +424,19 @@ if(WITH_PSCORE)
   list(APPEND third_party_deps extern_rocksdb)
 endif()
 
+if(WITH_RPC
+   AND NOT WITH_PSCORE
+   AND NOT WITH_PSLIB)
+  include(external/snappy)
+  list(APPEND third_party_deps extern_snappy)
+
+  include(external/leveldb)
+  list(APPEND third_party_deps extern_leveldb)
+
+  include(external/brpc)
+  list(APPEND third_party_deps extern_brpc)
+endif()
+
 if(WITH_XBYAK)
   include(external/xbyak) # download, build, install xbyak
   list(APPEND third_party_deps extern_xbyak)

paddle/fluid/distributed/CMakeLists.txt

@@ -28,6 +28,10 @@ if(WITH_PYTHON)
   endif()
 endif()
 
+if(WITH_RPC)
+  add_subdirectory(rpc)
+endif()
+
 if(NOT WITH_PSCORE)
   add_subdirectory(fleet_executor)
   return()
@@ -42,9 +46,6 @@ set(DISTRIBUTE_COMPILE_FLAGS
 if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
   set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
 endif()
-if(LINUX)
-  add_subdirectory(rpc)
-endif()
 add_subdirectory(common)
 add_subdirectory(ps)
 add_subdirectory(test)

paddle/fluid/distributed/rpc/CMakeLists.txt

 set(PADDLE_RPC_SRCS python_rpc_handler.cc rpc_agent.cc)
+set(DISTRIBUTE_COMPILE_FLAGS
+    "-Wno-error=unused-value -Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result"
+)
+
+if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
+  set(DISTRIBUTE_COMPILE_FLAGS "${DISTRIBUTE_COMPILE_FLAGS} -faligned-new")
+endif()
 
 set_source_files_properties(
   python_rpc_handler.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
 set_source_files_properties(rpc_agent.cc PROPERTIES COMPILE_FLAGS
                                                     ${DISTRIBUTE_COMPILE_FLAGS})
 
-set(PADDLE_RPC_DEPS brpc protobuf glog pybind)
+set(PADDLE_RPC_DEPS
+    brpc
+    ssl
+    crypto
+    protobuf
+    zlib
+    leveldb
+    snappy
+    gflags
+    glog
+    pybind)
 proto_library(paddle_rpc_proto SRCS rpc.proto)
 cc_library(
   paddle_rpc
   SRCS ${PADDLE_RPC_SRCS}
   DEPS ${PADDLE_RPC_DEPS} paddle_rpc_proto)
+add_dependencies(paddle_rpc brpc)

paddle/fluid/pybind/CMakeLists.txt

@@ -49,12 +49,7 @@ if(WITH_PSCORE)
     set(PYBIND_DEPS ${PYBIND_DEPS} graph_gpu_wrapper)
   endif()
 endif()
-if(WITH_DISTRIBUTE
-   AND LINUX
-   AND NOT WITH_ASCEND_CL
-   AND NOT WITH_XPU
-   AND NOT WITH_CINN
-   AND NOT WITH_ROCM)
+if(WITH_RPC)
   set(PYBIND_DEPS ${PYBIND_DEPS} paddle_rpc)
 endif()
 if(WITH_GPU OR WITH_ROCM)
@@ -226,12 +221,7 @@ if(WITH_PSCORE)
   set(PYBIND_SRCS fleet_py.cc ${PYBIND_SRCS})
 endif()
 
-if(WITH_DISTRIBUTE
-   AND LINUX
-   AND NOT WITH_ASCEND_CL
-   AND NOT WITH_XPU
-   AND NOT WITH_CINN
-   AND NOT WITH_ROCM)
+if(WITH_RPC)
   if(WITH_ARM_BRPC)
     set(DISTRIBUTE_COMPILE_FLAGS
         "-faligned-new -Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result"

paddle/fluid/pybind/pybind.cc

@@ -182,9 +182,7 @@ limitations under the License. */
 #include "paddle/fluid/framework/paddle2cinn/cinn_compiler.h"
 #endif
 
-#if defined(__linux__) && !defined(PADDLE_WITH_XPU) &&               \
-    !defined(PADDLE_WITH_ASCEND_CL) && !defined(PADDLE_WITH_CINN) && \
-    !defined(PADDLE_WITH_HIP)
+#if defined(PADDLE_WITH_RPC)
 #include "paddle/fluid/pybind/rpc.h"
 #endif
 
@@ -2610,9 +2608,7 @@ All parameter, weight, gradient are variables in Paddle.
   BindGraphGpuWrapper(&m);
 #endif
 #endif
-#if defined(__linux__) && !defined(PADDLE_WITH_XPU) &&               \
-    !defined(PADDLE_WITH_ASCEND_CL) && !defined(PADDLE_WITH_CINN) && \
-    !defined(PADDLE_WITH_HIP)
+#if defined(PADDLE_WITH_RPC)
   BindWorkerInfo(&m);
   BindFuture(&m);
   InitAndSetAgentInstance(&m);

python/paddle/fluid/tests/unittests/CMakeLists.txt

@@ -604,12 +604,16 @@ set_tests_properties(test_conv2d_api PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
 set_tests_properties(test_conv_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
 set_tests_properties(test_norm_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
 set_tests_properties(test_nn_grad PROPERTIES LABELS "RUN_TYPE=EXCLUSIVE")
+
+if(WITH_RPC)
+  add_subdirectory(rpc)
+endif()
+
 if(WITH_DISTRIBUTE)
   add_subdirectory(distributed_passes)
   add_subdirectory(ps)
   add_subdirectory(auto_parallel)
   add_subdirectory(collective)
-  add_subdirectory(rpc)
 
   # FIXME(typhoonzero): add these tests back
   list(REMOVE_ITEM DIST_TEST_OPS "test_dist_transformer")