solve brpc compile in arm-ubantu18 (#41649)

* back fl

* delete ssl cert

* .

* make warning

* .

* unittest paral degree

* solve unittest

* heter & multi cloud commm ready

* .

* .

* arm_brpc compile

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* only output is ok

* base is ok

* .

* .

* .

* .

* .

* .

* .

* .

* add switch server bin

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* adapt brpc ssl

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

* .

CMakeLists.txt

@@ -246,6 +246,7 @@ option(WITH_ASCEND_INT64 "Compile with int64 kernel for ascend NPU"    OFF)
 option(WITH_POCKETFFT    "Compile with pocketfft support"      ON)
 option(WITH_RECORD_BUILDTIME    "Compile PaddlePaddle with record all targets build time"       OFF)
 option(WITH_CUSTOM_DEVICE "Compile with custom device support"    OFF)
+option(WITH_ARM_BRPC "Supprot Brpc in Arm"    OFF)
 
 if(WITH_RECORD_BUILDTIME)
     set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CMAKE_CURRENT_SOURCE_DIR}/tools/get_build_time.sh")

cmake/configure.cmake

@@ -74,6 +74,10 @@ if(WITH_PSLIB)
     add_definitions(-DPADDLE_WITH_PSLIB)
 endif()
 
+if(WITH_ARM_BRPC)
+    add_definitions(-DPADDLE_WITH_ARM_BRPC)
+endif()
+
 if(WITH_GLOO)
     add_definitions(-DPADDLE_WITH_GLOO)
 endif()

cmake/external/arm_brpc.cmake

+# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+INCLUDE(ExternalProject)
+
+#find_package(OpenSSL REQUIRED)
+
+#message(STATUS "ssl:" ${OPENSSL_SSL_LIBRARY})
+#message(STATUS "crypto:" ${OPENSSL_CRYPTO_LIBRARY})
+
+#ADD_LIBRARY(ssl SHARED IMPORTED GLOBAL)
+#SET_PROPERTY(TARGET ssl PROPERTY IMPORTED_LOCATION ${OPENSSL_SSL_LIBRARY})
+
+#ADD_LIBRARY(crypto SHARED IMPORTED GLOBAL)
+#SET_PROPERTY(TARGET crypto PROPERTY IMPORTED_LOCATION ${OPENSSL_CRYPTO_LIBRARY})
+
+IF((NOT DEFINED ARM_BRPC_NAME) OR (NOT DEFINED ARM_BRPC_URL))
+  SET(ARM_BRPC_VER "1.1.0" CACHE STRING "" FORCE)
+  SET(ARM_BRPC_NAME "arm_brpc" CACHE STRING "" FORCE)
+ENDIF()
+
+MESSAGE(STATUS "ARM_BRPC_NAME: ${ARM_BRPC_NAME}, ARM_BRPC_URL: ${ARM_BRPC_URL}")
+SET(ARM_BRPC_PREFIX_DIR    "${THIRD_PARTY_PATH}/arm_brpc")
+SET(ARM_BRPC_PROJECT       "extern_arm_brpc")
+SET(ARM_BRPC_DOWNLOAD_DIR  "${ARM_BRPC_PREFIX_DIR}/src/${ARM_BRPC_PROJECT}")
+SET(ARM_BRPC_DST_DIR       "output")
+SET(ARM_BRPC_INSTALL_ROOT  "${THIRD_PARTY_PATH}/install")
+SET(ARM_BRPC_INSTALL_DIR   ${ARM_BRPC_INSTALL_ROOT}/arm_brpc/output)
+SET(ARM_BRPC_ROOT          ${ARM_BRPC_INSTALL_DIR})
+SET(ARM_BRPC_INC_DIR       ${ARM_BRPC_ROOT}/include)
+SET(ARM_BRPC_LIB_DIR       ${ARM_BRPC_ROOT}/lib)
+SET(ARM_BRPC_LIB           ${ARM_BRPC_LIB_DIR}/libbrpc.a)
+SET(CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_RPATH}" "${ARM_BRPC_ROOT}/lib")
+
+INCLUDE_DIRECTORIES(${ARM_BRPC_INSTALL_ROOT}/${ARM_BRPC_NAME}/output/include)
+
+FILE(WRITE ${ARM_BRPC_DOWNLOAD_DIR}/CMakeLists.txt
+  "PROJECT(ARM_BRPC)\n"
+  "cmake_minimum_required(VERSION 3.0)\n"
+  "install(DIRECTORY ${ARM_BRPC_DST_DIR} ${ARM_BRPC_DST_DIR} \n"
+  "        DESTINATION ${ARM_BRPC_NAME})\n")
+  
+SET(ARM_BRPC_URL "https://paddlerec.bj.bcebos.com/online_infer/arm_brpc_ubuntu18/output.tar.gz" CACHE STRING "" FORCE)
+ExternalProject_Add(
+    ${ARM_BRPC_PROJECT}
+    ${EXTERNAL_PROJECT_LOG_ARGS}
+    PREFIX                ${ARM_BRPC_PREFIX_DIR}
+    DOWNLOAD_DIR          ${ARM_BRPC_DOWNLOAD_DIR}
+    DOWNLOAD_COMMAND      rm -rf output.tar.gz
+                          && wget --no-check-certificate ${ARM_BRPC_URL}
+                          && tar zxvf output.tar.gz
+    #DOWNLOAD_COMMAND      cp /home/wangbin44/Paddle/build/output.tar.gz . 
+    #                      && tar zxvf output.tar.gz
+    DOWNLOAD_NO_PROGRESS  1
+    UPDATE_COMMAND        ""
+    CMAKE_ARGS            -DCMAKE_INSTALL_PREFIX=${ARM_BRPC_INSTALL_ROOT}
+                          -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+    CMAKE_CACHE_ARGS      -DCMAKE_INSTALL_PREFIX:PATH=${ARM_BRPC_INSTALL_ROOT}
+                          -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE}
+    BUILD_BYPRODUCTS      ${ARM_BRPC_LIB}
+)
+
+ADD_LIBRARY(arm_brpc STATIC IMPORTED GLOBAL)  # 直接导入已经生成的库
+SET_PROPERTY(TARGET arm_brpc PROPERTY IMPORTED_LOCATION ${ARM_BRPC_LIB})
+ADD_DEPENDENCIES(arm_brpc ${ARM_BRPC_PROJECT})

cmake/external/brpc.cmake

@@ -41,6 +41,7 @@ ExternalProject_Add(
         ${EXTERNAL_PROJECT_LOG_ARGS}
         # TODO(gongwb): change to de newst repo when they changed
         GIT_REPOSITORY  "https://github.com/wangjiawei04/brpc"
+        #GIT_REPOSITORY  "https://github.com/ziyoujiyi/brpc" # ssl error in the previous repo（can be mannual fixed）
         GIT_TAG         "e203afb794caf027da0f1e0776443e7d20c0c28e"
         PREFIX          ${BRPC_PREFIX_DIR}
         UPDATE_COMMAND  ""
@@ -74,4 +75,3 @@ ADD_DEPENDENCIES(brpc extern_brpc)
 add_definitions(-DBRPC_WITH_GLOG)
 
 LIST(APPEND external_project_dependencies brpc)
-

cmake/external/gflags.cmake

@@ -29,7 +29,34 @@ ENDIF(WIN32)
 
 INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR})
 
-ExternalProject_Add(
+if(WITH_ARM_BRPC)
+    SET(ARM_GFLAGS_URL "https://paddlerec.bj.bcebos.com/online_infer/arm_brpc_ubuntu18/arm_gflags.tar.gz" CACHE STRING "" FORCE)
+    set(GFLAGS_SOURCE_DIR ${THIRD_PARTY_PATH}/gflags/src/extern_gflags)
+    FILE(WRITE ${GFLAGS_SOURCE_DIR}/CMakeLists.txt
+    "PROJECT(ARM_GFLAGS)\n"
+    "cmake_minimum_required(VERSION 3.0)\n"
+    "install(DIRECTORY arm_gflags/bin  arm_gflags/include arm_gflags/lib \n"
+    "        DESTINATION . USE_SOURCE_PERMISSIONS)\n")
+    ExternalProject_Add(
+        extern_gflags
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        ${SHALLOW_CLONE}
+        PREFIX          ${GFLAGS_PREFIX_DIR}
+        DOWNLOAD_DIR          ${GFLAGS_SOURCE_DIR}
+        DOWNLOAD_COMMAND    rm -rf arm_gflags.tar.gz && 
+                            wget --no-check-certificate ${ARM_GFLAGS_URL}
+                            && tar zxvf arm_gflags.tar.gz
+        #DOWNLOAD_COMMAND    cp /home/wangbin44/Paddle/build/arm_gflags.tar.gz .
+        #                    && tar zxvf arm_gflags.tar.gz
+        UPDATE_COMMAND  ""
+        CMAKE_ARGS      -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR}
+                        -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+        CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR}
+                        -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+        BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES}
+    )
+else()
+    ExternalProject_Add(
         extern_gflags
         ${EXTERNAL_PROJECT_LOG_ARGS}
         ${SHALLOW_CLONE}
@@ -57,7 +84,8 @@ ExternalProject_Add(
                         -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
                         -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
         BUILD_BYPRODUCTS ${GFLAGS_LIBRARIES}
-)
+    )
+endif()
 
 ADD_LIBRARY(gflags STATIC IMPORTED GLOBAL)
 SET_PROPERTY(TARGET gflags PROPERTY IMPORTED_LOCATION ${GFLAGS_LIBRARIES})

cmake/external/glog.cmake

@@ -31,7 +31,35 @@ ENDIF(WIN32)
 
 INCLUDE_DIRECTORIES(${GLOG_INCLUDE_DIR})
 
-ExternalProject_Add(
+if(WITH_ARM_BRPC)
+    SET(ARM_GLOG_URL "https://paddlerec.bj.bcebos.com/online_infer/arm_brpc_ubuntu18/arm_glog.tar.gz" CACHE STRING "" FORCE)
+    set(GLOG_SOURCE_DIR ${THIRD_PARTY_PATH}/glog/src/extern_glog)
+    FILE(WRITE ${GLOG_SOURCE_DIR}/CMakeLists.txt
+    "PROJECT(ARM_GLOGS)\n"
+    "cmake_minimum_required(VERSION 3.0)\n"
+    "install(DIRECTORY arm_glog/include arm_glog/lib \n"
+    "        DESTINATION . USE_SOURCE_PERMISSIONS)\n")
+    ExternalProject_Add(
+        extern_glog
+        ${EXTERNAL_PROJECT_LOG_ARGS}
+        ${SHALLOW_CLONE}
+        DEPENDS         gflags
+        PREFIX          ${GLOG_PREFIX_DIR}
+        DOWNLOAD_DIR          ${GLOG_SOURCE_DIR}
+        DOWNLOAD_COMMAND    rm -rf arm_glog.tar.gz &&
+                            wget --no-check-certificate ${ARM_GLOG_URL}
+                            && tar zxvf arm_glog.tar.gz
+        #DOWNLOAD_COMMAND    cp /home/wangbin44/Paddle/build/arm_glog.tar.gz .
+        #                    && tar zxvf arm_glog.tar.gz
+        UPDATE_COMMAND  ""
+        CMAKE_ARGS      -DCMAKE_INSTALL_PREFIX:PATH=${GLOG_INSTALL_DIR}
+                        -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+        CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GLOG_INSTALL_DIR}
+                        -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+        BUILD_BYPRODUCTS ${GLOG_LIBRARIES}
+    )
+else()
+    ExternalProject_Add(
         extern_glog
         ${EXTERNAL_PROJECT_LOG_ARGS}
         ${SHALLOW_CLONE}
@@ -60,7 +88,8 @@ ExternalProject_Add(
                         -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON
                         -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
         BUILD_BYPRODUCTS ${GLOG_LIBRARIES}
-)
+    )
+endif()
 
 ADD_LIBRARY(glog STATIC IMPORTED GLOBAL)
 SET_PROPERTY(TARGET glog PROPERTY IMPORTED_LOCATION ${GLOG_LIBRARIES})

cmake/external/protobuf.cmake

@@ -219,7 +219,36 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
         SET(PROTOBUF_REPOSITORY  ${GIT_URL}/protocolbuffers/protobuf.git)
         SET(PROTOBUF_TAG         9f75c5aa851cd877fb0d93ccc31b8567a6706546)
     endif()
-
+    if(WITH_ARM_BRPC)
+        SET(ARM_PROTOBUF_URL "https://paddlerec.bj.bcebos.com/online_infer/arm_brpc_ubuntu18/arm_protobuf.tar.gz" CACHE STRING "" FORCE)
+        FILE(WRITE ${PROTOBUF_SOURCE_DIR}/CMakeLists.txt
+        "PROJECT(ARM_PROTOBUF)\n"
+        "cmake_minimum_required(VERSION 3.0)\n"
+        "install(DIRECTORY arm_protobuf/bin  arm_protobuf/include arm_protobuf/lib \n"
+	"        DESTINATION . USE_SOURCE_PERMISSIONS)\n")
+        ExternalProject_Add(
+            ${TARGET_NAME}
+            ${EXTERNAL_PROJECT_LOG_ARGS}
+            ${SHALLOW_CLONE}
+            PREFIX          ${PROTOBUF_PREFIX_DIR}
+            DOWNLOAD_DIR          ${PROTOBUF_SOURCE_DIR}
+            DOWNLOAD_COMMAND     rm -rf arm_protobuf.tar.gz
+                                 && wget --no-check-certificate ${ARM_PROTOBUF_URL}
+                                 && tar zxvf arm_protobuf.tar.gz
+            #DOWNLOAD_COMMAND    cp /home/wangbin44/Paddle/build/arm_protobuf.tar.gz .
+            #                    && tar zxvf arm_protobuf.tar.gz
+            UPDATE_COMMAND  ""
+            CMAKE_ARGS          -DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR}
+                                -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+            CMAKE_CACHE_ARGS
+                                -DCMAKE_INSTALL_PREFIX:PATH=${PROTOBUF_INSTALL_DIR}
+                                -DCMAKE_BUILD_TYPE:STRING=${THIRD_PARTY_BUILD_TYPE}
+            BUILD_BYPRODUCTS ${PROTOBUF_INSTALL_DIR}/lib/libprotobuf${CMAKE_STATIC_LIBRARY_SUFFIX}
+            BUILD_BYPRODUCTS ${PROTOBUF_INSTALL_DIR}/lib/libprotobuf-lite${CMAKE_STATIC_LIBRARY_SUFFIX}
+            BUILD_BYPRODUCTS ${PROTOBUF_INSTALL_DIR}/lib/libprotoc${CMAKE_STATIC_LIBRARY_SUFFIX}
+            BUILD_BYPRODUCTS ${PROTOBUF_INSTALL_DIR}/bin/protoc${CMAKE_EXECUTABLE_SUFFIX}
+        )
+    else()
         ExternalProject_Add(
             ${TARGET_NAME}
             ${EXTERNAL_PROJECT_LOG_ARGS}
@@ -250,6 +279,7 @@ FUNCTION(build_protobuf TARGET_NAME BUILD_FOR_HOST)
             BUILD_BYPRODUCTS ${PROTOBUF_INSTALL_DIR}/lib/libprotoc${CMAKE_STATIC_LIBRARY_SUFFIX}
             BUILD_BYPRODUCTS ${PROTOBUF_INSTALL_DIR}/bin/protoc${CMAKE_EXECUTABLE_SUFFIX}
         )
+    endif()
 ENDFUNCTION()
 
 if(WITH_ONNXRUNTIME)
@@ -258,6 +288,8 @@ elseif(WITH_ASCEND OR WITH_ASCEND_CL)
     SET(PROTOBUF_VERSION 3.8.0)
 elseif(WITH_IPU)
     SET(PROTOBUF_VERSION 3.6.1)
+elseif(WITH_ARM_BRPC)
+    SET(PROTOBUF_VERSION 3.7.1-baidu-ee-common)
 else()
     SET(PROTOBUF_VERSION 3.1.0)
 endif()

cmake/third_party.cmake

@@ -167,6 +167,13 @@ if(WIN32 OR APPLE)
         SET(WITH_PSLIB OFF CACHE STRING "Disable PSLIB package in Windows and MacOS" FORCE)
     endif()
 
+    if(WITH_ARM_BRPC)
+        MESSAGE(WARNING
+            "Windows or Mac is not supported with ARM_BRPC in Paddle yet."
+            "Force WITH_ARM_BRPC=OFF")
+        SET(WITH_ARM_BRPC OFF CACHE STRING "Disable ARM_BRPC package in Windows and MacOS" FORCE)
+    endif()
+
     if(WITH_LIBMCT)
         MESSAGE(WARNING
             "Windows or Mac is not supported with LIBMCT in Paddle yet."
@@ -339,8 +346,13 @@ if (WITH_PSCORE)
     include(external/leveldb)
     list(APPEND third_party_deps extern_leveldb)
     
+    if (WITH_ARM_BRPC)
+        include(external/arm_brpc)
+        list(APPEND third_party_deps extern_arm_brpc)
+    else()
         include(external/brpc)
         list(APPEND third_party_deps extern_brpc)
+    endif()
 
     include(external/libmct)     # download, build, install libmct
     list(APPEND third_party_deps extern_libmct)

paddle/fluid/distributed/fleet_executor/CMakeLists.txt

@@ -4,7 +4,9 @@ if(WITH_PYTHON)
 endif()
 proto_library(interceptor_message_proto SRCS interceptor_message.proto)
 
-if(WITH_DISTRIBUTE AND WITH_PSCORE)
+if(WITH_ARM_BRPC)
+  set(BRPC_DEPS arm_brpc snappy gflags glog)
+elseif(WITH_DISTRIBUTE AND WITH_PSCORE)
   set(BRPC_DEPS brpc ssl crypto protobuf zlib leveldb snappy gflags glog)
 else()
   set(BRPC_DEPS "")

paddle/fluid/distributed/ps/service/heter_client.h

@@ -93,7 +93,15 @@ class HeterClient {
     options.timeout_ms = FLAGS_pserver_timeout_ms;
     std::vector<std::shared_ptr<brpc::Channel>>* client_channels = nullptr;
     if (peer_role == PEER_ROLE_IS_SWITCH) {
+#ifdef PADDLE_WITH_ARM_BRPC
+      if (need_encrypt) {
+        options.mutable_ssl_options();
+      }
+      options.connection_type = "";
+      VLOG(4) << "ssl enabled in arm";
+#else
       options.ssl_options.enable = need_encrypt;
+#endif
       client_channels = &peer_switch_channels_;
     } else if (peer_role == PEER_ROLE_IS_WORKER) {
       client_channels = &peer_worker_channels_;

paddle/fluid/distributed/ps/service/heter_server.cc

@@ -32,8 +32,13 @@ void HeterServer::StartHeterService(bool neeed_encrypt) {
   server_.AddService(&service_, brpc::SERVER_DOESNT_OWN_SERVICE);
   brpc::ServerOptions options;
   if (neeed_encrypt) {
+#ifdef PADDLE_WITH_ARM_BRPC
+    options.mutable_ssl_options()->default_cert.certificate = "/cert.pem";
+    options.mutable_ssl_options()->default_cert.private_key = "/key.pem";
+#else
     options.ssl_options.default_cert.certificate = "/cert.pem";
     options.ssl_options.default_cert.private_key = "/key.pem";
+#endif
   }
   if (server_.Start(endpoint_.c_str(), &options) != 0) {
     VLOG(0) << "HeterServer start fail. Try again.";
@@ -67,8 +72,13 @@ void HeterServer::StartHeterInterService(bool neeed_encrypt) {
   server_inter_.AddService(&service_, brpc::SERVER_DOESNT_OWN_SERVICE);
   brpc::ServerOptions options;
   if (neeed_encrypt) {
+#ifdef PADDLE_WITH_ARM_BRPC
+    options.mutable_ssl_options()->default_cert.certificate = "/cert.pem";
+    options.mutable_ssl_options()->default_cert.private_key = "/key.pem";
+#else
     options.ssl_options.default_cert.certificate = "/cert.pem";
     options.ssl_options.default_cert.private_key = "/key.pem";
+#endif
   }
   if (server_inter_.Start(endpoint_inter_.c_str(), &options) != 0) {
     VLOG(4) << "switch inter server start fail. Try again.";

paddle/fluid/framework/CMakeLists.txt

@@ -126,6 +126,9 @@ if(WITH_PSLIB)
     elseif(NOT WITH_HETERPS)
         set(BRPC_DEPS brpc ssl crypto)
     endif()
+    if (WITH_ARM_BRPC)
+        set(BRPC_DEPS arm_brpc)
+    endif()
 endif()
 
 cc_library(scope SRCS scope.cc DEPS glog threadpool xxhash var_type_traits)

paddle/fluid/operators/pscore/CMakeLists.txt

@@ -6,7 +6,11 @@ include(operators)
 
 set(DISTRIBUTE_DEPS "")
 
-list(APPEND DISTRIBUTE_DEPS executor fleet ps_service brpc_utils heter_server heter_client ps_framework_proto framework_proto sendrecv_rpc brpc leveldb ssl crypto protobuf gflags glog zlib snappy device_context)
+if (WITH_ARM_BRPC)
+    list(APPEND DISTRIBUTE_DEPS executor fleet ps_service brpc_utils heter_server heter_client ps_framework_proto framework_proto sendrecv_rpc arm_brpc gflags glog snappy device_context)
+else()
+    list(APPEND DISTRIBUTE_DEPS executor fleet ps_service brpc_utils heter_server heter_client ps_framework_proto framework_proto sendrecv_rpc brpc leveldb ssl crypto protobuf gflags glog zlib snappy device_context)
+endif()
 
 set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=parentheses")
 
@@ -40,3 +44,6 @@ cc_test(heter_listen_and_server_test SRCS heter_listen_and_server_test.cc DEPS e
 
 #set_source_files_properties(heter_cloud_comm_cpu_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
 #cc_test(heter_cloud_comm_cpu_test SRCS heter_cloud_comm_cpu_test.cc DEPS executor scope proto_desc scale_op heter_listen_and_serv_op ${RPC_DEPS} ${DISTRIBUTE_DEPS} eigen_function)
+
+set_source_files_properties(switch_server_test.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
+cc_binary(switch_server_test SRCS switch_server_test.cc DEPS executor scope proto_desc scale_op heter_listen_and_serv_op ${RPC_DEPS} ${DISTRIBUTE_DEPS} eigen_function)

paddle/fluid/operators/pscore/switch_server_test.cc

+/* Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#if defined PADDLE_WITH_PSCORE
+#include <stdlib.h>
+
+#include <memory>
+#include <random>
+#include <sstream>
+#include <string>
+#include <thread>  // NOLINT
+
+#include "gflags/gflags.h"
+#include "gtest/gtest.h"
+#include "paddle/fluid/distributed/ps/service/heter_client.h"
+#include "paddle/fluid/distributed/ps/service/heter_server.h"
+
+namespace framework = paddle::framework;
+namespace platform = paddle::platform;
+namespace distributed = paddle::distributed;
+
+DEFINE_string(switch_addr_inner, "127.0.0.1:6000", "addr of inner cluster");
+DEFINE_string(switch_addr_heter, "127.0.0.1:6100", "add of inter cluster");
+DEFINE_string(peer_switch_addr, "127.0.0.1:7100", "add of inter cluster");
+
+void StartSwitchServer(
+    std::shared_ptr<distributed::HeterServer>& switch_server_ptr,  // NOLINT
+    std::vector<std::string> endpoints,
+    std::vector<std::string> peer_endpoints) {
+  switch_server_ptr->SetPeerEndPoints(peer_endpoints);
+  switch_server_ptr->SetEndPoint(endpoints[0]);
+  switch_server_ptr->StartHeterService(false);
+}
+
+void StartSwitchInterServer(
+    std::shared_ptr<distributed::HeterServer>& switch_server_ptr,  // NOLINT
+    std::vector<std::string> endpoints,
+    std::vector<std::string> peer_endpoints) {
+  LOG(INFO) << "switch heter service started";
+  switch_server_ptr->SetPeerEndPoints(peer_endpoints);
+  switch_server_ptr->SetInterEndpoint(endpoints[0]);
+  switch_server_ptr->StartHeterInterService(false);
+}
+
+int main(int argc, char* argv[]) {
+  platform::CPUPlace place;
+  platform::CPUDeviceContext ctx(place);
+  framework::Executor exe(place);
+
+  framework::ProgramDesc program;
+  exe.Prepare(program, 0);  // solve undefined symbol: tensor_table.cc
+
+  google::ParseCommandLineFlags(&argc, &argv, true);
+
+  std::string switch_a_endpoint(FLAGS_switch_addr_inner);
+  std::string switch_a_endpoint_inter(FLAGS_switch_addr_heter);
+  std::string switch_b_endpoint_inter(FLAGS_peer_switch_addr);
+
+  std::shared_ptr<distributed::HeterServer> switch_server_ptr_a =
+      std::make_shared<distributed::HeterServer>();
+
+  std::vector<std::string> end_points{switch_a_endpoint};
+  std::vector<std::string> peer_endpoints{switch_b_endpoint_inter};
+  std::thread switch_server_a_thread(StartSwitchServer,
+                                     std::ref(switch_server_ptr_a), end_points,
+                                     peer_endpoints);
+  switch_server_ptr_a->WaitServerReady();
+
+  end_points = {switch_a_endpoint_inter};
+  peer_endpoints = {switch_b_endpoint_inter};
+  std::thread switch_server_a_thread_inter(StartSwitchInterServer,
+                                           std::ref(switch_server_ptr_a),
+                                           end_points, peer_endpoints);
+  switch_server_ptr_a->WaitServerReady();
+
+  switch_server_a_thread.join();
+  LOG(INFO) << "switch_server_a_thread joined";
+
+  switch_server_a_thread_inter.join();
+  LOG(INFO) << "switch_server_a_thread_inter joined";
+
+  return 0;
+}
+#endif

paddle/fluid/pybind/CMakeLists.txt

@@ -133,7 +133,11 @@ if (WITH_PSLIB)
   set_source_files_properties(heter_wrapper_py.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
 endif(WITH_PSLIB)
 if (WITH_PSCORE)
+  if (WITH_ARM_BRPC)
+    set(DISTRIBUTE_COMPILE_FLAGS "-faligned-new -Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=type-limits -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result")
+  else()
     set(DISTRIBUTE_COMPILE_FLAGS "-Wno-non-virtual-dtor -Wno-error=non-virtual-dtor -Wno-error=delete-non-virtual-dtor -Wno-error=sign-compare -Wno-error=unused-variable -Wno-error=return-type -Wno-error=unused-but-set-variable -Wno-error=type-limits -Wno-error=unknown-pragmas -Wno-error=parentheses -Wno-error=unused-result")
+  endif()
   set_source_files_properties(fleet_py.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS})
   list(APPEND PYBIND_DEPS fleet communicator index_wrapper index_sampler)
   list(APPEND PYBIND_SRCS fleet_py.cc)