From 4748486ebebd8be9ee71d00d9edf826c30ac25fa Mon Sep 17 00:00:00 2001 From: Thunderbrook <52529258+Thunderbrook@users.noreply.github.com> Date: Thu, 27 Jan 2022 13:07:34 +0800 Subject: [PATCH] compile for afs api (#39113) * compile for afs api * with pslib --- CMakeLists.txt | 4 ++++ cmake/external/gloo.cmake | 2 +- cmake/flags.cmake | 10 ++++++++++ cmake/third_party.cmake | 6 ++++-- paddle/fluid/framework/CMakeLists.txt | 4 +++- paddle/fluid/framework/device_worker.h | 4 ++-- paddle/fluid/framework/device_worker_factory.cc | 2 +- paddle/fluid/framework/fleet/CMakeLists.txt | 4 +++- paddle/fluid/framework/fleet/heter_wrapper.cc | 2 +- paddle/fluid/framework/fleet/heter_wrapper.h | 2 +- paddle/fluid/framework/heter_service.h | 4 ++-- paddle/fluid/framework/heter_util.h | 10 +++++----- paddle/fluid/framework/hetercpu_worker.cc | 2 +- paddle/fluid/framework/heterxpu_trainer.cc | 2 +- paddle/fluid/framework/trainer.h | 2 +- paddle/fluid/framework/trainer_factory.cc | 2 +- paddle/fluid/pybind/heter_wrapper_py.cc | 2 +- paddle/fluid/pybind/heter_wrapper_py.h | 2 +- paddle/fluid/pybind/pybind.cc | 2 +- 19 files changed, 44 insertions(+), 24 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 43bd4e0fcf8..047db58cfdf 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -333,6 +333,10 @@ else() endif() endif() +if(WITH_HETERPS AND WITH_PSLIB) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0") +endif() + if(WITH_DISTRIBUTE) if(LINUX) set(WITH_GLOO ON CACHE STRING "Enable GLOO when compiling WITH_DISTRIBUTE=ON." FORCE) diff --git a/cmake/external/gloo.cmake b/cmake/external/gloo.cmake index 3b88e67f69f..778d7c2a0ae 100644 --- a/cmake/external/gloo.cmake +++ b/cmake/external/gloo.cmake @@ -55,7 +55,7 @@ else() UPDATE_COMMAND "" CONFIGURE_COMMAND "" BUILD_COMMAND mkdir -p ${GLOO_SOURCE_DIR}/build - && cd ${GLOO_SOURCE_DIR}/build && cmake .. && make + && cd ${GLOO_SOURCE_DIR}/build && cmake .. -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && make && mkdir -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/gloo INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR} COMMAND ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/" "${GLOO_INCLUDE_DIR}/gloo" diff --git a/cmake/flags.cmake b/cmake/flags.cmake index 7495ee32bab..f90b71f9e60 100644 --- a/cmake/flags.cmake +++ b/cmake/flags.cmake @@ -202,6 +202,16 @@ if (APPLE) set (COMMON_FLAGS -Wno-deprecated-register) endif(APPLE) +if(WITH_HETERPS AND WITH_PSLIB) + set(COMMON_FLAGS + -D_GLIBCXX_USE_CXX11_ABI=0 + ${COMMON_FLAGS}) + + set(GPU_COMMON_FLAGS + -D_GLIBCXX_USE_CXX11_ABI=0 + ${GPU_COMMON_FLAGS}) +endif() + if(LINUX) set(GPU_COMMON_FLAGS -Wall diff --git a/cmake/third_party.cmake b/cmake/third_party.cmake index 38e7d2039af..ac3eff04d53 100644 --- a/cmake/third_party.cmake +++ b/cmake/third_party.cmake @@ -299,8 +299,10 @@ if(WITH_PSLIB) include(external/leveldb) list(APPEND third_party_deps extern_leveldb) - include(external/brpc) - list(APPEND third_party_deps extern_brpc) + if(NOT WITH_HETERPS) + include(external/brpc) + list(APPEND third_party_deps extern_brpc) + endif() endif() endif(WITH_PSLIB) diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index bf6393544f7..27ba88b56f0 100644 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -118,7 +118,9 @@ cc_test(var_type_traits_test SRCS var_type_traits_test.cc DEPS var_type_traits) set(BRPC_DEPS "") if(WITH_PSLIB OR WITH_PSCORE) - set(BRPC_DEPS brpc ssl crypto) + if(NOT WITH_HETERPS) + set(BRPC_DEPS brpc ssl crypto) + endif() if(WITH_PSLIB_BRPC) set(BRPC_DEPS pslib_brpc) endif() diff --git a/paddle/fluid/framework/device_worker.h b/paddle/fluid/framework/device_worker.h index 7aadc856129..6c5adb5fcdf 100644 --- a/paddle/fluid/framework/device_worker.h +++ b/paddle/fluid/framework/device_worker.h @@ -61,7 +61,7 @@ bool CheckValidOutput(LoDTensor* tensor, size_t batch_size); class FleetWrapper; -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) class HeterWrapper; #endif @@ -359,7 +359,7 @@ class DownpourWorkerOpt : public DownpourWorker { uint64_t async_tid_ = 0; }; -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) class HeterCpuWorker : public HogwildWorker { public: HeterCpuWorker() {} diff --git a/paddle/fluid/framework/device_worker_factory.cc b/paddle/fluid/framework/device_worker_factory.cc index 8259d43cb9a..24834d39ce3 100644 --- a/paddle/fluid/framework/device_worker_factory.cc +++ b/paddle/fluid/framework/device_worker_factory.cc @@ -70,7 +70,7 @@ REGISTER_DEVICE_WORKER_CLASS(DownpourWorkerOpt); REGISTER_DEVICE_WORKER_CLASS(HeterSectionWorker); #endif -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) REGISTER_DEVICE_WORKER_CLASS(HeterCpuWorker); #endif diff --git a/paddle/fluid/framework/fleet/CMakeLists.txt b/paddle/fluid/framework/fleet/CMakeLists.txt index 19d71076542..65214cb2591 100644 --- a/paddle/fluid/framework/fleet/CMakeLists.txt +++ b/paddle/fluid/framework/fleet/CMakeLists.txt @@ -2,7 +2,9 @@ if(WITH_PSLIB) if(WITH_PSLIB_BRPC) set(BRPC_DEPS pslib_brpc) else() - set(BRPC_DEPS brpc) + if(NOT WITH_HETERPS) + set(BRPC_DEPS brpc) + endif() endif(WITH_PSLIB_BRPC) cc_library(fleet_wrapper SRCS fleet_wrapper.cc DEPS framework_proto proto_desc op_registry variable_helper scope ${BRPC_DEPS} pslib) else() diff --git a/paddle/fluid/framework/fleet/heter_wrapper.cc b/paddle/fluid/framework/fleet/heter_wrapper.cc index 5b54aa03bb3..88c9a363d27 100644 --- a/paddle/fluid/framework/fleet/heter_wrapper.cc +++ b/paddle/fluid/framework/fleet/heter_wrapper.cc @@ -27,7 +27,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/fluid/framework/fleet/heter_wrapper.h" -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) #include "paddle/fluid/framework/device_worker.h" namespace paddle { diff --git a/paddle/fluid/framework/fleet/heter_wrapper.h b/paddle/fluid/framework/fleet/heter_wrapper.h index 4e529de0775..76a21651bfa 100644 --- a/paddle/fluid/framework/fleet/heter_wrapper.h +++ b/paddle/fluid/framework/fleet/heter_wrapper.h @@ -23,7 +23,7 @@ limitations under the License. */ #include #include -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) #include "paddle/fluid/framework/heter_service.h" #include "paddle/fluid/framework/heter_util.h" #include "paddle/fluid/framework/scope.h" diff --git a/paddle/fluid/framework/heter_service.h b/paddle/fluid/framework/heter_service.h index 7e5bf138d9f..9d0e3c50953 100644 --- a/paddle/fluid/framework/heter_service.h +++ b/paddle/fluid/framework/heter_service.h @@ -25,7 +25,7 @@ limitations under the License. */ #include "paddle/fluid/framework/heter_service.pb.h" #include "paddle/fluid/framework/program_desc.h" #include "paddle/fluid/framework/scope.h" -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) #include "brpc/channel.h" #include "brpc/controller.h" #include "brpc/server.h" @@ -35,7 +35,7 @@ limitations under the License. */ namespace paddle { namespace framework { -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) typedef std::function HeterServiceHandler; class DataFeed; diff --git a/paddle/fluid/framework/heter_util.h b/paddle/fluid/framework/heter_util.h index eb9f3040afe..b32b4df349f 100644 --- a/paddle/fluid/framework/heter_util.h +++ b/paddle/fluid/framework/heter_util.h @@ -14,7 +14,6 @@ limitations under the License. */ #pragma once -#ifdef PADDLE_WITH_PSLIB #include #include #include // NOLINT @@ -23,11 +22,12 @@ limitations under the License. */ #include // NOLINT #include // NOLINT #include +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) #include "bthread/bthread.h" +#endif #include "paddle/fluid/framework/program_desc.h" #include "paddle/fluid/framework/scope.h" #include "paddle/fluid/platform/timer.h" - namespace paddle { namespace framework { class DataFeed; @@ -106,7 +106,7 @@ class HeterTask { double cpu_2_gpu_time{0}; platform::Timer timeline; }; -#endif + template class HeterObjectPool { public: @@ -143,7 +143,7 @@ class HeterObjectPool { int num_{0}; }; -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) struct BthreadMutextGuard { BthreadMutextGuard(bthread_mutex_t* rho) { mutex_ = rho; @@ -328,6 +328,6 @@ class HeterList { int cap_; int size; }; +#endif } // namespace framework } // namespace paddle -#endif diff --git a/paddle/fluid/framework/hetercpu_worker.cc b/paddle/fluid/framework/hetercpu_worker.cc index f50cc2769e9..75cc18887da 100644 --- a/paddle/fluid/framework/hetercpu_worker.cc +++ b/paddle/fluid/framework/hetercpu_worker.cc @@ -19,7 +19,7 @@ limitations under the License. */ #include "paddle/fluid/platform/cpu_helper.h" #include "paddle/fluid/string/string_helper.h" -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) #if defined _WIN32 || defined __APPLE__ #else diff --git a/paddle/fluid/framework/heterxpu_trainer.cc b/paddle/fluid/framework/heterxpu_trainer.cc index 01430781c64..e47681820e2 100644 --- a/paddle/fluid/framework/heterxpu_trainer.cc +++ b/paddle/fluid/framework/heterxpu_trainer.cc @@ -24,7 +24,7 @@ limitations under the License. */ #include "paddle/fluid/framework/fleet/heter_wrapper.h" #include "paddle/fluid/framework/trainer.h" #if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_XPU) && \ - (defined PADDLE_WITH_PSLIB) + (defined PADDLE_WITH_PSLIB) && (!defined(PADDLE_WITH_HETERPS)) #ifdef PADDLE_WITH_CUDA #include "paddle/fluid/platform/cuda_device_guard.h" #endif diff --git a/paddle/fluid/framework/trainer.h b/paddle/fluid/framework/trainer.h index 6c97c7fefb1..40a8aa459da 100644 --- a/paddle/fluid/framework/trainer.h +++ b/paddle/fluid/framework/trainer.h @@ -155,7 +155,7 @@ class DistMultiTrainer : public MultiTrainer { #if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_HIP || \ defined PADDLE_WITH_XPU) && \ - (defined PADDLE_WITH_PSLIB) + (defined PADDLE_WITH_PSLIB) && (!defined(PADDLE_WITH_HETERPS)) class HeterServiceContext { public: HeterServiceContext() {} diff --git a/paddle/fluid/framework/trainer_factory.cc b/paddle/fluid/framework/trainer_factory.cc index 6f003c2f497..f189d0213da 100644 --- a/paddle/fluid/framework/trainer_factory.cc +++ b/paddle/fluid/framework/trainer_factory.cc @@ -73,7 +73,7 @@ REGISTER_TRAINER_CLASS(HeterPipelineTrainer); #if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_HIP || \ defined PADDLE_WITH_XPU) && \ - (defined PADDLE_WITH_PSLIB) + (defined PADDLE_WITH_PSLIB) && (!defined(PADDLE_WITH_HETERPS)) REGISTER_TRAINER_CLASS(HeterXpuTrainer); #endif #if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \ diff --git a/paddle/fluid/pybind/heter_wrapper_py.cc b/paddle/fluid/pybind/heter_wrapper_py.cc index 0603cd3faae..0b3565d94a2 100644 --- a/paddle/fluid/pybind/heter_wrapper_py.cc +++ b/paddle/fluid/pybind/heter_wrapper_py.cc @@ -33,7 +33,7 @@ namespace py = pybind11; namespace paddle { namespace pybind { -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) void BindHeterWrapper(py::module* m) { py::class_>( *m, "Heter") diff --git a/paddle/fluid/pybind/heter_wrapper_py.h b/paddle/fluid/pybind/heter_wrapper_py.h index 047c1f359ea..b9bfcf070f5 100644 --- a/paddle/fluid/pybind/heter_wrapper_py.h +++ b/paddle/fluid/pybind/heter_wrapper_py.h @@ -22,7 +22,7 @@ namespace py = pybind11; namespace paddle { namespace pybind { -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) void BindHeterWrapper(py::module* m); #endif } // namespace pybind diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index d3d7e5794e7..3e94ccfd497 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -3676,7 +3676,7 @@ All parameter, weight, gradient are variables in Paddle. BindFleetWrapper(&m); BindIO(&m); -#ifdef PADDLE_WITH_PSLIB +#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS) BindHeterWrapper(&m); #endif #ifdef PADDLE_WITH_HETERPS -- GitLab