未验证 提交 4748486e 编写于 作者: T Thunderbrook 提交者: GitHub

compile for afs api (#39113)

* compile for afs api

* with pslib
上级 a7edb3f3
......@@ -333,6 +333,10 @@ else()
endif()
endif()
if(WITH_HETERPS AND WITH_PSLIB)
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D_GLIBCXX_USE_CXX11_ABI=0")
endif()
if(WITH_DISTRIBUTE)
if(LINUX)
set(WITH_GLOO ON CACHE STRING "Enable GLOO when compiling WITH_DISTRIBUTE=ON." FORCE)
......
......@@ -55,7 +55,7 @@ else()
UPDATE_COMMAND ""
CONFIGURE_COMMAND ""
BUILD_COMMAND mkdir -p ${GLOO_SOURCE_DIR}/build
&& cd ${GLOO_SOURCE_DIR}/build && cmake .. && make
&& cd ${GLOO_SOURCE_DIR}/build && cmake .. -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} && make
&& mkdir -p ${GLOO_LIBRARY_DIR} ${GLOO_INCLUDE_DIR}/gloo
INSTALL_COMMAND ${CMAKE_COMMAND} -E copy ${GLOO_SOURCE_DIR}/build/gloo/libgloo.a ${GLOO_LIBRARY_DIR}
COMMAND ${CMAKE_COMMAND} -E copy_directory "${GLOO_SOURCE_DIR}/gloo/" "${GLOO_INCLUDE_DIR}/gloo"
......
......@@ -202,6 +202,16 @@ if (APPLE)
set (COMMON_FLAGS -Wno-deprecated-register)
endif(APPLE)
if(WITH_HETERPS AND WITH_PSLIB)
set(COMMON_FLAGS
-D_GLIBCXX_USE_CXX11_ABI=0
${COMMON_FLAGS})
set(GPU_COMMON_FLAGS
-D_GLIBCXX_USE_CXX11_ABI=0
${GPU_COMMON_FLAGS})
endif()
if(LINUX)
set(GPU_COMMON_FLAGS
-Wall
......
......@@ -299,9 +299,11 @@ if(WITH_PSLIB)
include(external/leveldb)
list(APPEND third_party_deps extern_leveldb)
if(NOT WITH_HETERPS)
include(external/brpc)
list(APPEND third_party_deps extern_brpc)
endif()
endif()
endif(WITH_PSLIB)
if(NOT WIN32 AND NOT APPLE)
......
......@@ -118,7 +118,9 @@ cc_test(var_type_traits_test SRCS var_type_traits_test.cc DEPS var_type_traits)
set(BRPC_DEPS "")
if(WITH_PSLIB OR WITH_PSCORE)
if(NOT WITH_HETERPS)
set(BRPC_DEPS brpc ssl crypto)
endif()
if(WITH_PSLIB_BRPC)
set(BRPC_DEPS pslib_brpc)
endif()
......
......@@ -61,7 +61,7 @@ bool CheckValidOutput(LoDTensor* tensor, size_t batch_size);
class FleetWrapper;
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
class HeterWrapper;
#endif
......@@ -359,7 +359,7 @@ class DownpourWorkerOpt : public DownpourWorker {
uint64_t async_tid_ = 0;
};
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
class HeterCpuWorker : public HogwildWorker {
public:
HeterCpuWorker() {}
......
......@@ -70,7 +70,7 @@ REGISTER_DEVICE_WORKER_CLASS(DownpourWorkerOpt);
REGISTER_DEVICE_WORKER_CLASS(HeterSectionWorker);
#endif
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
REGISTER_DEVICE_WORKER_CLASS(HeterCpuWorker);
#endif
......
......@@ -2,7 +2,9 @@ if(WITH_PSLIB)
if(WITH_PSLIB_BRPC)
set(BRPC_DEPS pslib_brpc)
else()
if(NOT WITH_HETERPS)
set(BRPC_DEPS brpc)
endif()
endif(WITH_PSLIB_BRPC)
cc_library(fleet_wrapper SRCS fleet_wrapper.cc DEPS framework_proto proto_desc op_registry variable_helper scope ${BRPC_DEPS} pslib)
else()
......
......@@ -27,7 +27,7 @@ See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/fluid/framework/fleet/heter_wrapper.h"
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
#include "paddle/fluid/framework/device_worker.h"
namespace paddle {
......
......@@ -23,7 +23,7 @@ limitations under the License. */
#include <unordered_map>
#include <vector>
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
#include "paddle/fluid/framework/heter_service.h"
#include "paddle/fluid/framework/heter_util.h"
#include "paddle/fluid/framework/scope.h"
......
......@@ -25,7 +25,7 @@ limitations under the License. */
#include "paddle/fluid/framework/heter_service.pb.h"
#include "paddle/fluid/framework/program_desc.h"
#include "paddle/fluid/framework/scope.h"
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
#include "brpc/channel.h"
#include "brpc/controller.h"
#include "brpc/server.h"
......@@ -35,7 +35,7 @@ limitations under the License. */
namespace paddle {
namespace framework {
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
typedef std::function<int(const HeterRequest*, HeterResponse*)>
HeterServiceHandler;
class DataFeed;
......
......@@ -14,7 +14,6 @@ limitations under the License. */
#pragma once
#ifdef PADDLE_WITH_PSLIB
#include <fstream>
#include <memory>
#include <mutex> // NOLINT
......@@ -23,11 +22,12 @@ limitations under the License. */
#include <unordered_map> // NOLINT
#include <unordered_set> // NOLINT
#include <vector>
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
#include "bthread/bthread.h"
#endif
#include "paddle/fluid/framework/program_desc.h"
#include "paddle/fluid/framework/scope.h"
#include "paddle/fluid/platform/timer.h"
namespace paddle {
namespace framework {
class DataFeed;
......@@ -106,7 +106,7 @@ class HeterTask {
double cpu_2_gpu_time{0};
platform::Timer timeline;
};
#endif
template <class T>
class HeterObjectPool {
public:
......@@ -143,7 +143,7 @@ class HeterObjectPool {
int num_{0};
};
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
struct BthreadMutextGuard {
BthreadMutextGuard(bthread_mutex_t* rho) {
mutex_ = rho;
......@@ -328,6 +328,6 @@ class HeterList {
int cap_;
int size;
};
#endif
} // namespace framework
} // namespace paddle
#endif
......@@ -19,7 +19,7 @@ limitations under the License. */
#include "paddle/fluid/platform/cpu_helper.h"
#include "paddle/fluid/string/string_helper.h"
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
#if defined _WIN32 || defined __APPLE__
#else
......
......@@ -24,7 +24,7 @@ limitations under the License. */
#include "paddle/fluid/framework/fleet/heter_wrapper.h"
#include "paddle/fluid/framework/trainer.h"
#if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_XPU) && \
(defined PADDLE_WITH_PSLIB)
(defined PADDLE_WITH_PSLIB) && (!defined(PADDLE_WITH_HETERPS))
#ifdef PADDLE_WITH_CUDA
#include "paddle/fluid/platform/cuda_device_guard.h"
#endif
......
......@@ -155,7 +155,7 @@ class DistMultiTrainer : public MultiTrainer {
#if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_HIP || \
defined PADDLE_WITH_XPU) && \
(defined PADDLE_WITH_PSLIB)
(defined PADDLE_WITH_PSLIB) && (!defined(PADDLE_WITH_HETERPS))
class HeterServiceContext {
public:
HeterServiceContext() {}
......
......@@ -73,7 +73,7 @@ REGISTER_TRAINER_CLASS(HeterPipelineTrainer);
#if (defined PADDLE_WITH_CUDA || defined PADDLE_WITH_HIP || \
defined PADDLE_WITH_XPU) && \
(defined PADDLE_WITH_PSLIB)
(defined PADDLE_WITH_PSLIB) && (!defined(PADDLE_WITH_HETERPS))
REGISTER_TRAINER_CLASS(HeterXpuTrainer);
#endif
#if (defined PADDLE_WITH_NCCL || defined PADDLE_WITH_RCCL) && \
......
......@@ -33,7 +33,7 @@ namespace py = pybind11;
namespace paddle {
namespace pybind {
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
void BindHeterWrapper(py::module* m) {
py::class_<framework::HeterWrapper, std::shared_ptr<framework::HeterWrapper>>(
*m, "Heter")
......
......@@ -22,7 +22,7 @@ namespace py = pybind11;
namespace paddle {
namespace pybind {
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
void BindHeterWrapper(py::module* m);
#endif
} // namespace pybind
......
......@@ -3676,7 +3676,7 @@ All parameter, weight, gradient are variables in Paddle.
BindFleetWrapper(&m);
BindIO(&m);
#ifdef PADDLE_WITH_PSLIB
#if defined(PADDLE_WITH_PSLIB) && !defined(PADDLE_WITH_HETERPS)
BindHeterWrapper(&m);
#endif
#ifdef PADDLE_WITH_HETERPS
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册