From 2912d5311bccc3b89dd32a0e80f48be41ba7d1bc Mon Sep 17 00:00:00 2001 From: heqiaozhi Date: Thu, 13 Dec 2018 13:21:39 +0800 Subject: [PATCH] fix code style bug & change pslib.cmake & change Cmakelist adapt pslib --- CMakeLists.txt | 19 +++++++++++++------ cmake/external/pslib.cmake | 11 ++++++----- paddle/fluid/framework/async_executor.cc | 7 +++---- paddle/fluid/framework/async_executor.h | 4 ++-- paddle/fluid/framework/data_feed.cc | 1 + .../fluid/framework/executor_thread_worker.cc | 4 ++-- .../fluid/framework/executor_thread_worker.h | 2 +- 7 files changed, 28 insertions(+), 20 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5b5bf6c5b6..c3b4349c8c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -65,6 +65,7 @@ option(WITH_GOLANG "Compile PaddlePaddle with GOLANG" OFF) option(GLIDE_INSTALL "Download and install go dependencies " ON) option(USE_NNPACK "Compile PaddlePaddle with NNPACK library" OFF) option(WITH_DISTRIBUTE "Compile with distributed support" OFF) +option(WITH_PSLIB "Compile with pslib support" OFF) option(USE_EIGEN_FOR_BLAS "Use matrix multiplication in Eigen" OFF) option(EIGEN_USE_THREADS "Compile with multi-threaded Eigen" OFF) option(WITH_ARM_FP16 "Use half precision support on armv8.2-a cpu" OFF) @@ -216,9 +217,12 @@ include(external/warpctc) # download, build, install warpctc include(cupti) include(external/gzstream) endif (NOT WIN32) -include(external/libmct) -include(external/pslib_brpc) -include(external/pslib) + +if(WITH_PSLIB) + include(external/libmct) + include(external/pslib_brpc) + include(external/pslib) +endif() if(WITH_DISTRIBUTE) if(WITH_GRPC) @@ -279,11 +283,14 @@ set(EXTERNAL_LIBS protobuf zlib ${PYTHON_LIBRARIES} - pslib - pslib_brpc - libmct ) +if(WITH_PSLIB) + list(APPEND EXTERNAL_LIBS pslib) + list(APPEND EXTERNAL_LIBS pslib_brpc) + list(APPEND EXTERNAL_LIBS libmct) +endif(WITH_PSLIB) + if(WITH_AMD_GPU) find_package(HIP) include(hip) diff --git a/cmake/external/pslib.cmake b/cmake/external/pslib.cmake index 4d4dc195aa..3b495d78e2 100644 --- a/cmake/external/pslib.cmake +++ b/cmake/external/pslib.cmake @@ -30,9 +30,10 @@ SET(PSLIB_PROJECT "extern_pslib") IF((NOT DEFINED PSLIB_VER) OR (NOT DEFINED PSLIB_URL)) MESSAGE(STATUS "use pre defined download url") SET(PSLIB_VER "0.1.0" CACHE STRING "" FORCE) - SET(PSLIB_URL "https://raw.githubusercontent.com/PaddlePaddle/Fleet/release/${PSLIB_VER}/pslib.tar.gz" CACHE STRING "" FORCE) + SET(PSLIB_NAME "pslib" CACHE STRING "" FORCE) + SET(PSLIB_URL "https://raw.githubusercontent.com/PaddlePaddle/Fleet/release/${PSLIB_VER}/${PSLIB_NAME}.tar.gz" CACHE STRING "" FORCE) ENDIF() -MESSAGE(STATUS "PSLIB_VER: ${PSLIB_VER}, PSLIB_URL: ${PSLIB_URL}") +MESSAGE(STATUS "PSLIB_NAME: ${PSLIB_NAME}, PSLIB_URL: ${PSLIB_URL}") SET(PSLIB_SOURCE_DIR "${THIRD_PARTY_PATH}/pslib") SET(PSLIB_DOWNLOAD_DIR "${PSLIB_SOURCE_DIR}/src/${PSLIB_PROJECT}") SET(PSLIB_DST_DIR "pslib") @@ -50,7 +51,7 @@ INCLUDE_DIRECTORIES(${PSLIB_INC_DIR}) FILE(WRITE ${PSLIB_DOWNLOAD_DIR}/CMakeLists.txt "PROJECT(PSLIB)\n" "cmake_minimum_required(VERSION 3.0)\n" - "install(DIRECTORY ${PSLIB_VER}/include ${PSLIB_VER}/lib \n" + "install(DIRECTORY ${PSLIB_NAME}/include ${PSLIB_NAME}/lib \n" " DESTINATION ${PSLIB_DST_DIR})\n") ExternalProject_Add( @@ -58,8 +59,8 @@ ExternalProject_Add( ${EXTERNAL_PROJECT_LOG_ARGS} PREFIX ${PSLIB_SOURCE_DIR} DOWNLOAD_DIR ${PSLIB_DOWNLOAD_DIR} - DOWNLOAD_COMMAND wget --no-check-certificate ${PSLIB_URL} -c -q -O ${PSLIB_VER}.tar.gz - && tar zxvf ${PSLIB_VER}.tar.gz + DOWNLOAD_COMMAND wget --no-check-certificate ${PSLIB_URL} -c -q -O ${PSLIB_NAME}.tar.gz + && tar zxvf ${PSLIB_NAME}.tar.gz DOWNLOAD_NO_PROGRESS 1 UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_INSTALL_PREFIX=${PSLIB_INSTALL_ROOT} diff --git a/paddle/fluid/framework/async_executor.cc b/paddle/fluid/framework/async_executor.cc index c62d62a5dc..8231aff142 100644 --- a/paddle/fluid/framework/async_executor.cc +++ b/paddle/fluid/framework/async_executor.cc @@ -50,7 +50,6 @@ void AsyncExecutor::CreateThreads( worker->BindingDataFeedMemory(); worker->SetPSlibPtr(_pslib_ptr); worker->SetPullDenseThread(_pull_dense_thread); - worker->BindingSlotVariableMemory(); worker->SetParamConfig(&_param_config); } @@ -79,7 +78,7 @@ void AsyncExecutor::InitWorker(const std::string& dist_desc, _pslib_ptr = std::shared_ptr( new paddle::distributed::PSlib()); _pslib_ptr->init_worker( - dist_desc, host_sign_list.data(), node_num, index); + dist_desc, (uint64_t*)(host_sign_list.data()), node_num, index); InitParamConfig(); } @@ -93,8 +92,8 @@ void AsyncExecutor::StopServer() { } void AsyncExecutor::GatherServers( - std::vector& host_sign_list, int node_num) { - _pslib_ptr->gather_servers(host_sign_list.data(), node_num); + const std::vector& host_sign_list, int node_num) { + _pslib_ptr->gather_servers((uint64_t*)(host_sign_list.data()), node_num); } void AsyncExecutor::InitParamConfig() { diff --git a/paddle/fluid/framework/async_executor.h b/paddle/fluid/framework/async_executor.h index 184566dd39..16540c2df2 100644 --- a/paddle/fluid/framework/async_executor.h +++ b/paddle/fluid/framework/async_executor.h @@ -43,9 +43,9 @@ inline std::default_random_engine& local_random_engine() { struct engine_wrapper_t { std::default_random_engine engine; engine_wrapper_t() { - static std::atomic x(0); + static std::atomic x(0); std::seed_seq sseq = {x++, x++, x++, - static_cast(current_realtime() * 1000)}; + static_cast(current_realtime() * 1000)}; engine.seed(sseq); } }; diff --git a/paddle/fluid/framework/data_feed.cc b/paddle/fluid/framework/data_feed.cc index 851c7eda89..54a00f8ccf 100644 --- a/paddle/fluid/framework/data_feed.cc +++ b/paddle/fluid/framework/data_feed.cc @@ -68,6 +68,7 @@ bool DataFeed::PickOneFile(std::string* filename) { return false; } *filename = filelist_[file_idx_++]; + LOG(ERROR) << "pick file:" << *filename; return true; } diff --git a/paddle/fluid/framework/executor_thread_worker.cc b/paddle/fluid/framework/executor_thread_worker.cc index 412f4a2b6e..df15a4d293 100644 --- a/paddle/fluid/framework/executor_thread_worker.cc +++ b/paddle/fluid/framework/executor_thread_worker.cc @@ -637,7 +637,7 @@ void AsyncExecutorThreadWorker::collect_feasign_info( } void AsyncExecutorThreadWorker::check_pull_push_memory( - std::vector& features, + const std::vector& features, std::vector>& push_g, int dim) { push_g.resize(features.size() + 1); @@ -647,7 +647,7 @@ void AsyncExecutorThreadWorker::check_pull_push_memory( } void AsyncExecutorThreadWorker::check_pull_push_memory( - std::vector& features, + const std::vector& features, std::vector& push_g, int dim) { if (features.size() > push_g.size()) { diff --git a/paddle/fluid/framework/executor_thread_worker.h b/paddle/fluid/framework/executor_thread_worker.h index b6c4f950ec..93373b1d2e 100644 --- a/paddle/fluid/framework/executor_thread_worker.h +++ b/paddle/fluid/framework/executor_thread_worker.h @@ -155,7 +155,7 @@ class ExecutorThreadWorker { // set fetch variable names from python interface assigned by users void SetFetchVarNames(const std::vector& fetch_var_names); virtual void SetPSlibPtr( - std::shared_ptr pslib_ptr); + std::shared_ptr pslib_ptr) {}; virtual void SetPullDenseThread( std::shared_ptr dpt) {} virtual void SetParamConfig( -- GitLab