From 715d862868aca6ae4c865dc3db1cde6818e4ad1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=9F=B3=E6=99=93=E4=BC=9F?= <39303645+Shixiaowei02@users.noreply.github.com> Date: Fri, 15 Jan 2021 13:44:02 +0800 Subject: [PATCH] export global google flags to users, test=develop (#30448) --- cmake/external/gflags.cmake | 6 +- cmake/external/glog.cmake | 8 +-- .../fluid/distributed/service/communicator.cc | 2 +- paddle/fluid/distributed/service/service.cc | 6 +- paddle/fluid/inference/api/api.cc | 2 +- .../api/demo_ci/simple_on_word2vec.cc | 2 +- .../api/demo_ci/trt_mobilenet_demo.cc | 2 +- .../fluid/inference/api/demo_ci/vis_demo.cc | 2 +- .../api/demo_ci/windows_mobilenet.cc | 2 +- paddle/fluid/inference/check_symbol.sh | 2 +- paddle/fluid/inference/paddle_fluid.map | 1 + paddle/fluid/memory/detail/buddy_allocator.cc | 1 + paddle/fluid/operators/jit/benchmark.cc | 2 +- paddle/fluid/platform/enforce.h | 1 + paddle/fluid/platform/init.cc | 4 +- paddle/fluid/train/imdb_demo/demo_trainer.cc | 2 +- paddle/testing/paddle_gtest_main.cc | 55 ++++--------------- 17 files changed, 35 insertions(+), 65 deletions(-) diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index a077c8061b..34f5d7e2be 100644 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -19,7 +19,7 @@ SET(GFLAGS_SOURCE_DIR ${THIRD_PARTY_PATH}/gflags/src/extern_gflags) SET(GFLAGS_INSTALL_DIR ${THIRD_PARTY_PATH}/install/gflags) SET(GFLAGS_INCLUDE_DIR "${GFLAGS_INSTALL_DIR}/include" CACHE PATH "gflags include directory." FORCE) set(GFLAGS_REPOSITORY ${GIT_URL}/gflags/gflags.git) -set(GFLAGS_TAG 77592648e3f3be87d6c7123eb81cbad75f9aef5a) +set(GFLAGS_TAG "v2.2.2") IF(WIN32) set(GFLAGS_LIBRARIES "${GFLAGS_INSTALL_DIR}/lib/gflags_static.lib" CACHE FILEPATH "GFLAGS_LIBRARIES" FORCE) ELSE(WIN32) @@ -30,6 +30,8 @@ ENDIF(WIN32) INCLUDE_DIRECTORIES(${GFLAGS_INCLUDE_DIR}) +set(GFLAGS_NAMESPACE "paddle_gflags") + cache_third_party(extern_gflags REPOSITORY ${GFLAGS_REPOSITORY} TAG ${GFLAGS_TAG} @@ -44,7 +46,6 @@ ExternalProject_Add( SOURCE_DIR ${GFLAGS_SOURCE_DIR} BUILD_COMMAND ${BUILD_COMMAND} INSTALL_COMMAND ${INSTALL_COMMAND} - UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS} @@ -58,6 +59,7 @@ ExternalProject_Add( -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} + -DGFLAGS_NAMESPACE=${GFLAGS_NAMESPACE} ${EXTERNAL_OPTIONAL_ARGS} CMAKE_CACHE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${GFLAGS_INSTALL_DIR} -DCMAKE_POSITION_INDEPENDENT_CODE:BOOL=ON diff --git a/cmake/external/glog.cmake b/cmake/external/glog.cmake index 81d0e642f7..05b98e2b56 100644 --- a/cmake/external/glog.cmake +++ b/cmake/external/glog.cmake @@ -19,7 +19,7 @@ SET(GLOG_SOURCE_DIR ${THIRD_PARTY_PATH}/glog/src/extern_glog) SET(GLOG_INSTALL_DIR ${THIRD_PARTY_PATH}/install/glog) SET(GLOG_INCLUDE_DIR "${GLOG_INSTALL_DIR}/include" CACHE PATH "glog include directory." FORCE) SET(GLOG_REPOSITORY ${GIT_URL}/google/glog.git) -SET(GLOG_TAG v0.3.5) +SET(GLOG_TAG v0.4.0) IF(WIN32) SET(GLOG_LIBRARIES "${GLOG_INSTALL_DIR}/lib/glog.lib" CACHE FILEPATH "glog library." FORCE) @@ -45,7 +45,6 @@ ExternalProject_Add( DEPENDS gflags PREFIX ${GLOG_PREFIX_DIR} SOURCE_DIR ${GLOG_SOURCE_DIR} - UPDATE_COMMAND "" CMAKE_ARGS -DCMAKE_CXX_COMPILER=${CMAKE_CXX_COMPILER} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} -DCMAKE_CXX_FLAGS=${GLOG_CMAKE_CXX_FLAGS} @@ -57,8 +56,7 @@ ExternalProject_Add( -DCMAKE_INSTALL_PREFIX=${GLOG_INSTALL_DIR} -DCMAKE_INSTALL_LIBDIR=${GLOG_INSTALL_DIR}/lib -DCMAKE_POSITION_INDEPENDENT_CODE=ON - -DWITH_GFLAGS=ON - -Dgflags_DIR=${GFLAGS_INSTALL_DIR}/lib/cmake/gflags + -DWITH_GFLAGS=OFF -DBUILD_TESTING=OFF -DCMAKE_BUILD_TYPE=${THIRD_PARTY_BUILD_TYPE} ${EXTERNAL_OPTIONAL_ARGS} @@ -71,4 +69,4 @@ ExternalProject_Add( ADD_LIBRARY(glog STATIC IMPORTED GLOBAL) SET_PROPERTY(TARGET glog PROPERTY IMPORTED_LOCATION ${GLOG_LIBRARIES}) ADD_DEPENDENCIES(glog extern_glog gflags) -LINK_LIBRARIES(glog gflags) +LINK_LIBRARIES(glog) diff --git a/paddle/fluid/distributed/service/communicator.cc b/paddle/fluid/distributed/service/communicator.cc index 57b3636dee..6d26e65774 100644 --- a/paddle/fluid/distributed/service/communicator.cc +++ b/paddle/fluid/distributed/service/communicator.cc @@ -68,7 +68,7 @@ void Communicator::init_gflag(const std::string &gflags) { } int params_cnt = flags.size(); char **params_ptr = &(flags_ptr[0]); - ::google::ParseCommandLineFlags(¶ms_cnt, ¶ms_ptr, true); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags(¶ms_cnt, ¶ms_ptr, true); } std::once_flag Communicator::init_flag_; diff --git a/paddle/fluid/distributed/service/service.cc b/paddle/fluid/distributed/service/service.cc index 47b840cffd..d0e2585b60 100644 --- a/paddle/fluid/distributed/service/service.cc +++ b/paddle/fluid/distributed/service/service.cc @@ -21,7 +21,7 @@ #include "paddle/fluid/distributed/service/communicator.h" #include "paddle/fluid/string/string_helper.h" -using namespace std; +using namespace std; // NOLINT namespace paddle { namespace distributed { @@ -59,11 +59,11 @@ void PSCore::init_gflag(const std::string& gflags) { flags.insert(it, "exe default"); char* flags_ptr[flags.size()]; for (size_t i = 0; i < flags.size(); ++i) { - flags_ptr[i] = (char*)(flags[i].c_str()); + flags_ptr[i] = (char*)(flags[i].c_str()); // NOLINT } int params_cnt = flags.size(); char** params_ptr = &(flags_ptr[0]); - ::google::ParseCommandLineFlags(¶ms_cnt, ¶ms_ptr, true); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags(¶ms_cnt, ¶ms_ptr, true); } int PSCore::init_server( diff --git a/paddle/fluid/inference/api/api.cc b/paddle/fluid/inference/api/api.cc index 840541246a..f103eb7674 100644 --- a/paddle/fluid/inference/api/api.cc +++ b/paddle/fluid/inference/api/api.cc @@ -136,7 +136,7 @@ std::string UpdateDllFlag(const char *name, const char *value) { LOG(WARNING) << "The function \"UpdateDllFlag\" is only used to update the flag " "on the Windows shared library"; - ret = google::SetCommandLineOption(name, value); + ret = ::GFLAGS_NAMESPACE::SetCommandLineOption(name, value); PADDLE_ENFORCE_EQ( ret.empty(), false, diff --git a/paddle/fluid/inference/api/demo_ci/simple_on_word2vec.cc b/paddle/fluid/inference/api/demo_ci/simple_on_word2vec.cc index 166b84f282..611811954f 100644 --- a/paddle/fluid/inference/api/demo_ci/simple_on_word2vec.cc +++ b/paddle/fluid/inference/api/demo_ci/simple_on_word2vec.cc @@ -131,7 +131,7 @@ void MainThreads(int num_threads, bool use_gpu) { } // namespace paddle int main(int argc, char** argv) { - google::ParseCommandLineFlags(&argc, &argv, true); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); paddle::demo::Main(false /* use_gpu*/); paddle::demo::MainThreads(1, false /* use_gpu*/); paddle::demo::MainThreads(4, false /* use_gpu*/); diff --git a/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc b/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc index 4498a1bef2..9edb4ecbfd 100644 --- a/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc +++ b/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc @@ -73,7 +73,7 @@ void Main() { } // namespace paddle int main(int argc, char** argv) { - google::ParseCommandLineFlags(&argc, &argv, true); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); paddle::demo::Main(); return 0; } diff --git a/paddle/fluid/inference/api/demo_ci/vis_demo.cc b/paddle/fluid/inference/api/demo_ci/vis_demo.cc index 9dc1f56b69..293c90c202 100644 --- a/paddle/fluid/inference/api/demo_ci/vis_demo.cc +++ b/paddle/fluid/inference/api/demo_ci/vis_demo.cc @@ -82,7 +82,7 @@ void Main(bool use_gpu) { } // namespace paddle int main(int argc, char** argv) { - google::ParseCommandLineFlags(&argc, &argv, true); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); if (FLAGS_use_gpu) { paddle::demo::Main(true /*use_gpu*/); } else { diff --git a/paddle/fluid/inference/api/demo_ci/windows_mobilenet.cc b/paddle/fluid/inference/api/demo_ci/windows_mobilenet.cc index 58301c3448..8d0538f8fa 100644 --- a/paddle/fluid/inference/api/demo_ci/windows_mobilenet.cc +++ b/paddle/fluid/inference/api/demo_ci/windows_mobilenet.cc @@ -84,7 +84,7 @@ void RunAnalysis() { } // namespace paddle int main(int argc, char** argv) { - google::ParseCommandLineFlags(&argc, &argv, true); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); paddle::demo::RunAnalysis(); std::cout << "=========================Runs successfully====================" << std::endl; diff --git a/paddle/fluid/inference/check_symbol.sh b/paddle/fluid/inference/check_symbol.sh index 1d9b566e6c..acf7f2bac5 100755 --- a/paddle/fluid/inference/check_symbol.sh +++ b/paddle/fluid/inference/check_symbol.sh @@ -18,7 +18,7 @@ lib=$1 if [ $# -ne 1 ]; then echo "No input library"; exit -1 ; fi num_paddle_syms=$(nm -D "${lib}" | grep -c paddle ) -num_google_syms=$(nm -D "${lib}" | grep google | grep -v paddle | grep -v brpc | grep -c "T " ) +num_google_syms=$(nm -D "${lib}" | grep google | grep -v paddle | grep -v brpc | grep -v fL | grep -c "T " ) if [ $num_paddle_syms -le 0 ]; then echo "Have no paddle symbols"; exit -1 ; fi if [ $num_google_syms -ge 1 ]; then echo "Have some google symbols"; exit -1 ; fi diff --git a/paddle/fluid/inference/paddle_fluid.map b/paddle/fluid/inference/paddle_fluid.map index 5bb9b8d756..c1554a0088 100644 --- a/paddle/fluid/inference/paddle_fluid.map +++ b/paddle/fluid/inference/paddle_fluid.map @@ -3,6 +3,7 @@ *paddle*; *Pass*; *profile*; + *fL*; local: *; }; diff --git a/paddle/fluid/memory/detail/buddy_allocator.cc b/paddle/fluid/memory/detail/buddy_allocator.cc index 5b521e8968..3779571536 100644 --- a/paddle/fluid/memory/detail/buddy_allocator.cc +++ b/paddle/fluid/memory/detail/buddy_allocator.cc @@ -15,6 +15,7 @@ limitations under the License. */ #include "paddle/fluid/memory/detail/buddy_allocator.h" #include #include +#include "gflags/gflags.h" #include "glog/logging.h" #ifdef PADDLE_WITH_CUDA diff --git a/paddle/fluid/operators/jit/benchmark.cc b/paddle/fluid/operators/jit/benchmark.cc index d65cdc6c15..20df8a347f 100644 --- a/paddle/fluid/operators/jit/benchmark.cc +++ b/paddle/fluid/operators/jit/benchmark.cc @@ -567,7 +567,7 @@ BENCH_FP32_CPU(VBroadcast); // --max_size: the max size would be tested // --filter: the bench name would be run int main(int argc, char* argv[]) { - gflags::ParseCommandLineFlags(&argc, &argv, true); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); google::InitGoogleLogging(argv[0]); LOG(INFO) << "Burning " << FLAGS_burning << " times, Repeat " << FLAGS_repeat << " times."; diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h index b11a32e3ac..0b8a361abb 100644 --- a/paddle/fluid/platform/enforce.h +++ b/paddle/fluid/platform/enforce.h @@ -52,6 +52,7 @@ limitations under the License. */ #endif #define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h +#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/fluid/platform/errors.h" #include "paddle/fluid/platform/macros.h" diff --git a/paddle/fluid/platform/init.cc b/paddle/fluid/platform/init.cc index 4288dc66d6..3efdff2333 100644 --- a/paddle/fluid/platform/init.cc +++ b/paddle/fluid/platform/init.cc @@ -55,7 +55,7 @@ namespace paddle { namespace platform { void ParseCommandLineFlags(int argc, char **argv, bool remove) { - google::ParseCommandLineFlags(&argc, &argv, remove); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, remove); } } // namespace platform @@ -93,7 +93,7 @@ bool InitGflags(std::vector args) { << ", Init commandline: " << line; char **arr = argv.data(); - google::ParseCommandLineFlags(&argc, &arr, true); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &arr, true); successed = true; VLOG(1) << "After Parse: argc is " << argc; diff --git a/paddle/fluid/train/imdb_demo/demo_trainer.cc b/paddle/fluid/train/imdb_demo/demo_trainer.cc index 6272478dea..6d3b8e7ca4 100644 --- a/paddle/fluid/train/imdb_demo/demo_trainer.cc +++ b/paddle/fluid/train/imdb_demo/demo_trainer.cc @@ -80,7 +80,7 @@ bool IsPersistable(const paddle::framework::VarDesc* var) { } // namespace paddle int main(int argc, char* argv[]) { - gflags::ParseCommandLineFlags(&argc, &argv, true); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); std::cerr << "filelist: " << FLAGS_filelist << std::endl; std::cerr << "data_proto_desc: " << FLAGS_data_proto_desc << std::endl; diff --git a/paddle/testing/paddle_gtest_main.cc b/paddle/testing/paddle_gtest_main.cc index fab6eea49b..38ed76a87c 100644 --- a/paddle/testing/paddle_gtest_main.cc +++ b/paddle/testing/paddle_gtest_main.cc @@ -20,41 +20,10 @@ limitations under the License. */ int main(int argc, char** argv) { paddle::memory::allocation::UseAllocatorStrategyGFlag(); testing::InitGoogleTest(&argc, argv); - // Because the dynamic library libpaddle_fluid.so clips the symbol table, the - // external program cannot recognize the flag inside the so, and the flag - // defined by the external program cannot be accessed inside the so. - // Therefore, the ParseCommandLine function needs to be called separately - // inside and outside. - std::vector external_argv; - std::vector internal_argv; - - // ParseNewCommandLineFlags in gflags.cc starts processing - // commandline strings from idx 1. - // The reason is, it assumes that the first one (idx 0) is - // the filename of executable file. - external_argv.push_back(argv[0]); - internal_argv.push_back(argv[0]); - - std::vector all_flags; - std::vector external_flags_name; - google::GetAllFlags(&all_flags); - for (size_t i = 0; i < all_flags.size(); ++i) { - external_flags_name.push_back(all_flags[i].name); - } - + std::vector new_argv; + std::string gflags_env; for (int i = 0; i < argc; ++i) { - bool flag = true; - std::string tmp(argv[i]); - for (size_t j = 0; j < external_flags_name.size(); ++j) { - if (tmp.find(external_flags_name[j]) != std::string::npos) { - external_argv.push_back(argv[i]); - flag = false; - break; - } - } - if (flag) { - internal_argv.push_back(argv[i]); - } + new_argv.push_back(argv[i]); } std::vector envs; @@ -62,7 +31,8 @@ int main(int argc, char** argv) { #if defined(PADDLE_WITH_DISTRIBUTE) && !defined(PADDLE_WITH_GRPC) && \ !defined(PADDLE_WITH_PSLIB) std::string str_max_body_size; - if (google::GetCommandLineOption("max_body_size", &str_max_body_size)) { + if (::GFLAGS_NAMESPACE::GetCommandLineOption("max_body_size", + &str_max_body_size)) { setenv("FLAGS_max_body_size", "2147483647", 1); envs.push_back("max_body_size"); } @@ -99,7 +69,7 @@ int main(int argc, char** argv) { } env_string = env_string.substr(0, env_string.length() - 1); env_str = strdup(env_string.c_str()); - internal_argv.push_back(env_str); + new_argv.push_back(env_str); VLOG(1) << "gtest env_string:" << env_string; } @@ -111,17 +81,14 @@ int main(int argc, char** argv) { } undefok_string = undefok_string.substr(0, undefok_string.length() - 1); undefok_str = strdup(undefok_string.c_str()); - internal_argv.push_back(undefok_str); + new_argv.push_back(undefok_str); VLOG(1) << "gtest undefok_string:" << undefok_string; } - int new_argc = static_cast(external_argv.size()); - char** external_argv_address = external_argv.data(); - google::ParseCommandLineFlags(&new_argc, &external_argv_address, false); - - int internal_argc = internal_argv.size(); - char** arr = internal_argv.data(); - paddle::platform::ParseCommandLineFlags(internal_argc, arr, true); + int new_argc = static_cast(new_argv.size()); + char** new_argv_address = new_argv.data(); + ::GFLAGS_NAMESPACE::ParseCommandLineFlags( + &new_argc, &new_argv_address, false); paddle::framework::InitDevices(); int ret = RUN_ALL_TESTS(); -- GitLab