diff --git a/CMakeLists.txt b/CMakeLists.txt index 34e748c794f690b0037d2fdbccc8cfa43d6a0e21..318c9df489311bf206978e0172b4adb7d313fa4b 100755 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -254,6 +254,7 @@ option(WITH_BRPC_RDMA "Use brpc rdma as the rpc protocal" OFF) option(ON_INFER "Turn on inference optimization and inference-lib generation" ON) option(WITH_CPP_DIST "Install PaddlePaddle C++ distribution" OFF) +option(WITH_GFLAGS "Compile PaddlePaddle with gflags support" OFF) ################################ Internal Configurations ####################################### option(WITH_NV_JETSON "Compile PaddlePaddle with NV JETSON" OFF) option(WITH_PROFILER "Compile PaddlePaddle with GPU profiler and gperftools" diff --git a/cmake/cinn.cmake b/cmake/cinn.cmake index de13f71526c8a36e7e32cf995908a69137b1f47a..ca25a7d5d30a5655654f52cf98c5eea2838013a6 100644 --- a/cmake/cinn.cmake +++ b/cmake/cinn.cmake @@ -183,6 +183,11 @@ if(WITH_MKL) endif() endif() +if(NOT WITH_GFLAGS) + target_link_libraries(cinnapi gflags) + add_dependencies(cinnapi gflags) +endif() + if(WITH_GPU) target_link_libraries( cinnapi @@ -237,6 +242,11 @@ function(gen_cinncore LINKTYPE) endif() endif() + if(NOT WITH_GFLAGS) + target_link_libraries(${CINNCORE_TARGET} gflags) + add_dependencies(${CINNCORE_TARGET} gflags) + endif() + if(WITH_GPU) target_link_libraries( ${CINNCORE_TARGET} diff --git a/cmake/configure.cmake b/cmake/configure.cmake index dc661fce388fe182fdf60162814180ec8b5219c2..4d0b04209c059ba06fb0a988310bd3e0f345e09e 100644 --- a/cmake/configure.cmake +++ b/cmake/configure.cmake @@ -201,6 +201,10 @@ if(WITH_DISTRIBUTE) add_definitions(-DPADDLE_WITH_DISTRIBUTE) endif() +if(WITH_GFLAGS) + add_definitions(-DPADDLE_WITH_GFLAGS) +endif() + if(WITH_PSCORE) add_definitions(-DPADDLE_WITH_PSCORE) endif() diff --git a/cmake/external/brpc.cmake b/cmake/external/brpc.cmake index ff33e142addbd6a2df79e6a90c50ccdeb1fda332..3c9f2b696204836a7b29004b8bf4129504f39f2c 100755 --- a/cmake/external/brpc.cmake +++ b/cmake/external/brpc.cmake @@ -91,3 +91,16 @@ add_dependencies(brpc extern_brpc) add_definitions(-DBRPC_WITH_GLOG) list(APPEND external_project_dependencies brpc) + +set(EXTERNAL_BRPC_DEPS + brpc + protobuf + ssl + crypto + leveldb + glog + snappy) + +if(NOT WITH_GFLAGS) + set(EXTERNAL_BRPC_DEPS ${EXTERNAL_BRPC_DEPS} gflags) +endif() diff --git a/cmake/external/gflags.cmake b/cmake/external/gflags.cmake index 06827732e5966ad675bda369e13d33fc5616560e..75436783c7ede298f40f9f701cd5fd37af692d24 100755 --- a/cmake/external/gflags.cmake +++ b/cmake/external/gflags.cmake @@ -102,3 +102,14 @@ if(WIN32) set_property(GLOBAL PROPERTY OS_DEPENDENCY_MODULES shlwapi.lib) endif() endif() + +# We have implemented a custom flags tool paddle_flags to replace gflags. +# User can also choose to use gflags by setting WITH_GFLAGS=ON. But when +# using paddle_flags, gflags is also needed for other third party libraries +# including glog and brpc. So we can not remove gflags completely. +set(flags_dep) +if(WITH_GFLAGS) + list(APPEND flags_dep gflags) +else() + list(APPEND flags_dep paddle_flags) +endif() diff --git a/cmake/inference_lib.cmake b/cmake/inference_lib.cmake index c97a68a0175e1a06121251b534fa65a857b4463e..13fce9613650f14aed73ab1385ffcad0bfdf91e1 100755 --- a/cmake/inference_lib.cmake +++ b/cmake/inference_lib.cmake @@ -336,11 +336,22 @@ copy( inference_lib_dist SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/flat_hash_map.h DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/) +copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/flags.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/) copy( inference_lib_dist SRCS ${PADDLE_SOURCE_DIR}/paddle/extension.h DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/) +if(NOT WITH_GFLAGS) + copy( + inference_lib_dist + SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/flags_native.h + DSTS ${PADDLE_INFERENCE_INSTALL_DIR}/paddle/include/paddle/utils/) +endif() + # the include path of phi needs to be changed to adapt to inference api path add_custom_command( TARGET inference_lib_dist diff --git a/paddle/fluid/distributed/collective/process_group_custom.cc b/paddle/fluid/distributed/collective/process_group_custom.cc index 6d5c30da3133b577938fdb3a5822a2bea4ff87f0..a905fcc095599fdbcf7f0cee3e191720a6f79bf1 100644 --- a/paddle/fluid/distributed/collective/process_group_custom.cc +++ b/paddle/fluid/distributed/collective/process_group_custom.cc @@ -27,7 +27,7 @@ constexpr int64_t kWaitBlockTImeout = 10; -DECLARE_bool(use_stream_safe_cuda_allocator); +PD_DECLARE_bool(use_stream_safe_cuda_allocator); namespace paddle { namespace distributed { diff --git a/paddle/fluid/distributed/collective/process_group_nccl.cc b/paddle/fluid/distributed/collective/process_group_nccl.cc index a8539278d2f28076984d87df2101054956ba76b2..7ffe00b8cd824418520a5d772b77f8d7affb55e7 100644 --- a/paddle/fluid/distributed/collective/process_group_nccl.cc +++ b/paddle/fluid/distributed/collective/process_group_nccl.cc @@ -29,7 +29,7 @@ #include "paddle/phi/core/distributed/comm_context_manager.h" PHI_DECLARE_bool(nccl_blocking_wait); -DECLARE_bool(use_stream_safe_cuda_allocator); +PD_DECLARE_bool(use_stream_safe_cuda_allocator); // set this flag to `true` and recompile to enable dynamic checks constexpr bool FLAGS_enable_nccl_dynamic_check = false; diff --git a/paddle/fluid/distributed/collective/reducer.cc b/paddle/fluid/distributed/collective/reducer.cc index c3262b8db8d5688441ec6f9575da8683a0c8d5f1..378668499c90b6f2a736549a39fb3af7d7257c66 100644 --- a/paddle/fluid/distributed/collective/reducer.cc +++ b/paddle/fluid/distributed/collective/reducer.cc @@ -18,7 +18,7 @@ #include "paddle/phi/backends/device_manager.h" #include "paddle/phi/core/flags.h" -DECLARE_bool(use_stream_safe_cuda_allocator); +PD_DECLARE_bool(use_stream_safe_cuda_allocator); PHI_DECLARE_string(allocator_strategy); namespace paddle { diff --git a/paddle/fluid/distributed/fleet_executor/CMakeLists.txt b/paddle/fluid/distributed/fleet_executor/CMakeLists.txt index eea257b5ddf4fcdbfe1f6d83788a1e897947e5d1..9c28205520129c7a140ecfe629de27b2df4d1158 100755 --- a/paddle/fluid/distributed/fleet_executor/CMakeLists.txt +++ b/paddle/fluid/distributed/fleet_executor/CMakeLists.txt @@ -7,16 +7,7 @@ proto_library(interceptor_message_proto SRCS interceptor_message.proto) if(WITH_ARM_BRPC) set(BRPC_DEPS arm_brpc snappy phi glog) elseif(WITH_DISTRIBUTE AND NOT WITH_PSLIB) - set(BRPC_DEPS - brpc - ssl - crypto - protobuf - zlib - leveldb - snappy - phi - glog) + set(BRPC_DEPS ${EXTERNAL_BRPC_DEPS} zlib phi) else() set(BRPC_DEPS "") endif() diff --git a/paddle/fluid/distributed/fleet_executor/carrier.cc b/paddle/fluid/distributed/fleet_executor/carrier.cc index 1dc29493af9d8b8dd4f514f3dfdef3c2f1a96484..bb128c1287c9cf62d6e634bc3289a74f531b48e8 100644 --- a/paddle/fluid/distributed/fleet_executor/carrier.cc +++ b/paddle/fluid/distributed/fleet_executor/carrier.cc @@ -17,7 +17,6 @@ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/distributed/fleet_executor/global.h" #include "paddle/fluid/distributed/fleet_executor/interceptor.h" #include "paddle/fluid/distributed/fleet_executor/message_bus.h" @@ -29,6 +28,7 @@ #include "paddle/fluid/framework/variable.h" #include "paddle/fluid/framework/variable_helper.h" #include "paddle/fluid/platform/flags.h" +#include "paddle/utils/flags.h" PADDLE_DEFINE_EXPORTED_bool( fleet_executor_with_standalone, false, diff --git a/paddle/fluid/distributed/ps/service/CMakeLists.txt b/paddle/fluid/distributed/ps/service/CMakeLists.txt index 585dd111bf7d1194e99c2206ba9a85b46b37db41..c23f26c6352180a075f9a4541e9cf6109cbbe099 100755 --- a/paddle/fluid/distributed/ps/service/CMakeLists.txt +++ b/paddle/fluid/distributed/ps/service/CMakeLists.txt @@ -3,34 +3,11 @@ set_source_files_properties(${BRPC_SRCS}) if(WITH_HETERPS) - set(BRPC_DEPS - brpc - ssl - crypto - protobuf - phi - glog - zlib - leveldb - snappy - glog - device_context - rocksdb) + set(BRPC_DEPS ${EXTERNAL_BRPC_DEPS} phi zlib device_context rocksdb) else() - set(BRPC_DEPS - brpc - ssl - crypto - protobuf - phi - glog - zlib - leveldb - snappy - glog - device_context) + set(BRPC_DEPS ${EXTERNAL_BRPC_DEPS} phi zlib device_context) endif() diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_client.cc b/paddle/fluid/distributed/ps/service/brpc_ps_client.cc index 93fe8c849be69163fcd7ba180905363b161b157b..9ad8768e0927d530cdf80ae0d6c7bea83a17e3f0 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_client.cc +++ b/paddle/fluid/distributed/ps/service/brpc_ps_client.cc @@ -34,49 +34,53 @@ class Variable; namespace paddle { namespace distributed { -DEFINE_int32(pserver_push_dense_merge_limit, - 12, - "limit max push_dense local merge requests"); +PD_DEFINE_int32(pserver_push_dense_merge_limit, + 12, + "limit max push_dense local merge requests"); -DEFINE_int32(pserver_push_sparse_merge_limit, - 12, - "limit max push_sparse local merge requests"); +PD_DEFINE_int32(pserver_push_sparse_merge_limit, + 12, + "limit max push_sparse local merge requests"); -DEFINE_int32(pserver_pull_dense_limit, - 12, - "limit max push_sparse local merge requests"); +PD_DEFINE_int32(pserver_pull_dense_limit, + 12, + "limit max push_sparse local merge requests"); -DEFINE_int32(pserver_async_push_dense_interval_ms, - 10, - "async push_dense to server interval"); +PD_DEFINE_int32(pserver_async_push_dense_interval_ms, + 10, + "async push_dense to server interval"); -DEFINE_int32(pserver_async_push_sparse_interval_ms, - 10, - "async push_sparse to server interval"); +PD_DEFINE_int32(pserver_async_push_sparse_interval_ms, + 10, + "async push_sparse to server interval"); -DEFINE_bool(pserver_scale_gradient_by_merge, - false, - "scale dense gradient when merged"); +PD_DEFINE_bool(pserver_scale_gradient_by_merge, + false, + "scale dense gradient when merged"); -DEFINE_int32(pserver_communicate_compress_type, - 0, - "none:0 snappy:1 gzip:2 zlib:3 lz4:4"); +PD_DEFINE_int32(pserver_communicate_compress_type, + 0, + "none:0 snappy:1 gzip:2 zlib:3 lz4:4"); -DEFINE_int32(pserver_max_async_call_num, - 13, - "max task num in async_call_server"); +PD_DEFINE_int32(pserver_max_async_call_num, + 13, + "max task num in async_call_server"); -DEFINE_int32(pserver_timeout_ms, 500000, "pserver request server timeout_ms"); +PD_DEFINE_int32(pserver_timeout_ms, + 500000, + "pserver request server timeout_ms"); -DEFINE_int32(pserver_connect_timeout_ms, - 10000, - "pserver connect server timeout_ms"); +PD_DEFINE_int32(pserver_connect_timeout_ms, + 10000, + "pserver connect server timeout_ms"); -DEFINE_int32(pserver_sparse_merge_thread, 1, "pserver sparse merge thread num"); +PD_DEFINE_int32(pserver_sparse_merge_thread, + 1, + "pserver sparse merge thread num"); -DEFINE_int32(pserver_sparse_table_shard_num, - 1000, - "sparse table shard for save & load"); +PD_DEFINE_int32(pserver_sparse_table_shard_num, + 1000, + "sparse table shard for save & load"); inline size_t get_sparse_shard(uint32_t shard_num, uint32_t server_num, @@ -140,7 +144,7 @@ int32_t BrpcPsClient::StartFlClientService(const std::string &self_endpoint) { if (_fl_server.Start(self_endpoint.c_str(), &options) != 0) { VLOG(0) << "fl-ps > StartFlClientService failed. Try again."; - auto ip_port = paddle::string::Split(self_endpoint, ':'); + auto ip_port = ::paddle::string::Split(self_endpoint, ':'); std::string ip = ip_port[0]; int port = std::stoi(ip_port[1]); std::string int_ip_port = GetIntTypeEndpoint(ip, port); @@ -202,8 +206,7 @@ int32_t BrpcPsClient::InitializeFlWorker(const std::string &self_endpoint) { options.protocol = "baidu_std"; options.timeout_ms = FLAGS_pserver_timeout_ms; options.connection_type = "pooled"; - options.connect_timeout_ms = - paddle::distributed::FLAGS_pserver_connect_timeout_ms; + options.connect_timeout_ms = FLAGS_pserver_connect_timeout_ms; options.max_retry = 3; // 获取 coordinator 列表，并连接 std::string coordinator_ip_port; @@ -336,11 +339,11 @@ int32_t BrpcPsClient::Initialize() { auto table_id = worker_param.downpour_table_param(i).table_id(); if (type == PS_DENSE_TABLE) { _push_dense_task_queue_map[table_id] = - paddle::framework::MakeChannel(); + ::paddle::framework::MakeChannel(); } if (type == PS_SPARSE_TABLE) { _push_sparse_task_queue_map[table_id] = - paddle::framework::MakeChannel(); + ::paddle::framework::MakeChannel(); _push_sparse_merge_count_map[table_id] = 0; } } @@ -446,7 +449,7 @@ std::future BrpcPsClient::PrintTableStat(uint32_t table_id) { int ret = 0; uint64_t feasign_size = 0; uint64_t mf_size = 0; - paddle::framework::BinaryArchive ar; + ::paddle::framework::BinaryArchive ar; auto *closure = reinterpret_cast(done); for (size_t i = 0; i < request_call_num; ++i) { if (closure->check_response(i, PS_PRINT_TABLE_STAT) != 0) { diff --git a/paddle/fluid/distributed/ps/service/brpc_ps_server.cc b/paddle/fluid/distributed/ps/service/brpc_ps_server.cc index 84784c32f3b2d9eca44c51010905e9f163c3b01f..28ac123fa08ff7d426b149565db488670fccfa2f 100644 --- a/paddle/fluid/distributed/ps/service/brpc_ps_server.cc +++ b/paddle/fluid/distributed/ps/service/brpc_ps_server.cc @@ -30,15 +30,15 @@ class RpcController; } // namespace protobuf } // namespace google -DEFINE_int32(pserver_timeout_ms_s2s, - 10000, - "pserver request server timeout_ms"); -DEFINE_int32(pserver_connect_timeout_ms_s2s, - 10000, - "pserver connect server timeout_ms"); -DEFINE_string(pserver_connection_type_s2s, - "pooled", - "pserver connection_type[pooled:single]"); +PD_DEFINE_int32(pserver_timeout_ms_s2s, + 10000, + "pserver request server timeout_ms"); +PD_DEFINE_int32(pserver_connect_timeout_ms_s2s, + 10000, + "pserver connect server timeout_ms"); +PD_DEFINE_string(pserver_connection_type_s2s, + "pooled", + "pserver connection_type[pooled:single]"); namespace paddle { namespace distributed { @@ -169,7 +169,7 @@ int32_t BrpcPsServer::ReceiveFromPServer(int msg_type, LOG(WARNING) << "SERVER>>RESPONSE>>msg = 0 Finish S2S Response"; return 0; } - paddle::framework::BinaryArchive ar; + ::paddle::framework::BinaryArchive ar; ar.SetReadBuffer(const_cast(msg.c_str()), msg.length(), nullptr); if (ar.Cursor() == ar.Finish()) { LOG(WARNING) << "SERVER>>RESPONSE ar = 0>> Finish S2S Response"; @@ -598,7 +598,7 @@ int32_t BrpcPsService::PrintTableStat(Table *table, brpc::Controller *cntl) { CHECK_TABLE_EXIST(table, request, response) std::pair ret = table->PrintTableStat(); - paddle::framework::BinaryArchive ar; + ::paddle::framework::BinaryArchive ar; ar << ret.first << ret.second; std::string table_info(ar.Buffer(), ar.Length()); response.set_data(table_info); @@ -723,7 +723,7 @@ int32_t BrpcPsService::CacheShuffle(Table *table, table->Flush(); double cache_threshold = std::stod(request.params(2)); LOG(INFO) << "cache threshold for cache shuffle: " << cache_threshold; - // auto shuffled_ins = paddle::ps::make_channel>(); // shuffled_ins->set_block_size(80000); _server->StartS2S(); diff --git a/paddle/fluid/distributed/ps/service/communicator/communicator.cc b/paddle/fluid/distributed/ps/service/communicator/communicator.cc index 1ad58d9eb0fca039c5e2f1885d589853c8a24fd2..9932343fa779bd1e721f4158352628890c6e221c 100644 --- a/paddle/fluid/distributed/ps/service/communicator/communicator.cc +++ b/paddle/fluid/distributed/ps/service/communicator/communicator.cc @@ -16,11 +16,11 @@ limitations under the License. */ #include -#include "gflags/gflags.h" #include "paddle/fluid/distributed/ps/service/brpc_ps_client.h" #include "paddle/fluid/distributed/ps/wrapper/fleet.h" #include "paddle/fluid/platform/profiler.h" #include "paddle/fluid/string/string_helper.h" +#include "paddle/utils/flags.h" #define LEARNING_RATE_DECAY_COUNTER "@LR_DECAY_COUNTER@" #define STEP_COUNTER "@PS_STEP_COUNTER@" @@ -42,7 +42,7 @@ Communicator::Communicator() = default; void Communicator::InitGFlag(const std::string &gflags) { VLOG(3) << "Init With Gflags:" << gflags; - std::vector flags = paddle::string::split_string(gflags); + std::vector flags = ::paddle::string::split_string(gflags); if (flags.empty()) { flags.push_back("-max_body_size=314217728"); flags.push_back("-bthread_concurrency=40"); @@ -57,7 +57,7 @@ void Communicator::InitGFlag(const std::string &gflags) { } int params_cnt = flags.size(); char **params_ptr = &(flags_ptr[0]); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(¶ms_cnt, ¶ms_ptr, true); + ::paddle::flags::ParseCommandLineFlags(¶ms_cnt, ¶ms_ptr); } std::once_flag Communicator::init_flag_; @@ -66,7 +66,7 @@ std::shared_ptr Communicator::communicator_(nullptr); void Communicator::InitBrpcClient( const std::string &dist_desc, const std::vector &host_sign_list) { - auto fleet = paddle::distributed::FleetWrapper::GetInstance(); + auto fleet = ::paddle::distributed::FleetWrapper::GetInstance(); if (_worker_ptr.get() == nullptr) { _worker_ptr = fleet->worker_ptr_; } @@ -92,7 +92,7 @@ void Communicator::RpcRecvDense(const std::vector &varnames, platform::RecordEvent record_event("Communicator->RpcRecvDense", platform::TracerEventType::Communication, 1); - std::vector regions; + std::vector<::paddle::distributed::Region> regions; regions.reserve(varnames.size()); for (auto &t : varnames) { Variable *var = scope->Var(t); @@ -103,7 +103,7 @@ void Communicator::RpcRecvDense(const std::vector &varnames, phi::DenseTensor *temp_tensor = temp_var->GetMutable(); temp_tensor->Resize(tensor->dims()); float *temp_data = temp_tensor->mutable_data(platform::CPUPlace()); - paddle::distributed::Region reg(temp_data, tensor->numel()); + ::paddle::distributed::Region reg(temp_data, tensor->numel()); regions.emplace_back(std::move(reg)); VLOG(1) << "Communicator::RpcRecvDense Var " << t << " table_id " << table_id << " Temp_data[0] " << temp_data[0] @@ -111,7 +111,7 @@ void Communicator::RpcRecvDense(const std::vector &varnames, #endif } else { float *w = tensor->mutable_data(tensor->place()); - paddle::distributed::Region reg(w, tensor->numel()); + ::paddle::distributed::Region reg(w, tensor->numel()); regions.emplace_back(std::move(reg)); } } @@ -152,7 +152,7 @@ void Communicator::RpcSendDenseParam(const std::vector &varnames, platform::TracerEventType::Communication, 1); auto place = platform::CPUPlace(); - std::vector regions; + std::vector<::paddle::distributed::Region> regions; for (auto &t : varnames) { Variable *var = scope.FindVar(t); CHECK(var != nullptr) << "var[" << t << "] not found"; @@ -164,7 +164,7 @@ void Communicator::RpcSendDenseParam(const std::vector &varnames, temp_tensor->Resize(tensor->dims()); float *temp_data = temp_tensor->mutable_data(platform::CPUPlace()); framework::TensorCopy(*tensor, platform::CPUPlace(), temp_tensor); - paddle::distributed::Region reg(temp_data, tensor->numel()); + ::paddle::distributed::Region reg(temp_data, tensor->numel()); regions.emplace_back(std::move(reg)); VLOG(1) << "rpc_send_dense_param Var " << t << " table_id " << table_id << " Temp_data[0] " << temp_data[0] << " Temp_data[-1] " @@ -172,7 +172,7 @@ void Communicator::RpcSendDenseParam(const std::vector &varnames, #endif } else { float *w = tensor->mutable_data(place); - paddle::distributed::Region reg(w, tensor->numel()); + ::paddle::distributed::Region reg(w, tensor->numel()); regions.emplace_back(reg); VLOG(1) << "rpc_send_dense_param Var " << t << " table_id " << table_id << " Temp_data[0] " << w[0] << " Temp_data[-1] " @@ -1096,10 +1096,10 @@ void GeoCommunicator::InitImpl(const RpcCtxMap &send_varname_to_ctx, parallel_task_nums_ += 1; sparse_id_queues_.insert( std::pair>>>( splited_var, - paddle::framework::MakeChannel< + ::paddle::framework::MakeChannel< std::shared_ptr>>(send_queue_size_))); } } @@ -1509,7 +1509,7 @@ void GeoCommunicator::MainThread() { void FLCommunicator::InitBrpcClient( const std::string &dist_desc, const std::vector &host_sign_list) { - auto fleet = paddle::distributed::FleetWrapper::GetInstance(); + auto fleet = ::paddle::distributed::FleetWrapper::GetInstance(); if (_worker_ptr.get() == nullptr) { VLOG(0) << "fl-ps > FLCommunicator::InitBrpcClient get _worker_ptr"; _worker_ptr = diff --git a/paddle/fluid/distributed/ps/service/communicator/communicator.h b/paddle/fluid/distributed/ps/service/communicator/communicator.h index 643c91b5b05e940d77ed5cde21dc64a097d8cf35..f6d062460b826eccfa188dc02180a2dfc863c8c8 100644 --- a/paddle/fluid/distributed/ps/service/communicator/communicator.h +++ b/paddle/fluid/distributed/ps/service/communicator/communicator.h @@ -29,7 +29,6 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/distributed/ps/service/communicator/communicator_common.h" #include "paddle/fluid/distributed/ps/service/coordinator_client.h" #include "paddle/fluid/distributed/ps/service/ps_client.h" @@ -45,6 +44,7 @@ limitations under the License. */ #include "paddle/phi/kernels/funcs/blas/blas.h" #include "paddle/phi/kernels/funcs/math_function.h" #include "paddle/phi/kernels/funcs/selected_rows_functor.h" +#include "paddle/utils/flags.h" namespace paddle { namespace distributed { @@ -410,8 +410,8 @@ class Communicator { } void InitGFlag(const std::string &gflags); - paddle::distributed::PSParameter _ps_param; - paddle::distributed::PaddlePSEnvironment _ps_env; + ::paddle::distributed::PSParameter _ps_param; + ::paddle::distributed::PaddlePSEnvironment _ps_env; int servers_ = 0; int trainers_; int trainer_id_ = 0; @@ -661,7 +661,7 @@ class GeoCommunicator : public AsyncCommunicator { std::unordered_map< std::string, - paddle::framework::Channel>>> + ::paddle::framework::Channel>>> sparse_id_queues_; }; diff --git a/paddle/fluid/distributed/ps/service/coordinator_client.cc b/paddle/fluid/distributed/ps/service/coordinator_client.cc index 9a77170b37c53cce9b475758112783f32d0952a4..c9c2ba49c9bf398c67c502bb4bdcb6ef9bcd9ce0 100644 --- a/paddle/fluid/distributed/ps/service/coordinator_client.cc +++ b/paddle/fluid/distributed/ps/service/coordinator_client.cc @@ -28,8 +28,8 @@ static const int MAX_PORT = 65535; namespace paddle { namespace distributed { -DEFINE_uint64(total_fl_client_size, 100, "supported total fl client size"); -DEFINE_uint32(coordinator_wait_all_clients_max_time, 60, "uint32: s"); +PD_DEFINE_uint64(total_fl_client_size, 100, "supported total fl client size"); +PD_DEFINE_uint32(coordinator_wait_all_clients_max_time, 60, "uint32: s"); void CoordinatorService::FLService( ::google::protobuf::RpcController* controller, @@ -62,10 +62,10 @@ int32_t CoordinatorClient::Initialize( const std::vector& trainer_endpoints) { brpc::ChannelOptions options; options.protocol = "baidu_std"; - options.timeout_ms = paddle::distributed::FLAGS_pserver_timeout_ms; + options.timeout_ms = ::paddle::distributed::FLAGS_pserver_timeout_ms; options.connection_type = "pooled"; options.connect_timeout_ms = - paddle::distributed::FLAGS_pserver_connect_timeout_ms; + ::paddle::distributed::FLAGS_pserver_connect_timeout_ms; options.max_retry = 3; std::string server_ip_port; @@ -109,7 +109,7 @@ int32_t CoordinatorClient::Initialize( } for (size_t i = 0; i < trainer_endpoints.size(); i++) { std::vector addr = - paddle::string::Split(trainer_endpoints[i], ':'); + ::paddle::string::Split(trainer_endpoints[i], ':'); fl_client_list[i].ip = addr[0]; fl_client_list[i].port = std::stol(addr[1]); fl_client_list[i].rank = i; // TO CHECK @@ -152,7 +152,7 @@ int32_t CoordinatorClient::StartClientService() { LOG(ERROR) << "fl-ps > coordinator server endpoint not set"; return -1; } - auto addr = paddle::string::Split(_endpoint, ':'); + auto addr = ::paddle::string::Split(_endpoint, ':'); std::string ip = addr[0]; std::string port = addr[1]; std::string rank = addr[2]; diff --git a/paddle/fluid/distributed/ps/service/coordinator_client.h b/paddle/fluid/distributed/ps/service/coordinator_client.h index bd1f0f7754d8c8e8c99972063178cd48ede03b5a..8db08c3fc7999479950dc2c919bb182d892de099 100644 --- a/paddle/fluid/distributed/ps/service/coordinator_client.h +++ b/paddle/fluid/distributed/ps/service/coordinator_client.h @@ -34,10 +34,10 @@ namespace paddle { namespace distributed { -DECLARE_int32(pserver_timeout_ms); -DECLARE_int32(pserver_connect_timeout_ms); -DECLARE_uint64(total_fl_client_size); -DECLARE_uint32(coordinator_wait_all_clients_max_time); +PD_DECLARE_int32(pserver_timeout_ms); +PD_DECLARE_int32(pserver_connect_timeout_ms); +PD_DECLARE_uint64(total_fl_client_size); +PD_DECLARE_uint32(coordinator_wait_all_clients_max_time); using CoordinatorServiceFunc = std::function bool { while (query_wait_time < - paddle::distributed:: - FLAGS_coordinator_wait_all_clients_max_time) { // in case that - // some - // clients down + FLAGS_coordinator_wait_all_clients_max_time) { // in case that + // some + // clients down if (_is_all_clients_info_collected == true) { // LOG(INFO) << "fl-ps > _is_all_clients_info_collected"; return true; diff --git a/paddle/fluid/distributed/ps/service/env.h b/paddle/fluid/distributed/ps/service/env.h index aa230f86c9d1720ad4a73df1236ce44010420a4a..d6b403523496c580a10acf7a3be05a9cfb025e98 100644 --- a/paddle/fluid/distributed/ps/service/env.h +++ b/paddle/fluid/distributed/ps/service/env.h @@ -25,8 +25,8 @@ #include #include -#include "gflags/gflags.h" #include "paddle/phi/core/macros.h" +#include "paddle/utils/flags.h" namespace paddle { namespace distributed { diff --git a/paddle/fluid/distributed/ps/service/heter_client.cc b/paddle/fluid/distributed/ps/service/heter_client.cc index 2ca9fef5c0876566d7277e369e504d07bb9b996c..d7a1f5cf7c4f13d905456495772f9c06a1e60c90 100644 --- a/paddle/fluid/distributed/ps/service/heter_client.cc +++ b/paddle/fluid/distributed/ps/service/heter_client.cc @@ -19,8 +19,8 @@ namespace paddle { namespace distributed { -DEFINE_int32(heter_world_size, 100, "group size"); // group max size -DEFINE_int32(switch_send_recv_timeout_s, 600, "switch_send_recv_timeout_s"); +PD_DEFINE_int32(heter_world_size, 100, "group size"); // group max size +PD_DEFINE_int32(switch_send_recv_timeout_s, 600, "switch_send_recv_timeout_s"); std::shared_ptr HeterClient::s_instance_ = nullptr; std::mutex HeterClient::mtx_; @@ -85,7 +85,7 @@ void HeterClient::CreateClient2XpuConnection() { xpu_channels_[i].reset(new brpc::Channel()); if (xpu_channels_[i]->Init(xpu_list_[i].c_str(), "", &options) != 0) { VLOG(0) << "HeterClient channel init fail. Try Again"; - auto ip_port = paddle::string::Split(xpu_list_[i], ':'); + auto ip_port = ::paddle::string::Split(xpu_list_[i], ':'); std::string ip = ip_port[0]; int port = std::stoi(ip_port[1]); std::string int_ip_port = GetIntTypeEndpoint(ip, port); @@ -100,7 +100,7 @@ void HeterClient::CreateClient2XpuConnection() { if (previous_xpu_channels_[i]->Init( previous_xpu_list_[i].c_str(), "", &options) != 0) { VLOG(0) << "HeterClient channel init fail. Try Again"; - auto ip_port = paddle::string::Split(previous_xpu_list_[i], ':'); + auto ip_port = ::paddle::string::Split(previous_xpu_list_[i], ':'); std::string ip = ip_port[0]; int port = std::stoi(ip_port[1]); std::string int_ip_port = GetIntTypeEndpoint(ip, port); @@ -181,11 +181,11 @@ void HeterClient::SendAndRecvAsync( std::future HeterClient::SendCmd( uint32_t table_id, int cmd_id, const std::vector& params) { size_t request_call_num = xpu_channels_.size(); - paddle::distributed::DownpourBrpcClosure* closure = - new paddle::distributed::DownpourBrpcClosure( + ::paddle::distributed::DownpourBrpcClosure* closure = + new ::paddle::distributed::DownpourBrpcClosure( request_call_num, [request_call_num, cmd_id](void* done) { int ret = 0; - auto* closure = (paddle::distributed::DownpourBrpcClosure*)done; + auto* closure = (::paddle::distributed::DownpourBrpcClosure*)done; for (size_t i = 0; i < request_call_num; ++i) { if (closure->check_response(i, cmd_id) != 0) { ret = -1; diff --git a/paddle/fluid/distributed/ps/service/heter_client.h b/paddle/fluid/distributed/ps/service/heter_client.h index 10d916b71001186f1a2d91b0f4b74f3a17a61d26..e6c231338ac52912c109553f3ffcb691a21ac8f3 100755 --- a/paddle/fluid/distributed/ps/service/heter_client.h +++ b/paddle/fluid/distributed/ps/service/heter_client.h @@ -42,7 +42,7 @@ class Scope; namespace paddle { namespace distributed { -DECLARE_int32(pserver_timeout_ms); +PD_DECLARE_int32(pserver_timeout_ms); using MultiVarMsg = ::paddle::distributed::MultiVariableMessage; using VarMsg = ::paddle::distributed::VariableMessage; @@ -116,7 +116,7 @@ class HeterClient { if ((*client_channels)[i]->Init(node_list[i].c_str(), "", &options) != 0) { VLOG(0) << "client channel init failed! try again"; - auto ip_port = paddle::string::Split(node_list[i], ':'); + auto ip_port = ::paddle::string::Split(node_list[i], ':'); std::string ip = ip_port[0]; int port = std::stoi(ip_port[1]); std::string int_ip_port = GetIntTypeEndpoint(ip, port); diff --git a/paddle/fluid/distributed/ps/service/heter_server.cc b/paddle/fluid/distributed/ps/service/heter_server.cc index d5d8546bf792e995f4e88130b0b9b87dd39648b5..eb4d9b8304513967b4545a2ab9186559f29b91e3 100644 --- a/paddle/fluid/distributed/ps/service/heter_server.cc +++ b/paddle/fluid/distributed/ps/service/heter_server.cc @@ -18,8 +18,8 @@ namespace paddle { namespace distributed { -// DEFINE_string(cert_path, "./cert.pem", "cert.pem path"); -// DEFINE_string(key_path, "./key.pem", "key.pem path"); +// PD_DEFINE_string(cert_path, "./cert.pem", "cert.pem path"); +// PD_DEFINE_string(key_path, "./key.pem", "key.pem path"); std::shared_ptr HeterServer::s_instance_ = nullptr; std::mutex HeterServer::mtx_; @@ -37,7 +37,7 @@ void HeterServer::StartHeterService(bool neeed_encrypt) { } if (server_.Start(endpoint_.c_str(), &options) != 0) { VLOG(0) << "HeterServer start fail. Try again."; - auto ip_port = paddle::string::Split(endpoint_, ':'); + auto ip_port = ::paddle::string::Split(endpoint_, ':'); std::string ip = ip_port[0]; int port = std::stoi(ip_port[1]); std::string int_ip_port = GetIntTypeEndpoint(ip, port); @@ -72,7 +72,7 @@ void HeterServer::StartHeterInterService(bool neeed_encrypt) { } if (server_inter_.Start(endpoint_inter_.c_str(), &options) != 0) { VLOG(4) << "switch inter server start fail. Try again."; - auto ip_port = paddle::string::Split(endpoint_inter_, ':'); + auto ip_port = ::paddle::string::Split(endpoint_inter_, ':'); std::string ip = ip_port[0]; int port = std::stoi(ip_port[1]); std::string int_ip_port = GetIntTypeEndpoint(ip, port); diff --git a/paddle/fluid/distributed/ps/service/heter_server.h b/paddle/fluid/distributed/ps/service/heter_server.h index c4e9d05ac13520b5ff74af6416e8377a8b0cfd25..1f134d4db18aed67ab328dd7b60d61b347aafaf3 100755 --- a/paddle/fluid/distributed/ps/service/heter_server.h +++ b/paddle/fluid/distributed/ps/service/heter_server.h @@ -57,9 +57,9 @@ PHI_DECLARE_double(eager_delete_tensor_gb); namespace paddle { namespace distributed { -DECLARE_int32(pserver_timeout_ms); -DECLARE_int32(heter_world_size); -DECLARE_int32(switch_send_recv_timeout_s); +PD_DECLARE_int32(pserver_timeout_ms); +PD_DECLARE_int32(heter_world_size); +PD_DECLARE_int32(switch_send_recv_timeout_s); using MultiVarMsg = MultiVariableMessage; using VarMsg = VariableMessage; @@ -216,8 +216,8 @@ class SendAndRecvVariableHandler final : public ServiceHandlerBase { // get microID from request // deserialize variable to micro scope // Push to heter worker's task_queue - std::unique_ptr local_scope_ptr( - new paddle::framework::Scope()); + std::unique_ptr<::paddle::framework::Scope> local_scope_ptr( + new ::paddle::framework::Scope()); auto& local_scope = *(local_scope_ptr.get()); platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); platform::CPUPlace cpu_place; @@ -257,7 +257,7 @@ class SendAndRecvVariableHandler final : public ServiceHandlerBase { auto* minibatch_scope = &(scope_->NewScope()); (*mini_scopes_)[minibatch_index] = minibatch_scope; (*micro_scopes_)[minibatch_index].reset( - new std::vector{}); + new std::vector<::paddle::framework::Scope*>{}); for (int i = 0; i < num_microbatch_; i++) { auto* micro_scope = &(minibatch_scope->NewScope()); (*((*micro_scopes_)[minibatch_index])).push_back(micro_scope); @@ -300,7 +300,7 @@ class SendAndRecvVariableHandler final : public ServiceHandlerBase { public: using shard_type = SparseTableShard; - std::shared_ptr local_scope_ptr; // for switch + std::shared_ptr<::paddle::framework::Scope> local_scope_ptr; // for switch std::unordered_map> vars_ready_flag; std::unique_ptr _local_shards; @@ -344,7 +344,7 @@ class HeterService : public PsService { std::placeholders::_3); service_handler_.local_scope_ptr = - std::make_shared(); + std::make_shared<::paddle::framework::Scope>(); } virtual ~HeterService() {} @@ -613,7 +613,7 @@ class HeterServer { void SetLocalScope() { request_handler_->local_scope_ptr = - std::make_shared(); + std::make_shared<::paddle::framework::Scope>(); } void SetInterEndpoint(const std::string& endpoint) { diff --git a/paddle/fluid/distributed/ps/service/ps_client.cc b/paddle/fluid/distributed/ps/service/ps_client.cc index 3dd2b8c775cbee1f46afb441884aaa7509d6dde1..ecdcd8b874377bebc730cfe68b977398f4f15937 100644 --- a/paddle/fluid/distributed/ps/service/ps_client.cc +++ b/paddle/fluid/distributed/ps/service/ps_client.cc @@ -37,7 +37,8 @@ REGISTER_PSCORE_CLASS(PSClient, PsGraphClient); int32_t PSClient::Configure( // called in FleetWrapper::InitWorker const PSParameter &config, - const std::map> ®ions, + const std::map> + ®ions, PSEnvironment &env, size_t client_id) { _env = &env; @@ -88,7 +89,7 @@ PSClient *PSClientFactory::Create(const PSParameter &ps_config) { PSClient *client = NULL; #if defined(PADDLE_WITH_GLOO) && defined(PADDLE_WITH_GPU_GRAPH) - auto gloo = paddle::framework::GlooWrapper::GetInstance(); + auto gloo = ::paddle::framework::GlooWrapper::GetInstance(); if (client_name == "PsLocalClient" && gloo->Size() > 1) { client = CREATE_PSCORE_CLASS(PSClient, "PsGraphClient"); LOG(WARNING) << "change PsLocalClient to PsGraphClient"; diff --git a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h index 33490281981d54f52996b279bf4fa0762b5343cc..44836e7661b5f122da8f0f0e7fe46fd6a82cc118 100644 --- a/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h +++ b/paddle/fluid/distributed/ps/service/ps_service/graph_py_service.h @@ -143,13 +143,13 @@ class GraphPyServer : public GraphPyService { void start_server(bool block = true); ::paddle::distributed::PSParameter GetServerProto(); - std::shared_ptr get_ps_server() { + std::shared_ptr<::paddle::distributed::GraphBrpcServer> get_ps_server() { return pserver_ptr; } protected: int rank; - std::shared_ptr pserver_ptr; + std::shared_ptr<::paddle::distributed::GraphBrpcServer> pserver_ptr; std::thread* server_thread; }; class GraphPyClient : public GraphPyService { @@ -162,14 +162,14 @@ class GraphPyClient : public GraphPyService { set_client_id(client_id); GraphPyService::set_up(ips_str, shard_num, node_types, edge_types); } - std::shared_ptr get_ps_client() { + std::shared_ptr<::paddle::distributed::GraphBrpcClient> get_ps_client() { return worker_ptr; } void bind_local_server(int local_channel_index, GraphPyServer& server) { // NOLINT worker_ptr->set_local_channel(local_channel_index); worker_ptr->set_local_graph_service( - (paddle::distributed::GraphBrpcService*)server.get_ps_server() + (::paddle::distributed::GraphBrpcService*)server.get_ps_server() ->get_service()); } void StopServer(); @@ -209,7 +209,7 @@ class GraphPyClient : public GraphPyService { protected: mutable std::mutex mutex_; int client_id; - std::shared_ptr worker_ptr; + std::shared_ptr<::paddle::distributed::GraphBrpcClient> worker_ptr; std::thread* client_thread; bool stoped_ = false; }; diff --git a/paddle/fluid/distributed/ps/service/ps_service/service.cc b/paddle/fluid/distributed/ps/service/ps_service/service.cc index d62cdb4c133eda7c770c5d559a63b0f101570a07..e66475e88d87550ebcbda8320678ab55d2c865c2 100644 --- a/paddle/fluid/distributed/ps/service/ps_service/service.cc +++ b/paddle/fluid/distributed/ps/service/ps_service/service.cc @@ -28,9 +28,9 @@ using namespace std; // NOLINT namespace paddle { namespace distributed { -paddle::distributed::PSParameter load_from_prototxt( +::paddle::distributed::PSParameter load_from_prototxt( const std::string& filename) { - paddle::distributed::PSParameter param; + ::paddle::distributed::PSParameter param; int file_descriptor = open(filename.c_str(), O_RDONLY); if (file_descriptor == -1) { @@ -50,7 +50,7 @@ paddle::distributed::PSParameter load_from_prototxt( void PSCore::InitGFlag(const std::string& gflags) { VLOG(3) << "Init With Gflags:" << gflags; - std::vector flags = paddle::string::split_string(gflags); + std::vector flags = ::paddle::string::split_string(gflags); if (flags.empty()) { flags.push_back("-max_body_size=314217728"); flags.push_back("-socket_max_unwritten_bytes=2048000000"); @@ -64,7 +64,7 @@ void PSCore::InitGFlag(const std::string& gflags) { } int params_cnt = flags.size(); char** params_ptr = &(flags_ptr[0]); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(¶ms_cnt, ¶ms_ptr, true); + ::paddle::flags::ParseCommandLineFlags(¶ms_cnt, ¶ms_ptr); } int PSCore::InitServer( @@ -76,12 +76,12 @@ int PSCore::InitServer( const std::vector& server_sub_program) { google::protobuf::TextFormat::ParseFromString(dist_desc, &_ps_param); InitGFlag(_ps_param.init_gflags()); - _ps_env = paddle::distributed::PaddlePSEnvironment(); + _ps_env = ::paddle::distributed::PaddlePSEnvironment(); _ps_env.SetPsServers(host_sign_list, node_num); _ps_env.SetTrainers(trainers); int ret = 0; - _server_ptr = std::shared_ptr( - paddle::distributed::PSServerFactory::Create(_ps_param)); + _server_ptr = std::shared_ptr<::paddle::distributed::PSServer>( + ::paddle::distributed::PSServerFactory::Create(_ps_param)); ret = _server_ptr->Configure(_ps_param, _ps_env, index, server_sub_program); CHECK(ret == 0) << "failed to configure server"; return ret; @@ -89,13 +89,14 @@ int PSCore::InitServer( int PSCore::InitWorker( const std::string& dist_desc, - const std::map>& regions, + const std::map>& + regions, const std::vector* host_sign_list, int node_num, int index) { google::protobuf::TextFormat::ParseFromString(dist_desc, &_ps_param); InitGFlag(_ps_param.init_gflags()); - _ps_env = paddle::distributed::PaddlePSEnvironment(); + _ps_env = ::paddle::distributed::PaddlePSEnvironment(); _ps_env.SetPsServers(host_sign_list, node_num); int ret = 0; VLOG(1) << "PSCore::InitWorker"; @@ -132,6 +133,6 @@ int PSCore::StopServer() { stop_status.wait(); return 0; } -paddle::distributed::PSParameter* PSCore::GetParam() { return &_ps_param; } +::paddle::distributed::PSParameter* PSCore::GetParam() { return &_ps_param; } } // namespace distributed } // namespace paddle diff --git a/paddle/fluid/distributed/ps/service/ps_service/service.h b/paddle/fluid/distributed/ps/service/ps_service/service.h index eb190073fbd834a92793f6a6f99016d0fa916d46..4c3c6db61e2bc478f7fe743fb7e2f63587eac6b1 100644 --- a/paddle/fluid/distributed/ps/service/ps_service/service.h +++ b/paddle/fluid/distributed/ps/service/ps_service/service.h @@ -33,9 +33,9 @@ class PsRequestMessage; class PsResponseMessage; class PsService; -using paddle::distributed::PsRequestMessage; -using paddle::distributed::PsResponseMessage; -using paddle::distributed::PsService; +using ::paddle::distributed::PsRequestMessage; +using ::paddle::distributed::PsResponseMessage; +using ::paddle::distributed::PsService; class PSCore { public: @@ -51,7 +51,7 @@ class PSCore { const std::vector& server_sub_program = {}); virtual int InitWorker( const std::string& dist_desc, - const std::map>& + const std::map>& regions, const std::vector* host_sign_list, int node_num, @@ -63,16 +63,16 @@ class PSCore { virtual int CreateClient2ClientConnection(int pserver_timeout_ms, int pserver_connect_timeout_ms, int max_retry); - std::shared_ptr + std::shared_ptr<::paddle::distributed::PSServer> _server_ptr; // pointer to server - std::shared_ptr + std::shared_ptr<::paddle::distributed::PSClient> _worker_ptr; // pointer to worker - virtual paddle::distributed::PSParameter* GetParam(); + virtual ::paddle::distributed::PSParameter* GetParam(); private: void InitGFlag(const std::string& gflags); - paddle::distributed::PSParameter _ps_param; - paddle::distributed::PaddlePSEnvironment _ps_env; + ::paddle::distributed::PSParameter _ps_param; + ::paddle::distributed::PaddlePSEnvironment _ps_env; }; } // namespace distributed diff --git a/paddle/fluid/distributed/ps/service/server.h b/paddle/fluid/distributed/ps/service/server.h index 48b32d22cac799462de25590b7cddd0981a7b41e..fc1d4a2bd343ba8718133b827e8336ae701e7f3e 100644 --- a/paddle/fluid/distributed/ps/service/server.h +++ b/paddle/fluid/distributed/ps/service/server.h @@ -57,8 +57,8 @@ namespace distributed { class Table; -using paddle::distributed::PsRequestMessage; -using paddle::distributed::PsResponseMessage; +using ::paddle::distributed::PsRequestMessage; +using ::paddle::distributed::PsResponseMessage; class PSServer { public: @@ -134,7 +134,7 @@ class PSServer { return -1; } - paddle::framework::Channel> _shuffled_ins; + ::paddle::framework::Channel> _shuffled_ins; protected: virtual int32_t Initialize() = 0; diff --git a/paddle/fluid/distributed/ps/service/simple_rpc/baidu_rpc_server.cc b/paddle/fluid/distributed/ps/service/simple_rpc/baidu_rpc_server.cc index f3e501dd00ce1b127955d984a997c3334a51466d..a10e78fe94162938836e8c4b98da868e450b1d20 100644 --- a/paddle/fluid/distributed/ps/service/simple_rpc/baidu_rpc_server.cc +++ b/paddle/fluid/distributed/ps/service/simple_rpc/baidu_rpc_server.cc @@ -19,8 +19,8 @@ #include "paddle/phi/core/enforce.h" namespace brpc { -DECLARE_uint64(max_body_size); -DECLARE_int64(socket_max_unwritten_bytes); +PD_DECLARE_uint64(max_body_size); +PD_DECLARE_int64(socket_max_unwritten_bytes); } // namespace brpc namespace paddle { diff --git a/paddle/fluid/distributed/ps/table/common_graph_table.cc b/paddle/fluid/distributed/ps/table/common_graph_table.cc index 316b2295c3389fa44d792efb6a9007dccb32a6e0..153c67317d54dba2ff65a6333aa12b0b53631dff 100644 --- a/paddle/fluid/distributed/ps/table/common_graph_table.cc +++ b/paddle/fluid/distributed/ps/table/common_graph_table.cc @@ -58,14 +58,14 @@ int32_t GraphTable::Load_to_ssd(const std::string &path, return 0; } -paddle::framework::GpuPsCommGraphFea GraphTable::make_gpu_ps_graph_fea( +::paddle::framework::GpuPsCommGraphFea GraphTable::make_gpu_ps_graph_fea( int gpu_id, std::vector &node_ids, int slot_num) { size_t shard_num = 64; std::vector> bags(shard_num); std::vector feature_array[shard_num]; std::vector slot_id_array[shard_num]; std::vector node_id_array[shard_num]; - std::vector node_fea_info_array[shard_num]; + std::vector<::paddle::framework::GpuPsFeaInfo> node_fea_info_array[shard_num]; for (size_t i = 0; i < shard_num; i++) { auto predsize = node_ids.size() / shard_num; bags[i].reserve(predsize * 1.2); @@ -92,7 +92,7 @@ paddle::framework::GpuPsCommGraphFea GraphTable::make_gpu_ps_graph_fea( if (bags[i].size() > 0) { tasks.push_back(_cpu_worker_pool[gpu_id]->enqueue([&, i, this]() -> int { uint64_t node_id; - paddle::framework::GpuPsFeaInfo x; + ::paddle::framework::GpuPsFeaInfo x; std::vector feature_ids; for (size_t j = 0; j < bags[i].size(); j++) { Node *v = find_node(GraphTableType::FEATURE_TABLE, bags[i][j]); @@ -134,7 +134,7 @@ paddle::framework::GpuPsCommGraphFea GraphTable::make_gpu_ps_graph_fea( tasks.clear(); - paddle::framework::GpuPsCommGraphFea res; + ::paddle::framework::GpuPsCommGraphFea res; uint64_t tot_len = 0; for (size_t i = 0; i < shard_num; i++) { tot_len += feature_array[i].size(); @@ -165,7 +165,7 @@ paddle::framework::GpuPsCommGraphFea GraphTable::make_gpu_ps_graph_fea( return res; } -paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( +::paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( int idx, const std::vector &ids) { std::vector> bags(task_pool_size_); for (int i = 0; i < task_pool_size_; i++) { @@ -179,7 +179,7 @@ paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( std::vector> tasks; std::vector node_array[task_pool_size_]; // node id list - std::vector info_array[task_pool_size_]; + std::vector<::paddle::framework::GpuPsNodeInfo> info_array[task_pool_size_]; std::vector edge_array[task_pool_size_]; // edge id list for (size_t i = 0; i < bags.size(); i++) { @@ -215,7 +215,7 @@ paddle::framework::GpuPsCommGraph GraphTable::make_gpu_ps_graph( tot_len += edge_array[i].size(); } - paddle::framework::GpuPsCommGraph res; + ::paddle::framework::GpuPsCommGraph res; res.init_on_cpu(tot_len, ids.size()); int64_t offset = 0, ind = 0; for (int i = 0; i < task_pool_size_; i++) { @@ -516,13 +516,13 @@ void GraphTable::release_graph() { build_graph_type_keys(); if (FLAGS_gpugraph_storage_mode == - paddle::framework::GpuGraphStorageMode::WHOLE_HBM) { + ::paddle::framework::GpuGraphStorageMode::WHOLE_HBM) { build_graph_total_keys(); } // clear graph - if (FLAGS_gpugraph_storage_mode == paddle::framework::GpuGraphStorageMode:: + if (FLAGS_gpugraph_storage_mode == ::paddle::framework::GpuGraphStorageMode:: MEM_EMB_FEATURE_AND_GPU_GRAPH || - FLAGS_gpugraph_storage_mode == paddle::framework::GpuGraphStorageMode:: + FLAGS_gpugraph_storage_mode == ::paddle::framework::GpuGraphStorageMode:: SSD_EMB_AND_MEM_FEATURE_GPU_GRAPH) { clear_edge_shard(); } else { @@ -532,7 +532,7 @@ void GraphTable::release_graph() { void GraphTable::release_graph_edge() { if (FLAGS_gpugraph_storage_mode == - paddle::framework::GpuGraphStorageMode::WHOLE_HBM) { + ::paddle::framework::GpuGraphStorageMode::WHOLE_HBM) { build_graph_total_keys(); } clear_edge_shard(); @@ -543,10 +543,12 @@ void GraphTable::release_graph_node() { if (FLAGS_graph_metapath_split_opt) { clear_feature_shard(); } else { - if (FLAGS_gpugraph_storage_mode != paddle::framework::GpuGraphStorageMode:: - MEM_EMB_FEATURE_AND_GPU_GRAPH && - FLAGS_gpugraph_storage_mode != paddle::framework::GpuGraphStorageMode:: - SSD_EMB_AND_MEM_FEATURE_GPU_GRAPH) { + if (FLAGS_gpugraph_storage_mode != + ::paddle::framework::GpuGraphStorageMode:: + MEM_EMB_FEATURE_AND_GPU_GRAPH && + FLAGS_gpugraph_storage_mode != + ::paddle::framework::GpuGraphStorageMode:: + SSD_EMB_AND_MEM_FEATURE_GPU_GRAPH) { clear_feature_shard(); } else { merge_feature_shard(); @@ -666,7 +668,7 @@ int32_t GraphTable::load_edges_to_ssd(const std::string &path, idx = edge_to_id[edge_type]; } total_memory_cost = 0; - auto paths = paddle::string::split_string(path, ";"); + auto paths = ::paddle::string::split_string(path, ";"); int64_t count = 0; std::string sample_type = "random"; for (auto path : paths) { @@ -674,11 +676,12 @@ int32_t GraphTable::load_edges_to_ssd(const std::string &path, std::string line; while (std::getline(file, line)) { VLOG(0) << "get a line from file " << line; - auto values = paddle::string::split_string(line, "\t"); + auto values = ::paddle::string::split_string(line, "\t"); count++; if (values.size() < 2) continue; auto src_id = std::stoll(values[0]); - auto dist_ids = paddle::string::split_string(values[1], ";"); + auto dist_ids = + ::paddle::string::split_string(values[1], ";"); std::vector dist_data; for (auto x : dist_ids) { dist_data.push_back(std::stoll(x)); @@ -798,7 +801,7 @@ int CompleteGraphSampler::run_graph_sampling() { sample_nodes.resize(gpu_num); sample_neighbors.resize(gpu_num); sample_res.resize(gpu_num); - std::vector>> + std::vector>> sample_nodes_ex(graph_table->task_pool_size_); std::vector>> sample_neighbors_ex( graph_table->task_pool_size_); @@ -812,7 +815,7 @@ int CompleteGraphSampler::run_graph_sampling() { graph_table->_shards_task_pool[i % graph_table->task_pool_size_] ->enqueue([&, i, this]() -> int { if (this->status == GraphSamplerStatus::terminating) return 0; - paddle::framework::GpuPsGraphNode node; + ::paddle::framework::GpuPsGraphNode node; std::vector &v = this->graph_table->shards[i]->get_bucket(); size_t ind = i % this->graph_table->task_pool_size_; @@ -962,7 +965,7 @@ int BasicBfsGraphSampler::run_graph_sampling() { sample_nodes.resize(gpu_num); sample_neighbors.resize(gpu_num); sample_res.resize(gpu_num); - std::vector>> + std::vector>> sample_nodes_ex(graph_table->task_pool_size_); std::vector>> sample_neighbors_ex( graph_table->task_pool_size_); @@ -977,7 +980,7 @@ int BasicBfsGraphSampler::run_graph_sampling() { if (this->status == GraphSamplerStatus::terminating) { return 0; } - paddle::framework::GpuPsGraphNode node; + ::paddle::framework::GpuPsGraphNode node; auto iter = sample_neighbors_map[i].begin(); size_t ind = i; for (; iter != sample_neighbors_map[i].end(); iter++) { @@ -1237,7 +1240,7 @@ int32_t GraphTable::Load(const std::string &path, const std::string ¶m) { } std::string GraphTable::get_inverse_etype(std::string &etype) { - auto etype_split = paddle::string::split_string(etype, "2"); + auto etype_split = ::paddle::string::split_string(etype, "2"); std::string res; if (etype_split.size() == 3) { res = etype_split[2] + "2" + etype_split[1] + "2" + etype_split[0]; @@ -1253,13 +1256,13 @@ int32_t GraphTable::parse_type_to_typepath( std::vector &res_type, std::unordered_map &res_type2path) { auto type2files_split = - paddle::string::split_string(type2files, ","); + ::paddle::string::split_string(type2files, ","); if (type2files_split.empty()) { return -1; } for (auto one_type2file : type2files_split) { auto one_type2file_split = - paddle::string::split_string(one_type2file, ":"); + ::paddle::string::split_string(one_type2file, ":"); auto type = one_type2file_split[0]; auto type_dir = one_type2file_split[1]; res_type.push_back(type); @@ -1304,17 +1307,17 @@ int32_t GraphTable::parse_edge_and_load( VLOG(1) << "only_load_reverse_edge is False, etype[" << etypes[i] << "], file_path[" << etype_path << "]"; } - auto etype_path_list = paddle::framework::localfs_list(etype_path); + auto etype_path_list = ::paddle::framework::localfs_list(etype_path); std::string etype_path_str; if (part_num > 0 && part_num < static_cast(etype_path_list.size())) { std::vector sub_etype_path_list( etype_path_list.begin(), etype_path_list.begin() + part_num); etype_path_str = - paddle::string::join_strings(sub_etype_path_list, delim); + ::paddle::string::join_strings(sub_etype_path_list, delim); } else { etype_path_str = - paddle::string::join_strings(etype_path_list, delim); + ::paddle::string::join_strings(etype_path_list, delim); } if (!only_load_reverse_edge) { this->load_edges(etype_path_str, false, etypes[i]); @@ -1345,14 +1348,14 @@ int32_t GraphTable::parse_node_and_load(std::string ntype2files, } std::string delim = ";"; std::string npath = node_to_nodedir[ntypes[0]]; - auto npath_list = paddle::framework::localfs_list(npath); + auto npath_list = ::paddle::framework::localfs_list(npath); std::string npath_str; if (part_num > 0 && part_num < static_cast(npath_list.size())) { std::vector sub_npath_list(npath_list.begin(), npath_list.begin() + part_num); - npath_str = paddle::string::join_strings(sub_npath_list, delim); + npath_str = ::paddle::string::join_strings(sub_npath_list, delim); } else { - npath_str = paddle::string::join_strings(npath_list, delim); + npath_str = ::paddle::string::join_strings(npath_list, delim); } if (ntypes.empty()) { @@ -1425,17 +1428,18 @@ int32_t GraphTable::load_node_and_edge_file( VLOG(1) << "only_load_reverse_edge is False, etype[" << etypes[i] << "], file_path[" << etype_path << "]"; } - auto etype_path_list = paddle::framework::localfs_list(etype_path); + auto etype_path_list = + ::paddle::framework::localfs_list(etype_path); std::string etype_path_str; if (part_num > 0 && part_num < static_cast(etype_path_list.size())) { std::vector sub_etype_path_list( etype_path_list.begin(), etype_path_list.begin() + part_num); etype_path_str = - paddle::string::join_strings(sub_etype_path_list, delim); + ::paddle::string::join_strings(sub_etype_path_list, delim); } else { etype_path_str = - paddle::string::join_strings(etype_path_list, delim); + ::paddle::string::join_strings(etype_path_list, delim); } if (!only_load_reverse_edge) { this->load_edges(etype_path_str, false, etypes[i]); @@ -1448,15 +1452,15 @@ int32_t GraphTable::load_node_and_edge_file( } } else { std::string npath = node_to_nodedir[ntypes[0]]; - auto npath_list = paddle::framework::localfs_list(npath); + auto npath_list = ::paddle::framework::localfs_list(npath); std::string npath_str; if (part_num > 0 && part_num < static_cast(npath_list.size())) { std::vector sub_npath_list( npath_list.begin(), npath_list.begin() + part_num); - npath_str = paddle::string::join_strings(sub_npath_list, delim); + npath_str = ::paddle::string::join_strings(sub_npath_list, delim); } else { - npath_str = paddle::string::join_strings(npath_list, delim); + npath_str = ::paddle::string::join_strings(npath_list, delim); } if (ntypes.empty()) { @@ -1553,14 +1557,14 @@ std::pair GraphTable::parse_node_file( uint64_t local_valid_count = 0; int num = 0; - std::vector vals; + std::vector<::paddle::string::str_ptr> vals; size_t n = node_type.length(); while (std::getline(file, line)) { if (strncmp(line.c_str(), node_type.c_str(), n) != 0) { continue; } vals.clear(); - num = paddle::string::split_string_ptr( + num = ::paddle::string::split_string_ptr( line.c_str() + n + 1, line.length() - n - 1, '\t', &vals); if (num == 0) { continue; @@ -1603,15 +1607,15 @@ std::pair GraphTable::parse_node_file( uint64_t local_valid_count = 0; int idx = 0; - auto path_split = paddle::string::split_string(path, "/"); + auto path_split = ::paddle::string::split_string(path, "/"); auto path_name = path_split[path_split.size() - 1]; int num = 0; - std::vector vals; + std::vector<::paddle::string::str_ptr> vals; while (std::getline(file, line)) { vals.clear(); - num = paddle::string::split_string_ptr( + num = ::paddle::string::split_string_ptr( line.c_str(), line.length(), '\t', &vals); if (vals.empty()) { continue; @@ -1654,7 +1658,7 @@ std::pair GraphTable::parse_node_file( // // TODO(danleifeng): opt load all node_types in once reading int32_t GraphTable::load_nodes(const std::string &path, std::string node_type) { - auto paths = paddle::string::split_string(path, ";"); + auto paths = ::paddle::string::split_string(path, ";"); uint64_t count = 0; uint64_t valid_count = 0; int idx = 0; @@ -1725,8 +1729,8 @@ std::pair GraphTable::parse_edge_file( uint64_t local_valid_count = 0; uint64_t part_num = 0; if (FLAGS_graph_load_in_parallel) { - auto path_split = paddle::string::split_string(path, "/"); - auto part_name_split = paddle::string::split_string( + auto path_split = ::paddle::string::split_string(path, "/"); + auto part_name_split = ::paddle::string::split_string( path_split[path_split.size() - 1], "-"); part_num = std::stoull(part_name_split[part_name_split.size() - 1]); } @@ -1793,7 +1797,7 @@ int32_t GraphTable::load_edges(const std::string &path, idx = edge_to_id[edge_type]; } - auto paths = paddle::string::split_string(path, ";"); + auto paths = ::paddle::string::split_string(path, ";"); uint64_t count = 0; uint64_t valid_count = 0; @@ -1865,7 +1869,7 @@ Node *GraphTable::find_node(GraphTableType table_type, uint64_t id) { table_type == GraphTableType::EDGE_TABLE ? edge_shards : feature_shards; for (auto &search_shard : search_shards) { PADDLE_ENFORCE_NOT_NULL(search_shard[index], - paddle::platform::errors::InvalidArgument( + ::paddle::platform::errors::InvalidArgument( "search_shard[%d] should not be null.", index)); node = search_shard[index]->find_node(id); if (node != nullptr) { @@ -1885,7 +1889,7 @@ Node *GraphTable::find_node(GraphTableType table_type, int idx, uint64_t id) { ? edge_shards[idx] : feature_shards[idx]; PADDLE_ENFORCE_NOT_NULL(search_shards[index], - paddle::platform::errors::InvalidArgument( + ::paddle::platform::errors::InvalidArgument( "search_shard[%d] should not be null.", index)); Node *node = search_shards[index]->find_node(id); return node; @@ -2164,8 +2168,8 @@ void string_vector_2_string(std::vector::iterator strs_begin, } void string_vector_2_string( - std::vector::iterator strs_begin, - std::vector::iterator strs_end, + std::vector<::paddle::string::str_ptr>::iterator strs_begin, + std::vector<::paddle::string::str_ptr>::iterator strs_end, char delim, std::string *output) { size_t i = 0; @@ -2184,19 +2188,19 @@ int GraphTable::parse_feature(int idx, FeatureNode *node) { // Return (feat_id, btyes) if name are in this->feat_name, else return (-1, // "") - thread_local std::vector fields; + thread_local std::vector<::paddle::string::str_ptr> fields; fields.clear(); char c = slot_feature_separator_.at(0); - paddle::string::split_string_ptr(feat_str, len, c, &fields); + ::paddle::string::split_string_ptr(feat_str, len, c, &fields); - thread_local std::vector fea_fields; + thread_local std::vector<::paddle::string::str_ptr> fea_fields; fea_fields.clear(); c = feature_separator_.at(0); - paddle::string::split_string_ptr(fields[1].ptr, - fields[1].len, - c, - &fea_fields, - FLAGS_gpugraph_slot_feasign_max_num); + ::paddle::string::split_string_ptr(fields[1].ptr, + fields[1].len, + c, + &fea_fields, + FLAGS_gpugraph_slot_feasign_max_num); std::string name = fields[0].to_string(); auto it = feat_id_map[idx].find(name); if (it != feat_id_map[idx].end()) { @@ -2522,14 +2526,14 @@ int32_t GraphTable::Initialize(const TableParameter &config, } void GraphTable::load_node_weight(int type_id, int idx, std::string path) { - auto paths = paddle::string::split_string(path, ";"); + auto paths = ::paddle::string::split_string(path, ";"); int64_t count = 0; auto &weight_map = node_weight[type_id][idx]; for (auto path : paths) { std::ifstream file(path); std::string line; while (std::getline(file, line)) { - auto values = paddle::string::split_string(line, "\t"); + auto values = ::paddle::string::split_string(line, "\t"); count++; if (values.size() < 2) continue; auto src_id = std::stoull(values[0]); @@ -2546,7 +2550,7 @@ int32_t GraphTable::Initialize(const GraphParameter &graph) { _db = NULL; search_level = graph.search_level(); if (search_level >= 2) { - _db = paddle::distributed::RocksDBHandler::GetInstance(); + _db = ::paddle::distributed::RocksDBHandler::GetInstance(); _db->initialize("./temp_gpups_db", task_pool_size_); } // gpups_mode = true; diff --git a/paddle/fluid/distributed/ps/table/common_graph_table.h b/paddle/fluid/distributed/ps/table/common_graph_table.h index 142327368281adcae2f1c7aa71dee5ae5056d678..91967ccb07c25e19ccc6891a2613634350fc6313 100644 --- a/paddle/fluid/distributed/ps/table/common_graph_table.h +++ b/paddle/fluid/distributed/ps/table/common_graph_table.h @@ -712,9 +712,9 @@ class GraphTable : public Table { int &actual_size); // NOLINT virtual int32_t add_node_to_ssd( int type_id, int idx, uint64_t src_id, char *data, int len); - virtual paddle::framework::GpuPsCommGraph make_gpu_ps_graph( + virtual ::paddle::framework::GpuPsCommGraph make_gpu_ps_graph( int idx, const std::vector &ids); - virtual paddle::framework::GpuPsCommGraphFea make_gpu_ps_graph_fea( + virtual ::paddle::framework::GpuPsCommGraphFea make_gpu_ps_graph_fea( int gpu_id, std::vector &node_ids, int slot_num); // NOLINT int32_t Load_to_ssd(const std::string &path, const std::string ¶m); int64_t load_graph_to_memory_from_ssd(int idx, @@ -786,7 +786,7 @@ class GraphTable : public Table { std::shared_ptr rw_lock; #ifdef PADDLE_WITH_HETERPS // paddle::framework::GpuPsGraphTable gpu_graph_table; - paddle::distributed::RocksDBHandler *_db; + ::paddle::distributed::RocksDBHandler *_db; // std::shared_ptr<::ThreadPool> graph_sample_pool; // std::shared_ptr graph_sampler; // REGISTER_GRAPH_FRIEND_CLASS(2, CompleteGraphSampler, BasicBfsGraphSampler) @@ -847,8 +847,8 @@ class BasicBfsGraphSampler : public GraphSampler { namespace std { template <> -struct hash { - size_t operator()(const paddle::distributed::SampleKey &s) const { +struct hash<::paddle::distributed::SampleKey> { + size_t operator()(const ::paddle::distributed::SampleKey &s) const { return s.idx ^ s.node_key ^ s.sample_size; } }; diff --git a/paddle/fluid/distributed/ps/table/ctr_accessor.cc b/paddle/fluid/distributed/ps/table/ctr_accessor.cc index 61e748a5413673d6e84cf88cf8ee244846cbdfe6..ca634572a462ec81a391de8260d5e28c4ab0fde2 100644 --- a/paddle/fluid/distributed/ps/table/ctr_accessor.cc +++ b/paddle/fluid/distributed/ps/table/ctr_accessor.cc @@ -14,10 +14,9 @@ #include "paddle/fluid/distributed/ps/table/ctr_accessor.h" -#include - #include "glog/logging.h" #include "paddle/fluid/string/string_helper.h" +#include "paddle/utils/flags.h" namespace paddle { namespace distributed { diff --git a/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc b/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc index 35c27242fe3556f3fb5afe5537a8aaf7883fd262..46d3ebf400d5f4b099dc22778f433aeb5b68c57a 100644 --- a/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc +++ b/paddle/fluid/distributed/ps/table/ctr_double_accessor.cc @@ -14,10 +14,9 @@ #include "paddle/fluid/distributed/ps/table/ctr_double_accessor.h" -#include - #include "glog/logging.h" #include "paddle/fluid/string/string_helper.h" +#include "paddle/utils/flags.h" namespace paddle { namespace distributed { diff --git a/paddle/fluid/distributed/ps/table/ctr_dymf_accessor.cc b/paddle/fluid/distributed/ps/table/ctr_dymf_accessor.cc index 4824ab8946b9d03c922318b95c81a21897f15f94..a2943c2237cec0b43954afab8fa22251e6fd66d3 100644 --- a/paddle/fluid/distributed/ps/table/ctr_dymf_accessor.cc +++ b/paddle/fluid/distributed/ps/table/ctr_dymf_accessor.cc @@ -14,10 +14,9 @@ #include "paddle/fluid/distributed/ps/table/ctr_dymf_accessor.h" -#include - #include "glog/logging.h" #include "paddle/fluid/string/string_helper.h" +#include "paddle/utils/flags.h" namespace paddle { namespace distributed { diff --git a/paddle/fluid/distributed/ps/table/depends/dense.h b/paddle/fluid/distributed/ps/table/depends/dense.h index 3f09376b42db3711613539ec1fe3e74b819a0ea3..272fb0b00504f1a239966a1685eb445feb67a66d 100644 --- a/paddle/fluid/distributed/ps/table/depends/dense.h +++ b/paddle/fluid/distributed/ps/table/depends/dense.h @@ -22,8 +22,8 @@ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/distributed/common/utils.h" +#include "paddle/utils/flags.h" namespace paddle { namespace distributed { diff --git a/paddle/fluid/distributed/ps/table/depends/feature_value.h b/paddle/fluid/distributed/ps/table/depends/feature_value.h index c91502a8552e842b3cd583cc36024de67ec2652c..ce01c8762253cb2628f5a1197cd4f93760a41c86 100644 --- a/paddle/fluid/distributed/ps/table/depends/feature_value.h +++ b/paddle/fluid/distributed/ps/table/depends/feature_value.h @@ -14,11 +14,12 @@ #pragma once -#include #include -#include "gflags/gflags.h" +#include + #include "paddle/fluid/distributed/common/chunk_allocator.h" +#include "paddle/utils/flags.h" namespace paddle { namespace distributed { diff --git a/paddle/fluid/distributed/ps/table/depends/initializers.h b/paddle/fluid/distributed/ps/table/depends/initializers.h index 467227097674b1d3694ec6ba69f17ccca5f83d03..3ebe5549de4f769d34cdbb820f3cfabc166633da 100644 --- a/paddle/fluid/distributed/ps/table/depends/initializers.h +++ b/paddle/fluid/distributed/ps/table/depends/initializers.h @@ -21,9 +21,9 @@ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/operators/truncated_gaussian_random_op.h" #include "paddle/phi/core/generator.h" +#include "paddle/utils/flags.h" namespace paddle { namespace distributed { @@ -124,13 +124,13 @@ class TruncatedGaussianInitializer : public Initializer { } float GetValue() override { - paddle::operators::TruncatedNormal truncated_normal(mean_, std_); + ::paddle::operators::TruncatedNormal truncated_normal(mean_, std_); float value = truncated_normal(dist_(*random_engine_)); return value; } void GetValue(float *value, int numel) { - paddle::operators::TruncatedNormal truncated_normal(mean_, std_); + ::paddle::operators::TruncatedNormal truncated_normal(mean_, std_); for (int x = 0; x < numel; ++x) { value[x] = truncated_normal(dist_(*random_engine_)); } diff --git a/paddle/fluid/distributed/ps/table/graph/class_macro.h b/paddle/fluid/distributed/ps/table/graph/class_macro.h index bf59dbacb253707efdc527a23232fcb6c11554b4..895e59d09af62a38376b4196a3d995b0dea84741 100644 --- a/paddle/fluid/distributed/ps/table/graph/class_macro.h +++ b/paddle/fluid/distributed/ps/table/graph/class_macro.h @@ -36,4 +36,4 @@ #define DECLARE_11_FRIEND_CLASS(a, ...) \ DECLARE_GRAPH_FRIEND_CLASS(a) DECLARE_10_FRIEND_CLASS(__VA_ARGS__) #define REGISTER_GRAPH_FRIEND_CLASS(n, ...) \ - DECLARE_##n##_FRIEND_CLASS(__VA_ARGS__) + PD_DECLARE_##n##_FRIEND_CLASS(__VA_ARGS__) diff --git a/paddle/fluid/distributed/ps/table/memory_sparse_table.cc b/paddle/fluid/distributed/ps/table/memory_sparse_table.cc index 5ee23010b526e0a5803718454899acceb4dcccd7..dbdff119141a5c90243078323201bdc0226fa8b2 100644 --- a/paddle/fluid/distributed/ps/table/memory_sparse_table.cc +++ b/paddle/fluid/distributed/ps/table/memory_sparse_table.cc @@ -26,16 +26,18 @@ // #include "boost/lexical_cast.hpp" #include "paddle/fluid/platform/enforce.h" -DEFINE_bool(pserver_print_missed_key_num_every_push, - false, - "pserver_print_missed_key_num_every_push"); -DEFINE_bool(pserver_create_value_when_push, - true, - "pserver create value when push"); -DEFINE_bool(pserver_enable_create_feasign_randomly, - false, - "pserver_enable_create_feasign_randomly"); -DEFINE_int32(pserver_table_save_max_retry, 3, "pserver_table_save_max_retry"); +PD_DEFINE_bool(pserver_print_missed_key_num_every_push, + false, + "pserver_print_missed_key_num_every_push"); +PD_DEFINE_bool(pserver_create_value_when_push, + true, + "pserver create value when push"); +PD_DEFINE_bool(pserver_enable_create_feasign_randomly, + false, + "pserver_enable_create_feasign_randomly"); +PD_DEFINE_int32(pserver_table_save_max_retry, + 3, + "pserver_table_save_max_retry"); namespace paddle { namespace distributed { @@ -333,7 +335,7 @@ int32_t MemorySparseTable::Save(const std::string &dirname, TopkCalculator tk(_real_local_shard_num, tk_size); std::string table_path = TableDir(dirname); - _afs_client.remove(paddle::string::format_string( + _afs_client.remove(::paddle::string::format_string( "%s/part-%03d-*", table_path.c_str(), _shard_idx)); std::atomic feasign_size_all{0}; @@ -350,15 +352,15 @@ int32_t MemorySparseTable::Save(const std::string &dirname, FsChannelConfig channel_config; if (_config.compress_in_save() && (save_param == 0 || save_param == 3)) { channel_config.path = - paddle::string::format_string("%s/part-%03d-%05d.gz", - table_path.c_str(), - _shard_idx, - file_start_idx + i); + ::paddle::string::format_string("%s/part-%03d-%05d.gz", + table_path.c_str(), + _shard_idx, + file_start_idx + i); } else { - channel_config.path = paddle::string::format_string("%s/part-%03d-%05d", - table_path.c_str(), - _shard_idx, - file_start_idx + i); + channel_config.path = ::paddle::string::format_string("%s/part-%03d-%05d", + table_path.c_str(), + _shard_idx, + file_start_idx + i); } channel_config.converter = _value_accesor->Converter(save_param).converter; channel_config.deconverter = @@ -385,7 +387,7 @@ int32_t MemorySparseTable::Save(const std::string &dirname, if (_value_accesor->Save(it.value().data(), save_param)) { std::string format_value = _value_accesor->ParseToString( it.value().data(), it.value().size()); - if (0 != write_channel->write_line(paddle::string::format_string( + if (0 != write_channel->write_line(::paddle::string::format_string( "%lu %s", it.key(), format_value.c_str()))) { ++retry_num; is_write_failed = true; @@ -432,7 +434,7 @@ int32_t MemorySparseTable::SavePatch(const std::string &path, int save_param) { } size_t file_start_idx = _m_avg_local_shard_num * _shard_idx; std::string table_path = TableDir(path); - _afs_client.remove(paddle::string::format_string( + _afs_client.remove(::paddle::string::format_string( "%s/part-%03d-*", table_path.c_str(), _shard_idx)); int thread_num = _m_real_local_shard_num < 20 ? _m_real_local_shard_num : 20; @@ -442,10 +444,10 @@ int32_t MemorySparseTable::SavePatch(const std::string &path, int save_param) { #pragma omp parallel for schedule(dynamic) for (int i = 0; i < _m_real_local_shard_num; ++i) { FsChannelConfig channel_config; - channel_config.path = paddle::string::format_string("%s/part-%03d-%05d", - table_path.c_str(), - _shard_idx, - file_start_idx + i); + channel_config.path = ::paddle::string::format_string("%s/part-%03d-%05d", + table_path.c_str(), + _shard_idx, + file_start_idx + i); channel_config.converter = _value_accesor->Converter(save_param).converter; channel_config.deconverter = @@ -469,8 +471,9 @@ int32_t MemorySparseTable::SavePatch(const std::string &path, int save_param) { if (_value_accesor->Save(it.value().data(), save_param)) { std::string format_value = _value_accesor->ParseToString( it.value().data(), it.value().size()); - if (0 != write_channel->write_line(paddle::string::format_string( - "%lu %s", it.key(), format_value.c_str()))) { + if (0 != + write_channel->write_line(::paddle::string::format_string( + "%lu %s", it.key(), format_value.c_str()))) { ++retry_num; is_write_failed = true; LOG(ERROR) << "MemorySparseTable save failed, retry it! path:" @@ -503,10 +506,10 @@ int32_t MemorySparseTable::SavePatch(const std::string &path, int save_param) { feasign_size_all += feasign_size; } LOG(INFO) << "MemorySparseTable save patch success, path:" - << paddle::string::format_string("%s/%03d/part-%03d-", - path.c_str(), - _config.table_id(), - _shard_idx) + << ::paddle::string::format_string("%s/%03d/part-%03d-", + path.c_str(), + _config.table_id(), + _shard_idx) << " from " << file_start_idx << " to " << file_start_idx + _m_real_local_shard_num - 1 << ", feasign size: " << feasign_size_all; @@ -519,7 +522,7 @@ int64_t MemorySparseTable::CacheShuffle( double cache_threshold, std::function( int msg_type, int to_pserver_id, std::string &msg)> send_msg_func, - paddle::framework::Channel> + ::paddle::framework::Channel> &shuffled_channel, const std::vector &table_ptrs) { LOG(INFO) << "cache shuffle with cache threshold: " << cache_threshold; @@ -536,24 +539,24 @@ int64_t MemorySparseTable::CacheShuffle( int thread_num = _real_local_shard_num < 20 ? _real_local_shard_num : 20; std::vector< - paddle::framework::ChannelWriter>> + ::paddle::framework::ChannelWriter>> writers(_real_local_shard_num); std::vector>> datas( _real_local_shard_num); int feasign_size = 0; - std::vector>> + std::vector<::paddle::framework::Channel>> tmp_channels; for (int i = 0; i < _real_local_shard_num; ++i) { tmp_channels.push_back( - paddle::framework::MakeChannel>()); + ::paddle::framework::MakeChannel>()); } omp_set_num_threads(thread_num); #pragma omp parallel for schedule(dynamic) for (int i = 0; i < _real_local_shard_num; ++i) { - paddle::framework::ChannelWriter> &writer = - writers[i]; + ::paddle::framework::ChannelWriter> + &writer = writers[i]; writer.Reset(tmp_channels[i].get()); for (auto table_ptr : table_ptrs) { @@ -579,15 +582,15 @@ int64_t MemorySparseTable::CacheShuffle( // shard num: " << _real_local_shard_num; std::vector> local_datas; for (int idx_shard = 0; idx_shard < _real_local_shard_num; ++idx_shard) { - paddle::framework::ChannelWriter> &writer = - writers[idx_shard]; + ::paddle::framework::ChannelWriter> + &writer = writers[idx_shard]; auto channel = writer.channel(); std::vector> &data = datas[idx_shard]; - std::vector ars(shuffle_node_num); + std::vector<::paddle::framework::BinaryArchive> ars(shuffle_node_num); while (channel->Read(data)) { for (auto &t : data) { auto pserver_id = - paddle::distributed::local_random_engine()() % shuffle_node_num; + ::paddle::distributed::local_random_engine()() % shuffle_node_num; if (pserver_id != _shard_idx) { ars[pserver_id] << t; } else { @@ -618,7 +621,7 @@ int64_t MemorySparseTable::CacheShuffle( t.wait(); } ars.clear(); - ars = std::vector(shuffle_node_num); + ars = std::vector<::paddle::framework::BinaryArchive>(shuffle_node_num); data = std::vector>(); } } @@ -629,20 +632,20 @@ int64_t MemorySparseTable::CacheShuffle( int32_t MemorySparseTable::SaveCache( const std::string &path, const std::string ¶m, - paddle::framework::Channel> + ::paddle::framework::Channel> &shuffled_channel) { if (_shard_idx >= _config.sparse_table_cache_file_num()) { return 0; } int save_param = atoi(param.c_str()); // batch_model:0 xbox:1 - std::string table_path = paddle::string::format_string( + std::string table_path = ::paddle::string::format_string( "%s/%03d_cache/", path.c_str(), _config.table_id()); - _afs_client.remove(paddle::string::format_string( + _afs_client.remove(::paddle::string::format_string( "%s/part-%03d", table_path.c_str(), _shard_idx)); uint32_t feasign_size = 0; FsChannelConfig channel_config; // not compress cache model - channel_config.path = paddle::string::format_string( + channel_config.path = ::paddle::string::format_string( "%s/part-%03d", table_path.c_str(), _shard_idx); channel_config.converter = _value_accesor->Converter(save_param).converter; channel_config.deconverter = @@ -654,7 +657,7 @@ int32_t MemorySparseTable::SaveCache( while (shuffled_channel->Read(data)) { for (auto &t : data) { ++feasign_size; - if (0 != write_channel->write_line(paddle::string::format_string( + if (0 != write_channel->write_line(::paddle::string::format_string( "%lu %s", t.first, t.second.c_str()))) { LOG(ERROR) << "Cache Table save failed, " "path:" diff --git a/paddle/fluid/distributed/ps/table/sparse_accessor.cc b/paddle/fluid/distributed/ps/table/sparse_accessor.cc index afa94703233cf05248e1ce12364b71b30be0cf07..77460a8d17e47b198f79fd12561f7b85f5a3fe62 100644 --- a/paddle/fluid/distributed/ps/table/sparse_accessor.cc +++ b/paddle/fluid/distributed/ps/table/sparse_accessor.cc @@ -14,10 +14,9 @@ #include "paddle/fluid/distributed/ps/table/sparse_accessor.h" -#include - #include "glog/logging.h" #include "paddle/fluid/string/string_helper.h" +#include "paddle/utils/flags.h" namespace paddle { namespace distributed { @@ -300,7 +299,7 @@ std::string SparseAccessor::ParseToString(const float* v, int param) { int SparseAccessor::ParseFromString(const std::string& str, float* value) { _embedx_sgd_rule->InitValue(value + sparse_feature_value.EmbedxWIndex(), value + sparse_feature_value.EmbedxG2SumIndex()); - auto ret = paddle::string::str_to_float(str.data(), value); + auto ret = ::paddle::string::str_to_float(str.data(), value); CHECK(ret >= 6) << "expect more than 6 real:" << ret; return ret; } diff --git a/paddle/fluid/distributed/ps/table/sparse_sgd_rule.cc b/paddle/fluid/distributed/ps/table/sparse_sgd_rule.cc index 6ab4506d29e4c661bccfa33d87ad867bdf336418..0c66e9d407aa4d1c68481ec641c2825c69488367 100644 --- a/paddle/fluid/distributed/ps/table/sparse_sgd_rule.cc +++ b/paddle/fluid/distributed/ps/table/sparse_sgd_rule.cc @@ -14,11 +14,11 @@ #include "paddle/fluid/distributed/ps/table/sparse_sgd_rule.h" -#include - #include "glog/logging.h" -DEFINE_bool(enable_show_scale_gradient, true, "enable show scale gradient"); +#include "paddle/utils/flags.h" + +PD_DEFINE_bool(enable_show_scale_gradient, true, "enable show scale gradient"); namespace paddle { namespace distributed { diff --git a/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc b/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc index 7d96e0f49d1adff789f327c795c3046ea06cf045..bb6de81cbb357e152349c50475f0c79fadae73e9 100644 --- a/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc +++ b/paddle/fluid/distributed/ps/table/ssd_sparse_table.cc @@ -20,11 +20,11 @@ #include "paddle/fluid/framework/archive.h" #include "paddle/fluid/platform/flags.h" #include "paddle/utils/string/string_helper.h" -DECLARE_bool(pserver_print_missed_key_num_every_push); -DECLARE_bool(pserver_create_value_when_push); -DECLARE_bool(pserver_enable_create_feasign_randomly); -DEFINE_bool(pserver_open_strict_check, false, "pserver_open_strict_check"); -DEFINE_int32(pserver_load_batch_size, 5000, "load batch size for ssd"); +PD_DECLARE_bool(pserver_print_missed_key_num_every_push); +PD_DECLARE_bool(pserver_create_value_when_push); +PD_DECLARE_bool(pserver_enable_create_feasign_randomly); +PD_DEFINE_bool(pserver_open_strict_check, false, "pserver_open_strict_check"); +PD_DEFINE_int32(pserver_load_batch_size, 5000, "load batch size for ssd"); PADDLE_DEFINE_EXPORTED_string(rocksdb_path, "database", "path of sparse table rocksdb file"); @@ -34,7 +34,7 @@ namespace distributed { int32_t SSDSparseTable::Initialize() { MemorySparseTable::Initialize(); - _db = paddle::distributed::RocksDBHandler::GetInstance(); + _db = ::paddle::distributed::RocksDBHandler::GetInstance(); _db->initialize(FLAGS_rocksdb_path, _real_local_shard_num); VLOG(0) << "initalize SSDSparseTable succ"; VLOG(0) << "SSD FLAGS_pserver_print_missed_key_num_every_push:" @@ -135,7 +135,7 @@ int32_t SSDSparseTable::PullSparse(float* pull_values, } else { data_size = tmp_string.size() / sizeof(float); memcpy(data_buffer_ptr, - paddle::string::str_to_float(tmp_string), + ::paddle::string::str_to_float(tmp_string), data_size * sizeof(float)); // from rocksdb to mem auto& feature_value = local_shard[key]; @@ -239,7 +239,7 @@ int32_t SSDSparseTable::PullSparsePtr(int shard_id, auto& feature_value = local_shard[cur_key]; feature_value.resize(data_size); memcpy(const_cast(feature_value.data()), - paddle::string::str_to_float( + ::paddle::string::str_to_float( cur_ctx->batch_values[idx].data()), data_size * sizeof(float)); _db->del_data(shard_id, @@ -302,7 +302,7 @@ int32_t SSDSparseTable::PullSparsePtr(int shard_id, feature_value.resize(data_size); memcpy( const_cast(feature_value.data()), - paddle::string::str_to_float(cur_ctx->batch_values[idx].data()), + ::paddle::string::str_to_float(cur_ctx->batch_values[idx].data()), data_size * sizeof(float)); _db->del_data( shard_id, reinterpret_cast(&cur_key), sizeof(uint64_t)); @@ -529,7 +529,7 @@ int32_t SSDSparseTable::Shrink(const std::string& param) { auto* it = _db->get_iterator(i); for (it->SeekToFirst(); it->Valid(); it->Next()) { if (_value_accesor->Shrink( - paddle::string::str_to_float(it->value().data()))) { + ::paddle::string::str_to_float(it->value().data()))) { _db->del_data(i, it->key().data(), it->key().size()); ssd_count++; } else { @@ -627,7 +627,7 @@ int32_t SSDSparseTable::SaveWithString(const std::string& path, VLOG(0) << "TopkCalculator top n:" << _cache_tk_size; size_t file_start_idx = _avg_local_shard_num * _shard_idx; std::string table_path = TableDir(path); - _afs_client.remove(paddle::string::format_string( + _afs_client.remove(::paddle::string::format_string( "%s/part-%03d-*", table_path.c_str(), _shard_idx)); #ifdef PADDLE_WITH_GPU_GRAPH int thread_num = _real_local_shard_num; @@ -640,12 +640,11 @@ int32_t SSDSparseTable::SaveWithString(const std::string& path, // feasign_size = 0; std::vector< - paddle::framework::Channel>>> + ::paddle::framework::Channel>>> fs_channel; for (int i = 0; i < _real_local_shard_num; i++) { - fs_channel.push_back( - paddle::framework::MakeChannel>>( - 10240)); + fs_channel.push_back(::paddle::framework::MakeChannel< + std::pair>>(10240)); } std::vector threads; threads.resize(_real_local_shard_num); @@ -659,29 +658,29 @@ int32_t SSDSparseTable::SaveWithString(const std::string& path, FsChannelConfig channel_config; if (_config.compress_in_save() && (save_param == 0 || save_param == 3)) { channel_config.path = - paddle::string::format_string("%s/part-%03d-%05d.gz", - table_path.c_str(), - _shard_idx, - file_start_idx + file_num); + ::paddle::string::format_string("%s/part-%03d-%05d.gz", + table_path.c_str(), + _shard_idx, + file_start_idx + file_num); } else { channel_config.path = - paddle::string::format_string("%s/part-%03d-%05d", - table_path.c_str(), - _shard_idx, - file_start_idx + file_num); + ::paddle::string::format_string("%s/part-%03d-%05d", + table_path.c_str(), + _shard_idx, + file_start_idx + file_num); } channel_config.converter = _value_accesor->Converter(save_param).converter; channel_config.deconverter = _value_accesor->Converter(save_param).deconverter; auto write_channel = _afs_client.open_w(channel_config, 1024 * 1024 * 40, &err_no); - paddle::framework::ChannelReader>> + ::paddle::framework::ChannelReader>> reader(fs_channel[file_num].get()); std::pair> out_str; while (reader >> out_str) { std::string format_value = _value_accesor->ParseToString( out_str.second.data(), out_str.second.size()); - if (0 != write_channel->write_line(paddle::string::format_string( + if (0 != write_channel->write_line(::paddle::string::format_string( "%lu %s", out_str.first, format_value.c_str()))) { LOG(FATAL) << "SSDSparseTable save failed, retry it! path:" << channel_config.path; @@ -693,8 +692,8 @@ int32_t SSDSparseTable::SaveWithString(const std::string& path, threads[i] = std::thread(save_func, i); } - std::vector< - paddle::framework::ChannelWriter>>> + std::vector<::paddle::framework::ChannelWriter< + std::pair>>> writers(_real_local_shard_num); omp_set_num_threads(thread_num); #pragma omp parallel for schedule(dynamic) @@ -726,14 +725,14 @@ int32_t SSDSparseTable::SaveWithString(const std::string& path, auto* it = _db->get_iterator(i); for (it->SeekToFirst(); it->Valid(); it->Next()) { bool need_save = _value_accesor->Save( - paddle::string::str_to_float(it->value().data()), save_param); + ::paddle::string::str_to_float(it->value().data()), save_param); _value_accesor->UpdateStatAfterSave( - paddle::string::str_to_float(it->value().data()), save_param); + ::paddle::string::str_to_float(it->value().data()), save_param); if (need_save) { std::vector feature_value; feature_value.resize(it->value().size() / sizeof(float)); memcpy(const_cast(feature_value.data()), - paddle::string::str_to_float(it->value().data()), + ::paddle::string::str_to_float(it->value().data()), it->value().size()); writer << std::make_pair(*(reinterpret_cast( const_cast(it->key().data()))), @@ -766,10 +765,10 @@ int32_t SSDSparseTable::SaveWithString(const std::string& path, } VLOG(0) << "SSDSparseTable save success, feasign size:" << feasign_size_all << ", path:" - << paddle::string::format_string("%s/%03d/part-%03d-", - path.c_str(), - _config.table_id(), - _shard_idx) + << ::paddle::string::format_string("%s/%03d/part-%03d-", + path.c_str(), + _config.table_id(), + _shard_idx) << " from " << file_start_idx << " to " << file_start_idx + _real_local_shard_num - 1; _local_show_threshold = tk.top(); @@ -800,7 +799,7 @@ int32_t SSDSparseTable::SaveWithStringMultiOutput(const std::string& path, VLOG(0) << "TopkCalculator top n:" << _cache_tk_size; size_t file_start_idx = _avg_local_shard_num * _shard_idx; std::string table_path = TableDir(path); - _afs_client.remove(paddle::string::format_string( + _afs_client.remove(::paddle::string::format_string( "%s/part-%03d-*", table_path.c_str(), _shard_idx)); #ifdef PADDLE_WITH_GPU_GRAPH int thread_num = _real_local_shard_num; @@ -809,17 +808,17 @@ int32_t SSDSparseTable::SaveWithStringMultiOutput(const std::string& path, #endif std::atomic feasign_size_all{0}; - std::vector>> + std::vector<::paddle::framework::Channel>> busy_channel; - std::vector>> + std::vector<::paddle::framework::Channel>> free_channel; std::vector threads; for (int i = 0; i < _real_local_shard_num; i++) { busy_channel.push_back( - paddle::framework::MakeChannel>()); + ::paddle::framework::MakeChannel>()); free_channel.push_back( - paddle::framework::MakeChannel>()); + ::paddle::framework::MakeChannel>()); } threads.resize(_real_local_shard_num); @@ -848,14 +847,14 @@ int32_t SSDSparseTable::SaveWithStringMultiOutput(const std::string& path, int split_num) { if (compress && (save_param == 0 || save_param == 3)) { // return - // paddle::string::format_string("%s/part-%03d-%05d-%03d-%03d.gz", + // ::paddle::string::format_string("%s/part-%03d-%05d-%03d-%03d.gz", // table_path, node_num, shard_num, part_num, split_num); - return paddle::string::format_string( + return ::paddle::string::format_string( "%s/part-%05d-%03d.gz", table_path, shard_num, split_num); } else { - // return paddle::string::format_string("%s/part-%03d-%05d-%03d-%03d", + // return ::paddle::string::format_string("%s/part-%03d-%05d-%03d-%03d", // table_path, node_num, shard_num, part_num, split_num); - return paddle::string::format_string( + return ::paddle::string::format_string( "%s/part-%05d-%03d", table_path, shard_num, split_num); } }; @@ -899,7 +898,7 @@ int32_t SSDSparseTable::SaveWithStringMultiOutput(const std::string& path, int dim = len / sizeof(float); std::string format_value = _value_accesor->ParseToString(value, dim); - if (0 != write_channel->write_line(paddle::string::format_string( + if (0 != write_channel->write_line(::paddle::string::format_string( "%lu %s", k, format_value.c_str()))) { VLOG(0) << "SSDSparseTable save failed, retry it! path:" << channel_config.path; @@ -985,9 +984,9 @@ int32_t SSDSparseTable::SaveWithStringMultiOutput(const std::string& path, auto* it = _db->get_iterator(i); for (it->SeekToFirst(); it->Valid(); it->Next()) { bool need_save = _value_accesor->Save( - paddle::string::str_to_float(it->value().data()), save_param); + ::paddle::string::str_to_float(it->value().data()), save_param); _value_accesor->UpdateStatAfterSave( - paddle::string::str_to_float(it->value().data()), save_param); + ::paddle::string::str_to_float(it->value().data()), save_param); if (need_save) { uint32_t len = sizeof(uint64_t) + it->value().size() + sizeof(uint32_t); @@ -1052,10 +1051,10 @@ int32_t SSDSparseTable::SaveWithStringMultiOutput(const std::string& path, } VLOG(0) << "DownpourSparseSSDTable save success, feasign size:" << feasign_size_all << " ,path:" - << paddle::string::format_string("%s/%03d/part-%03d-", - path.c_str(), - _config.table_id(), - _shard_idx) + << ::paddle::string::format_string("%s/%03d/part-%03d-", + path.c_str(), + _config.table_id(), + _shard_idx) << " from " << file_start_idx << " to " << file_start_idx + _real_local_shard_num - 1; if (_config.enable_sparse_table_cache()) { @@ -1085,7 +1084,7 @@ int32_t SSDSparseTable::SaveWithBinary(const std::string& path, VLOG(0) << "TopkCalculator top n:" << _cache_tk_size; size_t file_start_idx = _avg_local_shard_num * _shard_idx; std::string table_path = TableDir(path); - _afs_client.remove(paddle::string::format_string( + _afs_client.remove(::paddle::string::format_string( "%s/part-%03d-*", table_path.c_str(), _shard_idx)); #ifdef PADDLE_WITH_GPU_GRAPH int thread_num = _real_local_shard_num; @@ -1094,17 +1093,17 @@ int32_t SSDSparseTable::SaveWithBinary(const std::string& path, #endif std::atomic feasign_size_all{0}; - std::vector>> + std::vector<::paddle::framework::Channel>> busy_channel; - std::vector>> + std::vector<::paddle::framework::Channel>> free_channel; std::vector threads; for (int i = 0; i < _real_local_shard_num; i++) { busy_channel.push_back( - paddle::framework::MakeChannel>()); + ::paddle::framework::MakeChannel>()); free_channel.push_back( - paddle::framework::MakeChannel>()); + ::paddle::framework::MakeChannel>()); } threads.resize(_real_local_shard_num); @@ -1132,19 +1131,19 @@ int32_t SSDSparseTable::SaveWithBinary(const std::string& path, int part_num, int split_num) { if (compress && (save_param == 0 || save_param == 3)) { - return paddle::string::format_string("%s/part-%03d-%05d-%03d-%03d.gz", - table_path, - node_num, - shard_num, - part_num, - split_num); + return ::paddle::string::format_string("%s/part-%03d-%05d-%03d-%03d.gz", + table_path, + node_num, + shard_num, + part_num, + split_num); } else { - return paddle::string::format_string("%s/part-%03d-%05d-%03d-%03d", - table_path, - node_num, - shard_num, - part_num, - split_num); + return ::paddle::string::format_string("%s/part-%03d-%05d-%03d-%03d", + table_path, + node_num, + shard_num, + part_num, + split_num); } }; std::shared_ptr region = nullptr; @@ -1206,7 +1205,7 @@ int32_t SSDSparseTable::SaveWithBinary(const std::string& path, int dim = len / sizeof(float); std::string format_value = _value_accesor->ParseToString(value, dim); - if (0 != write_channel->write_line(paddle::string::format_string( + if (0 != write_channel->write_line(::paddle::string::format_string( "%lu %s", k, format_value.c_str()))) { LOG(FATAL) << "SSDSparseTable save failed, retry it! path:" << channel_config.path; @@ -1277,9 +1276,9 @@ int32_t SSDSparseTable::SaveWithBinary(const std::string& path, auto* it = _db->get_iterator(i); for (it->SeekToFirst(); it->Valid(); it->Next()) { bool need_save = _value_accesor->Save( - paddle::string::str_to_float(it->value().data()), save_param); + ::paddle::string::str_to_float(it->value().data()), save_param); _value_accesor->UpdateStatAfterSave( - paddle::string::str_to_float(it->value().data()), save_param); + ::paddle::string::str_to_float(it->value().data()), save_param); if (need_save) { uint32_t len = sizeof(uint64_t) + it->value().size() + sizeof(uint32_t); @@ -1344,10 +1343,10 @@ int32_t SSDSparseTable::SaveWithBinary(const std::string& path, } VLOG(0) << "DownpourSparseSSDTable save success, feasign size:" << feasign_size_all << " ,path:" - << paddle::string::format_string("%s/%03d/part-%03d-", - path.c_str(), - _config.table_id(), - _shard_idx) + << ::paddle::string::format_string("%s/%03d/part-%03d-", + path.c_str(), + _config.table_id(), + _shard_idx) << " from " << file_start_idx << " to " << file_start_idx + _real_local_shard_num - 1; if (_config.enable_sparse_table_cache()) { @@ -1364,7 +1363,7 @@ int64_t SSDSparseTable::CacheShuffle( double cache_threshold, std::function( int msg_type, int to_pserver_id, std::string& msg)> send_msg_func, - paddle::framework::Channel>& + ::paddle::framework::Channel>& shuffled_channel, const std::vector& table_ptrs) { LOG(INFO) << "cache shuffle with cache threshold: " << cache_threshold @@ -1381,27 +1380,27 @@ int64_t SSDSparseTable::CacheShuffle( int thread_num = _real_local_shard_num < 20 ? _real_local_shard_num : 20; std::vector< - paddle::framework::ChannelWriter>> + ::paddle::framework::ChannelWriter>> writers(_real_local_shard_num); std::vector>> datas( _real_local_shard_num); int feasign_size = 0; - std::vector>> + std::vector<::paddle::framework::Channel>> tmp_channels; for (int i = 0; i < _real_local_shard_num; ++i) { tmp_channels.push_back( - paddle::framework::MakeChannel>()); + ::paddle::framework::MakeChannel>()); } omp_set_num_threads(thread_num); #pragma omp parallel for schedule(dynamic) for (int i = 0; i < _real_local_shard_num; ++i) { - paddle::framework::ChannelWriter>& writer = - writers[i]; - // std::shared_ptr>& + writer = writers[i]; + // std::shared_ptr<::paddle::framework::ChannelObject>> tmp_chan = - // paddle::framework::MakeChannel>(); writer.Reset(tmp_channels[i].get()); @@ -1426,15 +1425,15 @@ int64_t SSDSparseTable::CacheShuffle( << _real_local_shard_num; std::vector> local_datas; for (int idx_shard = 0; idx_shard < _real_local_shard_num; ++idx_shard) { - paddle::framework::ChannelWriter>& writer = - writers[idx_shard]; + ::paddle::framework::ChannelWriter>& + writer = writers[idx_shard]; auto channel = writer.channel(); std::vector>& data = datas[idx_shard]; - std::vector ars(shuffle_node_num); + std::vector<::paddle::framework::BinaryArchive> ars(shuffle_node_num); while (channel->Read(data)) { for (auto& t : data) { auto pserver_id = - paddle::distributed::local_random_engine()() % shuffle_node_num; + ::paddle::distributed::local_random_engine()() % shuffle_node_num; if (pserver_id != _shard_idx) { ars[pserver_id] << t; } else { @@ -1465,7 +1464,7 @@ int64_t SSDSparseTable::CacheShuffle( t.wait(); } ars.clear(); - ars = std::vector(shuffle_node_num); + ars = std::vector<::paddle::framework::BinaryArchive>(shuffle_node_num); data = std::vector>(); } } @@ -1477,20 +1476,20 @@ int64_t SSDSparseTable::CacheShuffle( int32_t SSDSparseTable::SaveCache( const std::string& path, const std::string& param, - paddle::framework::Channel>& + ::paddle::framework::Channel>& shuffled_channel) { if (_shard_idx >= _config.sparse_table_cache_file_num()) { return 0; } int save_param = atoi(param.c_str()); // batch_model:0 xbox:1 - std::string table_path = paddle::string::format_string( + std::string table_path = ::paddle::string::format_string( "%s/%03d_cache/", path.c_str(), _config.table_id()); - _afs_client.remove(paddle::string::format_string( + _afs_client.remove(::paddle::string::format_string( "%s/part-%03d", table_path.c_str(), _shard_idx)); uint32_t feasign_size = 0; FsChannelConfig channel_config; // not compress cache model - channel_config.path = paddle::string::format_string( + channel_config.path = ::paddle::string::format_string( "%s/part-%03d", table_path.c_str(), _shard_idx); channel_config.converter = _value_accesor->Converter(save_param).converter; channel_config.deconverter = @@ -1502,7 +1501,7 @@ int32_t SSDSparseTable::SaveCache( while (shuffled_channel->Read(data)) { for (auto& t : data) { ++feasign_size; - if (0 != write_channel->write_line(paddle::string::format_string( + if (0 != write_channel->write_line(::paddle::string::format_string( "%lu %s", t.first, t.second.c_str()))) { LOG(ERROR) << "Cache Table save failed, " "path:" @@ -1580,7 +1579,7 @@ int32_t SSDSparseTable::LoadWithString( #endif for (int i = 0; i < _real_local_shard_num; i++) { - _fs_channel.push_back(paddle::framework::MakeChannel(30000)); + _fs_channel.push_back(::paddle::framework::MakeChannel(30000)); } std::vector threads; @@ -1598,7 +1597,7 @@ int32_t SSDSparseTable::LoadWithString( std::string line_data; auto read_channel = _afs_client.open_r(channel_config, 0, &err_no); - paddle::framework::ChannelWriter writer( + ::paddle::framework::ChannelWriter writer( _fs_channel[file_num].get()); while (read_channel->read_line(line_data) == 0 && line_data.size() > 1) { writer << line_data; @@ -1638,7 +1637,8 @@ int32_t SSDSparseTable::LoadWithString( uint64_t filter_time = 0; uint64_t filter_begin = 0; - paddle::framework::ChannelReader reader(_fs_channel[i].get()); + ::paddle::framework::ChannelReader reader( + _fs_channel[i].get()); while (reader >> line_data) { uint64_t key = std::strtoul(line_data.data(), &end, 10); @@ -1724,8 +1724,8 @@ int32_t SSDSparseTable::LoadWithBinary(const std::string& path, int param) { _value_accesor->GetAccessorInfo().mf_size / sizeof(float); // task pool _file_num_one_shard default 7 auto task_pool = std::make_shared<::ThreadPool>(_real_local_shard_num * 7); - auto filelists = _afs_client.list( - paddle::string::format_string("%s/part-%03d*", path.c_str(), _shard_idx)); + auto filelists = _afs_client.list(::paddle::string::format_string( + "%s/part-%03d*", path.c_str(), _shard_idx)); // #pragma omp parallel for schedule(dynamic) std::vector> tasks; @@ -1736,7 +1736,7 @@ int32_t SSDSparseTable::LoadWithBinary(const std::string& path, int param) { // _value_accesor->Converter(param).deconverter; for (auto& filename : filelists) { std::vector split_filename_string = - paddle::string::split_string(filename, "-"); + ::paddle::string::split_string(filename, "-"); int file_split_idx = atoi(split_filename_string[split_filename_string.size() - 1].c_str()); int file_shard_idx = @@ -1798,10 +1798,10 @@ int32_t SSDSparseTable::LoadWithBinary(const std::string& path, int param) { int use_sst = 0; if (file_split_idx != 0) { std::string path = - paddle::string::format_string("%s_%d/part-%03d.sst", - FLAGS_rocksdb_path.c_str(), - shard_idx, - file_split_idx); + ::paddle::string::format_string("%s_%d/part-%03d.sst", + FLAGS_rocksdb_path.c_str(), + shard_idx, + file_split_idx); rocksdb::Status status = sst_writer.Open(path); if (!status.ok()) { VLOG(0) << "sst writer open " << path << "failed"; @@ -1925,7 +1925,7 @@ int32_t SSDSparseTable::LoadWithBinary(const std::string& path, int param) { } tasks.clear(); for (int shard_idx = 0; shard_idx < _real_local_shard_num; shard_idx++) { - auto sst_filelist = _afs_client.list(paddle::string::format_string( + auto sst_filelist = _afs_client.list(::paddle::string::format_string( "%s_%d/part-*", FLAGS_rocksdb_path.c_str(), shard_idx)); if (!sst_filelist.empty()) { int ret = _db->ingest_externel_file(shard_idx, sst_filelist); @@ -2034,10 +2034,10 @@ int32_t SSDSparseTable::CacheTable(uint16_t pass_id) { if (!datas.empty()) { rocksdb::SstFileWriter sst_writer(rocksdb::EnvOptions(), options); std::string filename = - paddle::string::format_string("%s_%d/cache-%05d.sst", - FLAGS_rocksdb_path.c_str(), - shard_id, - cache_table_count); + ::paddle::string::format_string("%s_%d/cache-%05d.sst", + FLAGS_rocksdb_path.c_str(), + shard_id, + cache_table_count); rocksdb::Status status = sst_writer.Open(filename); if (!status.ok()) { VLOG(0) << "sst writer open " << filename << "failed" diff --git a/paddle/fluid/distributed/ps/table/ssd_sparse_table.h b/paddle/fluid/distributed/ps/table/ssd_sparse_table.h index e5561c5e42b991f48bceac85b9f7cb9fb1ce875d..c003061d81516270c0a166f5d06d08e5ec69701e 100644 --- a/paddle/fluid/distributed/ps/table/ssd_sparse_table.h +++ b/paddle/fluid/distributed/ps/table/ssd_sparse_table.h @@ -14,9 +14,9 @@ #pragma once -#include "gflags/gflags.h" #include "paddle/fluid/distributed/ps/table/depends/rocksdb_warpper.h" #include "paddle/fluid/distributed/ps/table/memory_sparse_table.h" +#include "paddle/utils/flags.h" namespace paddle { namespace distributed { diff --git a/paddle/fluid/distributed/ps/table/table.h b/paddle/fluid/distributed/ps/table/table.h index b64e05e3b0a111b314341d837dbc33e533ee102c..dc44831e891ca1a87845afc38fb1048b888373f9 100644 --- a/paddle/fluid/distributed/ps/table/table.h +++ b/paddle/fluid/distributed/ps/table/table.h @@ -118,7 +118,7 @@ class Table { virtual int32_t SaveCache( const std::string &path UNUSED, const std::string ¶m UNUSED, - paddle::framework::Channel> + ::paddle::framework::Channel> &shuffled_channel UNUSED) { return 0; } @@ -130,7 +130,7 @@ class Table { std::function( int msg_type, int to_pserver_id, std::string &msg)> // NOLINT send_msg_func UNUSED, - paddle::framework::Channel> + ::paddle::framework::Channel> &shuffled_channel UNUSED, const std::vector
&table_ptrs UNUSED) { return 0; @@ -161,7 +161,7 @@ class Table { virtual int32_t InitializeAccessor(); virtual int32_t InitializeShard() = 0; virtual std::string TableDir(const std::string &model_dir) { - return paddle::string::format_string( + return ::paddle::string::format_string( "%s/%03d/", model_dir.c_str(), _config.table_id()); } diff --git a/paddle/fluid/distributed/ps/wrapper/fleet.cc b/paddle/fluid/distributed/ps/wrapper/fleet.cc index d25ad89d504fada11bbbb22e2aadb02e75d493b8..87eb250545a5bf8d337461fc740bea1b76ebe5b0 100644 --- a/paddle/fluid/distributed/ps/wrapper/fleet.cc +++ b/paddle/fluid/distributed/ps/wrapper/fleet.cc @@ -30,8 +30,10 @@ const uint32_t MAX_FEASIGN_NUM = 1024 * 100 * 100; std::shared_ptr FleetWrapper::s_instance_ = NULL; bool FleetWrapper::is_initialized_ = false; -std::shared_ptr FleetWrapper::pserver_ptr_ = NULL; -std::shared_ptr FleetWrapper::worker_ptr_ = NULL; +std::shared_ptr<::paddle::distributed::PSCore> FleetWrapper::pserver_ptr_ = + NULL; +std::shared_ptr<::paddle::distributed::PSClient> FleetWrapper::worker_ptr_ = + NULL; int FleetWrapper::RegisterHeterCallback(HeterCallBackFunc handler) { VLOG(0) << "RegisterHeterCallback support later"; @@ -76,8 +78,8 @@ void FleetWrapper::InitServer( const std::vector& server_sub_program) { if (!is_initialized_) { VLOG(3) << "Going to init server"; - pserver_ptr_ = std::shared_ptr( - new paddle::distributed::PSCore()); + pserver_ptr_ = std::shared_ptr<::paddle::distributed::PSCore>( + new ::paddle::distributed::PSCore()); pserver_ptr_->InitServer(dist_desc, &host_sign_list, host_sign_list.size(), @@ -92,7 +94,7 @@ void FleetWrapper::InitServer( void FleetWrapper::InitGFlag(const std::string& gflags) { VLOG(3) << "Init With Gflags:" << gflags; - std::vector flags = paddle::string::split_string(gflags); + std::vector flags = ::paddle::string::split_string(gflags); if (flags.empty()) { flags.push_back("-max_body_size=314217728"); flags.push_back("-bthread_concurrency=40"); @@ -107,7 +109,7 @@ void FleetWrapper::InitGFlag(const std::string& gflags) { } int params_cnt = flags.size(); char** params_ptr = &(flags_ptr[0]); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(¶ms_cnt, ¶ms_ptr, true); + ::paddle::flags::ParseCommandLineFlags(¶ms_cnt, ¶ms_ptr); } void FleetWrapper::InitWorker(const std::string& dist_desc, @@ -116,17 +118,17 @@ void FleetWrapper::InitWorker(const std::string& dist_desc, if (!is_initialized_) { // not used, just for psclient's init // TODO(zhaocaibei123): remove this later - std::map> + std::map> dense_pull_regions; if (worker_ptr_.get() == nullptr) { - paddle::distributed::PSParameter ps_param; + ::paddle::distributed::PSParameter ps_param; google::protobuf::TextFormat::ParseFromString(dist_desc, &ps_param); InitGFlag(ps_param.init_gflags()); int servers = host_sign_list.size(); ps_env_.SetPsServers(&host_sign_list, servers); - worker_ptr_ = std::shared_ptr( - paddle::distributed::PSClientFactory::Create(ps_param)); + worker_ptr_ = std::shared_ptr<::paddle::distributed::PSClient>( + ::paddle::distributed::PSClientFactory::Create(ps_param)); worker_ptr_->Configure(ps_param, dense_pull_regions, ps_env_, index); } dist_desc_ = dist_desc; @@ -392,7 +394,7 @@ void FleetWrapper::PullDenseVarsAsync( Variable* var = scope.FindVar(varname); phi::DenseTensor* tensor = var->GetMutable(); float* w = tensor->data(); - paddle::distributed::Region reg(w, tensor->numel()); + ::paddle::distributed::Region reg(w, tensor->numel()); regions[i] = std::move(reg); } @@ -412,7 +414,7 @@ void FleetWrapper::PullDenseVarsSync( phi::DenseTensor* tensor = var->GetMutable(); if (!platform::is_gpu_place(tensor->place())) { float* w = tensor->data(); - paddle::distributed::Region reg(w, tensor->numel()); + ::paddle::distributed::Region reg(w, tensor->numel()); regions.emplace_back(std::move(reg)); } } @@ -425,14 +427,14 @@ void FleetWrapper::PushDenseParamSync( const uint64_t table_id, const std::vector& var_names) { auto place = platform::CPUPlace(); - std::vector regions; + std::vector<::paddle::distributed::Region> regions; for (auto& t : var_names) { Variable* var = scope.FindVar(t); CHECK(var != nullptr) << "var[" << t << "] not found"; phi::DenseTensor* tensor = var->GetMutable(); if (!platform::is_gpu_place(tensor->place())) { float* g = tensor->mutable_data(place); - paddle::distributed::Region reg(g, tensor->numel()); + ::paddle::distributed::Region reg(g, tensor->numel()); regions.emplace_back(std::move(reg)); } } @@ -456,7 +458,7 @@ void FleetWrapper::PushDenseVarsAsync( float scale_datanorm, int batch_size) { auto place = platform::CPUPlace(); - std::vector regions; + std::vector<::paddle::distributed::Region> regions; for (auto& t : var_names) { Variable* var = scope.FindVar(t); CHECK(var != nullptr) << "var[" << t << "] not found"; @@ -479,7 +481,7 @@ void FleetWrapper::PushDenseVarsAsync( } } - paddle::distributed::Region reg(g, tensor->numel()); + ::paddle::distributed::Region reg(g, tensor->numel()); regions.emplace_back(std::move(reg)); VLOG(3) << "FleetWrapper::PushDenseVarsAsync Var " << t << " talbe_id " << table_id << " Temp_data[0] " << g[0] << " Temp_data[-1] " @@ -774,7 +776,7 @@ void FleetWrapper::ShrinkDenseTable(int table_id, std::vector var_list, float decay, int emb_dim) { - std::vector regions; + std::vector<::paddle::distributed::Region> regions; for (std::string& name : var_list) { if (name.find("batch_sum") != std::string::npos) { Variable* var = scope->FindVar(name); @@ -795,14 +797,14 @@ void FleetWrapper::ShrinkDenseTable(int table_id, for (int k = 0; k < tensor->numel(); k += emb_dim) { g[k] = g[k] + g_size[k] * log(decay); } - paddle::distributed::Region reg(g, tensor->numel()); + ::paddle::distributed::Region reg(g, tensor->numel()); regions.emplace_back(std::move(reg)); } else { Variable* var = scope->FindVar(name); CHECK(var != nullptr) << "var[" << name << "] not found"; phi::DenseTensor* tensor = var->GetMutable(); float* g = tensor->data(); - paddle::distributed::Region reg(g, tensor->numel()); + ::paddle::distributed::Region reg(g, tensor->numel()); regions.emplace_back(std::move(reg)); } } diff --git a/paddle/fluid/distributed/ps/wrapper/fleet.h b/paddle/fluid/distributed/ps/wrapper/fleet.h index 9bf6f3c84a945da4652c72b00707c9d0d9c7c301..22dc0f1af724abc889e6f1b3805ee56bcc3ecbe5 100644 --- a/paddle/fluid/distributed/ps/wrapper/fleet.h +++ b/paddle/fluid/distributed/ps/wrapper/fleet.h @@ -295,7 +295,7 @@ class FleetWrapper { // FleetWrapper singleton static std::shared_ptr GetInstance() { if (NULL == s_instance_) { - s_instance_.reset(new paddle::distributed::FleetWrapper()); + s_instance_.reset(new ::paddle::distributed::FleetWrapper()); } return s_instance_; } @@ -322,13 +322,13 @@ class FleetWrapper { std::string PullFlStrategy(); //********** - static std::shared_ptr pserver_ptr_; - static std::shared_ptr worker_ptr_; + static std::shared_ptr<::paddle::distributed::PSCore> pserver_ptr_; + static std::shared_ptr<::paddle::distributed::PSClient> worker_ptr_; private: static std::shared_ptr s_instance_; std::string dist_desc_; - paddle::distributed::PaddlePSEnvironment ps_env_; + ::paddle::distributed::PaddlePSEnvironment ps_env_; size_t GetAbsoluteSum(size_t start, size_t end, size_t level, @@ -336,7 +336,7 @@ class FleetWrapper { protected: static bool is_initialized_; - std::map> regions_; + std::map> regions_; bool scale_sparse_gradient_with_batch_size_; int32_t sleep_seconds_before_fail_exit_; int client2client_request_timeout_ms_; diff --git a/paddle/fluid/distributed/rpc/CMakeLists.txt b/paddle/fluid/distributed/rpc/CMakeLists.txt index 76c6dc001104c3a56bcdd58a06493e659bbbf3ee..4042a6fe3ccfeb58a0cbee51ecfee67ccf4bea4a 100644 --- a/paddle/fluid/distributed/rpc/CMakeLists.txt +++ b/paddle/fluid/distributed/rpc/CMakeLists.txt @@ -12,17 +12,7 @@ set_source_files_properties( set_source_files_properties(rpc_agent.cc PROPERTIES COMPILE_FLAGS ${DISTRIBUTE_COMPILE_FLAGS}) -set(PADDLE_RPC_DEPS - brpc - ssl - crypto - protobuf - zlib - leveldb - snappy - phi - glog - pybind) +set(PADDLE_RPC_DEPS ${EXTERNAL_BRPC_DEPS} zlib phi pybind) proto_library(paddle_rpc_proto SRCS rpc.proto) cc_library( paddle_rpc diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index 182c83cc37bdee0a22e2a4abbf7f23f3d8344208..055163ed6206bed48e513c8fc4b6f32dbbc3e7f1 100755 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -228,13 +228,13 @@ cc_test( set(BRPC_DEPS "") if(WITH_PSCORE) - set(BRPC_DEPS brpc ssl crypto) + set(BRPC_DEPS ${EXTERNAL_BRPC_DEPS}) endif() if(WITH_PSLIB) if(WITH_PSLIB_BRPC) set(BRPC_DEPS pslib_brpc) elseif(NOT WITH_HETERPS) - set(BRPC_DEPS brpc ssl crypto) + set(BRPC_DEPS ${EXTERNAL_BRPC_DEPS}) endif() if(WITH_ARM_BRPC) set(BRPC_DEPS arm_brpc) @@ -833,7 +833,7 @@ if(WITH_DISTRIBUTE) heter_service_proto fleet heter_server - brpc + ${${EXTERNAL_BRPC_DEPS}} phi) set(DISTRIBUTE_COMPILE_FLAGS "") if(CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0) diff --git a/paddle/fluid/framework/async_executor.cc b/paddle/fluid/framework/async_executor.cc index c5d898beba1ae2d33435c28d130b1f03c4fb1a7a..055381f0d58aa0586aaed023c6dffb4f75cdc116 100644 --- a/paddle/fluid/framework/async_executor.cc +++ b/paddle/fluid/framework/async_executor.cc @@ -14,7 +14,6 @@ limitations under the License. */ #include "paddle/fluid/framework/async_executor.h" -#include "gflags/gflags.h" #include "google/protobuf/io/zero_copy_stream_impl.h" #include "google/protobuf/message.h" #include "google/protobuf/text_format.h" @@ -32,6 +31,7 @@ limitations under the License. */ #include "paddle/fluid/inference/io.h" #include "paddle/fluid/platform/place.h" #include "paddle/fluid/pybind/pybind.h" +#include "paddle/utils/flags.h" // phi #include "paddle/phi/kernels/declarations.h" diff --git a/paddle/fluid/framework/copy_same_tensor_test.cc b/paddle/fluid/framework/copy_same_tensor_test.cc index 10e0b76f00459db71394c382eefd5c256fb3c66f..9b892c0c1b092e3d5da5e398433f045714f2a12f 100644 --- a/paddle/fluid/framework/copy_same_tensor_test.cc +++ b/paddle/fluid/framework/copy_same_tensor_test.cc @@ -16,15 +16,15 @@ #include -#include "gflags/gflags.h" #include "gtest/gtest.h" #include "paddle/fluid/framework/tensor.h" #include "paddle/fluid/framework/tensor_util.h" #include "paddle/fluid/platform/device_context.h" #include "paddle/fluid/platform/place.h" #include "paddle/phi/core/ddim.h" +#include "paddle/utils/flags.h" -DECLARE_bool(use_system_allocator); +PD_DECLARE_bool(use_system_allocator); namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/details/bkcl_op_handle.h b/paddle/fluid/framework/details/bkcl_op_handle.h index 4ca8bf4cb587491bacb1fdb47616f9a9020d60f4..8a5afcf04bf9a0bfb632263b1fce97bf88d3097d 100644 --- a/paddle/fluid/framework/details/bkcl_op_handle.h +++ b/paddle/fluid/framework/details/bkcl_op_handle.h @@ -24,7 +24,7 @@ #include "paddle/fluid/platform/device/xpu/bkcl_helper.h" #include "xpu/bkcl.h" -DECLARE_bool(sync_bkcl_allreduce); +PD_DECLARE_bool(sync_bkcl_allreduce); namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/details/build_strategy.cc b/paddle/fluid/framework/details/build_strategy.cc index d295ff6ad5a685a9db87dd795ea4c0bc70ef5a81..5a6f4e6e70d4c15ffe5a041c14d1a60a98540fdd 100644 --- a/paddle/fluid/framework/details/build_strategy.cc +++ b/paddle/fluid/framework/details/build_strategy.cc @@ -22,10 +22,10 @@ limitations under the License. */ #include "paddle/fluid/framework/ir/multi_devices_graph_pass/multi_devices_graph_pass.h" #include "paddle/phi/core/flags.h" -DECLARE_bool(convert_all_blocks); +PD_DECLARE_bool(convert_all_blocks); PHI_DECLARE_bool(use_mkldnn); #ifdef PADDLE_WITH_CINN -DECLARE_bool(use_cinn); +PD_DECLARE_bool(use_cinn); #endif namespace paddle { diff --git a/paddle/fluid/framework/details/build_strategy_test.cc b/paddle/fluid/framework/details/build_strategy_test.cc index 7ec7d93ee661092514874cddf6465dbea35e50d2..0990f134b3e1bfb32e0135acdbcc5137cfb77143 100644 --- a/paddle/fluid/framework/details/build_strategy_test.cc +++ b/paddle/fluid/framework/details/build_strategy_test.cc @@ -30,7 +30,7 @@ #include "paddle/fluid/framework/var_type_inference.h" #include "paddle/fluid/platform/place.h" -DECLARE_bool(convert_all_blocks); +PD_DECLARE_bool(convert_all_blocks); namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/details/fused_all_reduce_op_handle.cc b/paddle/fluid/framework/details/fused_all_reduce_op_handle.cc index 6642866150312082ada86d7d6ef84b1cf3c744f7..a075b4702e946fd599ed46867b02a047b8ec9aaf 100644 --- a/paddle/fluid/framework/details/fused_all_reduce_op_handle.cc +++ b/paddle/fluid/framework/details/fused_all_reduce_op_handle.cc @@ -21,7 +21,7 @@ #include "paddle/phi/backends/device_memory_aligment.h" #include "paddle/phi/core/flags.h" -DEFINE_bool(skip_fused_all_reduce_check, false, ""); // NOLINT +PD_DEFINE_bool(skip_fused_all_reduce_check, false, ""); // NOLINT PHI_DECLARE_bool(allreduce_record_one_event); namespace paddle { diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc index 50c8cc926dc7f511c12f274699430c2c9618413d..f18705ef099abfff2a7f9e704e38f42d92e7eed8 100644 --- a/paddle/fluid/framework/executor.cc +++ b/paddle/fluid/framework/executor.cc @@ -31,7 +31,7 @@ limitations under the License. */ #include "paddle/fluid/framework/executor_gc_helper.h" #include "paddle/phi/core/flags.h" -DECLARE_bool(benchmark); +PD_DECLARE_bool(benchmark); PHI_DECLARE_bool(use_mkldnn); namespace paddle { diff --git a/paddle/fluid/framework/executor_thread_worker.cc b/paddle/fluid/framework/executor_thread_worker.cc index 4bbcba2151b26c40a01b5eabff830ea05d9a40d9..a3fb850373a5a51f79a0c9660f790ca7d39cb3b6 100644 --- a/paddle/fluid/framework/executor_thread_worker.cc +++ b/paddle/fluid/framework/executor_thread_worker.cc @@ -17,7 +17,6 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "google/protobuf/io/zero_copy_stream_impl.h" #include "google/protobuf/message.h" #include "google/protobuf/text_format.h" @@ -34,6 +33,7 @@ limitations under the License. */ #include "paddle/fluid/platform/place.h" #include "paddle/fluid/platform/timer.h" #include "paddle/fluid/pybind/pybind.h" +#include "paddle/utils/flags.h" // phi #include "paddle/phi/kernels/declarations.h" diff --git a/paddle/fluid/framework/fleet/CMakeLists.txt b/paddle/fluid/framework/fleet/CMakeLists.txt index 7ebc58e61b588f5b88fdb5e06c7a52f913c32f38..e2ba9c6d4f43ea9f57a750f6cdc259ef989b5ccb 100644 --- a/paddle/fluid/framework/fleet/CMakeLists.txt +++ b/paddle/fluid/framework/fleet/CMakeLists.txt @@ -3,7 +3,7 @@ if(WITH_PSLIB) set(BRPC_DEPS pslib_brpc) else() if(NOT WITH_HETERPS) - set(BRPC_DEPS brpc) + set(BRPC_DEPS ${EXTERNAL_BRPC_DEPS}) endif() endif() cc_library( diff --git a/paddle/fluid/framework/garbage_collector.cc b/paddle/fluid/framework/garbage_collector.cc index 3a514d26ddb47675a4c9eac2f8b925981e325745..01f1baf9be1919b7b382ad386dbf6fe9e0ccfb44 100644 --- a/paddle/fluid/framework/garbage_collector.cc +++ b/paddle/fluid/framework/garbage_collector.cc @@ -16,10 +16,10 @@ #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) #include "paddle/fluid/platform/cuda_device_guard.h" #endif -#include "gflags/gflags.h" #include "paddle/fluid/framework/garbage_collector.h" #include "paddle/fluid/platform/device/device_wrapper.h" #include "paddle/phi/core/flags.h" +#include "paddle/utils/flags.h" PHI_DECLARE_double(eager_delete_tensor_gb); PHI_DECLARE_double(memory_fraction_of_eager_deletion); diff --git a/paddle/fluid/framework/garbage_collector.h b/paddle/fluid/framework/garbage_collector.h index ec5766882e664302d8a90c6f005343d8c15c8c64..fa45bbcbdd2311725836c9a98a45d44ad4edc440 100644 --- a/paddle/fluid/framework/garbage_collector.h +++ b/paddle/fluid/framework/garbage_collector.h @@ -20,9 +20,9 @@ #include // NOLINT #include -#include "gflags/gflags.h" #include "paddle/fluid/platform/device_context.h" #include "paddle/fluid/platform/stream_callback_manager.h" +#include "paddle/utils/flags.h" namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/ir/graph.h b/paddle/fluid/framework/ir/graph.h index 3eb2df7011c7ed15b54d427beb4602f4c2cddf4f..3596f4e0f0e29e4dff313fc4615b530b658a575d 100644 --- a/paddle/fluid/framework/ir/graph.h +++ b/paddle/fluid/framework/ir/graph.h @@ -14,8 +14,6 @@ limitations under the License. */ #pragma once -#include - #include #include #include @@ -27,8 +25,9 @@ limitations under the License. */ #include "paddle/fluid/platform/enforce.h" #include "paddle/utils/any.h" +#include "paddle/utils/flags.h" -DECLARE_bool(convert_all_blocks); +PD_DECLARE_bool(convert_all_blocks); namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/ir/graph_helper.cc b/paddle/fluid/framework/ir/graph_helper.cc index aee64ba89ac0d57e1a7cf9ac15a5e1f46a9d1c53..87a710cd036aff183c8c8d40120e45a8dc2f5336 100644 --- a/paddle/fluid/framework/ir/graph_helper.cc +++ b/paddle/fluid/framework/ir/graph_helper.cc @@ -29,7 +29,7 @@ limitations under the License. */ #include "paddle/fluid/platform/collective_helper.h" #endif #include "paddle/fluid/platform/flags.h" -DECLARE_bool(convert_all_blocks); +PD_DECLARE_bool(convert_all_blocks); PADDLE_DEFINE_EXPORTED_string(print_sub_graph_dir, "", "FLAGS_print_sub_graph_dir is used " diff --git a/paddle/fluid/framework/ir/graph_to_program_pass.cc b/paddle/fluid/framework/ir/graph_to_program_pass.cc index f57cdd9d9746ca867ad2b506738650ae8e48791e..e1ed5ff5b041f54ed018bb936261055fc6010ad4 100644 --- a/paddle/fluid/framework/ir/graph_to_program_pass.cc +++ b/paddle/fluid/framework/ir/graph_to_program_pass.cc @@ -14,11 +14,10 @@ limitations under the License. */ #include "paddle/fluid/framework/ir/graph_to_program_pass.h" -#include - #include #include "paddle/fluid/framework/op_proto_maker.h" +#include "paddle/utils/flags.h" namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/new_executor/executor_statistics.cc b/paddle/fluid/framework/new_executor/executor_statistics.cc index 14436e9c76358596214840133b3b5d8ee725730e..9d789c9957e6cbc6427a42888c896461027eb3ac 100644 --- a/paddle/fluid/framework/new_executor/executor_statistics.cc +++ b/paddle/fluid/framework/new_executor/executor_statistics.cc @@ -28,7 +28,7 @@ #include "paddle/fluid/platform/os_info.h" #include "paddle/fluid/platform/profiler/utils.h" -DECLARE_bool(use_stream_safe_cuda_allocator); +PD_DECLARE_bool(use_stream_safe_cuda_allocator); PADDLE_DEFINE_EXPORTED_string(static_executor_perfstat_filepath, "", "FLAGS_static_executor_perfstat_filepath " diff --git a/paddle/fluid/framework/new_executor/interpreter/dependency_builder.h b/paddle/fluid/framework/new_executor/interpreter/dependency_builder.h index 4e3fb8d1b2450584a067e31394603db8daf8ce23..18a26ea770cece828078fbc3c9fb316d2e251ba4 100644 --- a/paddle/fluid/framework/new_executor/interpreter/dependency_builder.h +++ b/paddle/fluid/framework/new_executor/interpreter/dependency_builder.h @@ -19,7 +19,7 @@ #include "paddle/fluid/framework/new_executor/new_executor_defs.h" -DECLARE_bool(new_executor_sequential_run); +PD_DECLARE_bool(new_executor_sequential_run); namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/new_executor/interpreter/execution_config.cc b/paddle/fluid/framework/new_executor/interpreter/execution_config.cc index 1e6a6f02e2230339cbb9b73146e38fdbb4f51b2b..cf3195bb8c2960827901523327472bfb9f4dc54d 100644 --- a/paddle/fluid/framework/new_executor/interpreter/execution_config.cc +++ b/paddle/fluid/framework/new_executor/interpreter/execution_config.cc @@ -22,7 +22,7 @@ #include "paddle/phi/backends/gpu/gpu_info.h" #include "paddle/phi/backends/xpu/xpu_info.h" -DECLARE_bool(new_executor_serial_run); +PD_DECLARE_bool(new_executor_serial_run); namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/new_executor/interpreter_base_impl.h b/paddle/fluid/framework/new_executor/interpreter_base_impl.h index 934dd44771e7fa060c60675560c9325094dbd978..88ac481d16c5b796553d6f9cb0cfdd5d0a52f193 100644 --- a/paddle/fluid/framework/new_executor/interpreter_base_impl.h +++ b/paddle/fluid/framework/new_executor/interpreter_base_impl.h @@ -20,8 +20,8 @@ #include #include #include -#include "gflags/gflags.h" #include "paddle/fluid/platform/flags.h" +#include "paddle/utils/flags.h" #include "paddle/fluid/framework/details/exception_holder.h" #include "paddle/fluid/framework/new_executor/garbage_collector/garbage_collector.h" @@ -38,14 +38,14 @@ #include "paddle/fluid/platform/device_event.h" #include "paddle/phi/backends/device_manager.h" -DECLARE_bool(new_executor_serial_run); -DECLARE_bool(new_executor_static_build); -DECLARE_bool(new_executor_use_inplace); -DECLARE_bool(new_executor_use_local_scope); +PD_DECLARE_bool(new_executor_serial_run); +PD_DECLARE_bool(new_executor_static_build); +PD_DECLARE_bool(new_executor_use_inplace); +PD_DECLARE_bool(new_executor_use_local_scope); PHI_DECLARE_bool(check_nan_inf); -DECLARE_bool(benchmark); -DECLARE_uint64(executor_log_deps_every_microseconds); +PD_DECLARE_bool(benchmark); +PD_DECLARE_uint64(executor_log_deps_every_microseconds); PHI_DECLARE_bool(new_executor_use_cuda_graph); PHI_DECLARE_bool(enable_new_ir_in_executor); #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) diff --git a/paddle/fluid/framework/new_executor/interpretercore.h b/paddle/fluid/framework/new_executor/interpretercore.h index 191f0b92eb8b2ac7b17b7e24f5ed82dfc2e367d7..f01c12b27c3a1fb1e2f5353124b7f58fd64a55c4 100644 --- a/paddle/fluid/framework/new_executor/interpretercore.h +++ b/paddle/fluid/framework/new_executor/interpretercore.h @@ -15,7 +15,7 @@ #include "paddle/fluid/framework/new_executor/interpreter_base_impl.h" -DECLARE_bool(new_executor_use_local_scope); +PD_DECLARE_bool(new_executor_use_local_scope); namespace ir { class Program; diff --git a/paddle/fluid/framework/new_executor/new_ir_interpreter.cc b/paddle/fluid/framework/new_executor/new_ir_interpreter.cc index df9f398770e283db94fd7ba5dc7a076ba4411c86..e026e914adb575cc312288eba37c8d4da95eedc8 100644 --- a/paddle/fluid/framework/new_executor/new_ir_interpreter.cc +++ b/paddle/fluid/framework/new_executor/new_ir_interpreter.cc @@ -16,7 +16,7 @@ #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/fluid/framework/details/nan_inf_utils.h" #include "paddle/fluid/framework/details/share_tensor_buffer_functor.h" diff --git a/paddle/fluid/framework/op_kernel_type.h b/paddle/fluid/framework/op_kernel_type.h index c7193c68d3c9ccaaa106071fb726c29db905acdb..558ae8f9b9078a6c9ce36e4848d860d2810f2ba4 100644 --- a/paddle/fluid/framework/op_kernel_type.h +++ b/paddle/fluid/framework/op_kernel_type.h @@ -16,7 +16,6 @@ limitations under the License. */ #include -#include "gflags/gflags.h" #include "paddle/fluid/framework/data_layout.h" #include "paddle/fluid/framework/data_type.h" #include "paddle/fluid/framework/library_type.h" @@ -25,8 +24,9 @@ limitations under the License. */ #include "paddle/phi/core/device_context.h" #include "paddle/phi/core/enforce.h" #include "paddle/phi/core/kernel_factory.h" +#include "paddle/utils/flags.h" -DECLARE_bool(use_stride_kernel); +PD_DECLARE_bool(use_stride_kernel); namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 65a150b7ccbdf33df5b3342d9251a8ac9e48881f..0c03486fdd7500d92530f31c65d50f5e8a991ab0 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -17,7 +17,6 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/framework/convert_utils.h" #include "paddle/fluid/framework/data_transform.h" #include "paddle/fluid/framework/data_type_transform.h" @@ -43,6 +42,7 @@ limitations under the License. */ #include "paddle/phi/core/kernel_context.h" #include "paddle/phi/core/kernel_factory.h" #include "paddle/phi/ops/compat/signatures.h" +#include "paddle/utils/flags.h" namespace phi { class DenseTensor; @@ -62,9 +62,9 @@ class DenseTensor; #include "paddle/fluid/platform/device/gpu/gpu_dnn.h" #endif -DECLARE_bool(benchmark); +PD_DECLARE_bool(benchmark); PHI_DECLARE_bool(check_nan_inf); -DECLARE_bool(enable_unused_var_check); +PD_DECLARE_bool(enable_unused_var_check); PHI_DECLARE_bool(run_kp_kernel); PHI_DECLARE_bool(enable_host_event_recorder_hook); diff --git a/paddle/fluid/framework/operator_test.cc b/paddle/fluid/framework/operator_test.cc index 1d57efd875f069ce14743adf9feee2bdb4beff40..baca5b3f06743a940805dccc06f009e1e5794e8b 100644 --- a/paddle/fluid/framework/operator_test.cc +++ b/paddle/fluid/framework/operator_test.cc @@ -19,7 +19,7 @@ limitations under the License. */ #include "paddle/fluid/platform/errors.h" #include "paddle/fluid/platform/init.h" -DECLARE_bool(enable_unused_var_check); +PD_DECLARE_bool(enable_unused_var_check); namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/paddle2cinn/build_cinn_pass.cc b/paddle/fluid/framework/paddle2cinn/build_cinn_pass.cc index 9e2da446b266060bab2f87dd73e31a11a75523f3..fd0dcff440b5dbecf9866c2e6723f5c3ee7dcc3a 100644 --- a/paddle/fluid/framework/paddle2cinn/build_cinn_pass.cc +++ b/paddle/fluid/framework/paddle2cinn/build_cinn_pass.cc @@ -24,7 +24,6 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/cinn/frontend/op_mapper_registry.h" #include "paddle/cinn/frontend/op_mappers/use_op_mappers.h" @@ -38,9 +37,10 @@ limitations under the License. */ #include "paddle/fluid/operators/cinn/cinn_launch_op.h" #include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/errors.h" +#include "paddle/utils/flags.h" -DECLARE_string(allow_cinn_ops); -DECLARE_string(deny_cinn_ops); +PD_DECLARE_string(allow_cinn_ops); +PD_DECLARE_string(deny_cinn_ops); namespace paddle { namespace framework { diff --git a/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc b/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc index 29ea6098088b67aa14576828b447d1e7dcb77662..e0ddafd37da704af60b5f05b8a542ba9ac4afae2 100644 --- a/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc +++ b/paddle/fluid/framework/paddle2cinn/cinn_compiler.cc @@ -22,7 +22,6 @@ #include #include -#include "gflags/gflags.h" #include "paddle/cinn/auto_schedule/auto_tuner.h" #include "paddle/cinn/auto_schedule/tuning.h" #include "paddle/cinn/common/target.h" @@ -52,6 +51,7 @@ #include "paddle/ir/core/program.h" #include "paddle/ir/core/value.h" #include "paddle/phi/core/flags.h" +#include "paddle/utils/flags.h" PHI_DECLARE_bool(enable_pe_launch_cinn); PHI_DECLARE_bool(enable_cinn_auto_tune); diff --git a/paddle/fluid/framework/paddle2cinn/cinn_compiler_test.cc b/paddle/fluid/framework/paddle2cinn/cinn_compiler_test.cc index 4c66bc787ef894de68984c7ae1e811e7e18d39ee..519b78115748be311aad0fc393bce4189102289d 100644 --- a/paddle/fluid/framework/paddle2cinn/cinn_compiler_test.cc +++ b/paddle/fluid/framework/paddle2cinn/cinn_compiler_test.cc @@ -23,7 +23,6 @@ #include #include -#include "gflags/gflags.h" #include "glog/logging.h" #include "gtest/gtest.h" #include "paddle/cinn/common/target.h" @@ -38,6 +37,7 @@ #include "paddle/fluid/platform/place.h" #include "paddle/phi/core/ddim.h" #include "paddle/phi/core/flags.h" +#include "paddle/utils/flags.h" PHI_DECLARE_string(allow_cinn_ops); PHI_DECLARE_string(deny_cinn_ops); diff --git a/paddle/fluid/framework/scope.cc b/paddle/fluid/framework/scope.cc index 36b40657bb2b40f9ccbe880272daafd9063f102f..744ce8923a2d52474063cd4fa611cf3b1e6eac71 100644 --- a/paddle/fluid/framework/scope.cc +++ b/paddle/fluid/framework/scope.cc @@ -17,7 +17,7 @@ limitations under the License. */ #include "glog/logging.h" #include "paddle/fluid/framework/threadpool.h" #include "paddle/fluid/platform/flags.h" -DECLARE_bool(benchmark); +PD_DECLARE_bool(benchmark); PADDLE_DEFINE_EXPORTED_bool( eager_delete_scope, diff --git a/paddle/fluid/framework/unused_var_check.h b/paddle/fluid/framework/unused_var_check.h index cc4977e439c4cc7aca6b291eef9136912b199773..55a3a020a06b5e9ace4d917d31f2a0d4664ffdd0 100644 --- a/paddle/fluid/framework/unused_var_check.h +++ b/paddle/fluid/framework/unused_var_check.h @@ -19,7 +19,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" namespace paddle { namespace framework { diff --git a/paddle/fluid/imperative/prepared_operator.cc b/paddle/fluid/imperative/prepared_operator.cc index 11a4d37d799f2613eeac6f178d2b858e8e316ce2..d336488a42327cd78899260188d847293aca25b6 100644 --- a/paddle/fluid/imperative/prepared_operator.cc +++ b/paddle/fluid/imperative/prepared_operator.cc @@ -36,7 +36,7 @@ #include "paddle/phi/core/flags.h" PHI_DECLARE_bool(check_nan_inf); -DECLARE_bool(benchmark); +PD_DECLARE_bool(benchmark); PHI_DECLARE_bool(run_kp_kernel); namespace paddle { diff --git a/paddle/fluid/imperative/tracer.cc b/paddle/fluid/imperative/tracer.cc index f1374bc8f7bd7cacb1c96575a6b3472b1cc3d9b4..bd3bbf2108efc17df9d980c5a4e3b9c04a847ed8 100644 --- a/paddle/fluid/imperative/tracer.cc +++ b/paddle/fluid/imperative/tracer.cc @@ -38,7 +38,7 @@ PHI_DECLARE_bool(use_mkldnn); PHI_DECLARE_string(tracer_mkldnn_ops_on); PHI_DECLARE_string(tracer_mkldnn_ops_off); -DECLARE_bool(use_stride_kernel); +PD_DECLARE_bool(use_stride_kernel); namespace paddle { namespace imperative { diff --git a/paddle/fluid/imperative/tracer.h b/paddle/fluid/imperative/tracer.h index 5e426724aaec3a0b9161f0a7146e6afdcdd4294b..49b94c743fdb8d35620ca7a5c47efa61455fa10d 100644 --- a/paddle/fluid/imperative/tracer.h +++ b/paddle/fluid/imperative/tracer.h @@ -32,7 +32,7 @@ #include "paddle/fluid/platform/macros.h" #include "paddle/phi/core/compat/arg_map_context.h" -DECLARE_bool(use_stride_kernel); +PD_DECLARE_bool(use_stride_kernel); namespace paddle { namespace imperative { diff --git a/paddle/fluid/inference/CMakeLists.txt b/paddle/fluid/inference/CMakeLists.txt index 7b6175a975676c91c9bd8a51471d4bb5d3a02fdf..48c9f79f34de191522190ebd4d9923e9ee3cf684 100644 --- a/paddle/fluid/inference/CMakeLists.txt +++ b/paddle/fluid/inference/CMakeLists.txt @@ -37,6 +37,10 @@ get_property(ir_targets GLOBAL PROPERTY IR_TARGETS) get_property(not_infer_modules GLOBAL PROPERTY NOT_INFER_MODULES) set(utils_modules pretty_log string_helper benchmark utf8proc) +if(NOT WITH_GFLAGS) + set(utils_modules ${utils_modules} paddle_flags) +endif() + add_subdirectory(api) # Create static inference library if needed diff --git a/paddle/fluid/inference/analysis/analyzer.h b/paddle/fluid/inference/analysis/analyzer.h index 3c053283666748a33a2dfa8167fe126d8ab61219..60f86ba10eb197e8578f69f0e1a06c7269ed79f3 100644 --- a/paddle/fluid/inference/analysis/analyzer.h +++ b/paddle/fluid/inference/analysis/analyzer.h @@ -37,8 +37,8 @@ limitations under the License. */ #include -#include "gflags/gflags.h" #include "paddle/fluid/inference/analysis/analysis_pass.h" +#include "paddle/utils/flags.h" namespace paddle { namespace inference { diff --git a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc index fa35ffc45c2a863921f2d29f7f292fe940e076af..221b25cae00992cf27df03df716e83e47093797b 100644 --- a/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc +++ b/paddle/fluid/inference/analysis/passes/ir_params_sync_among_devices_pass.cc @@ -27,7 +27,7 @@ #include "paddle/fluid/platform/place.h" #include "paddle/phi/core/dense_tensor.h" -DEFINE_bool( // NOLINT +PD_DEFINE_bool( // NOLINT custom_model_save_cpu, false, "Keep old mode for developers, the model is saved on cpu not device."); diff --git a/paddle/fluid/inference/analysis/ut_helper.h b/paddle/fluid/inference/analysis/ut_helper.h index 6c7690a4779bf94f87255b8200c0ec3a384026a2..29d123d44ad45055d0ab47408e33c87ad6dddfb2 100644 --- a/paddle/fluid/inference/analysis/ut_helper.h +++ b/paddle/fluid/inference/analysis/ut_helper.h @@ -18,9 +18,9 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/framework/executor.h" #include "paddle/fluid/inference/analysis/helper.h" +#include "paddle/utils/flags.h" namespace paddle { namespace inference { @@ -30,7 +30,7 @@ extern void ReadBinaryFile(const std::string& filename, std::string* contents); namespace analysis { -DEFINE_string(inference_model_dir, "", "inference test model dir"); +PD_DEFINE_string(inference_model_dir, "", "inference test model dir"); } // namespace analysis } // namespace inference diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 7d60d203f2b2814e555832f21d1ff6b6062aff08..1fb7e2c157134d5a7136c1a76adc736d9afcd8aa 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -1708,10 +1708,10 @@ CreatePaddlePredictor( auto SetGflags = [](const AnalysisConfig &config) { auto SetGflag = [](const char *name, const char *value) { - std::string ret = ::GFLAGS_NAMESPACE::SetCommandLineOption(name, value); + bool success = paddle::flags::SetFlagValue(name, value); PADDLE_ENFORCE_EQ( - ret.empty(), - false, + success, + true, platform::errors::InvalidArgument( "Fail to set gflag: %s, please make sure the gflag exists.", name)); @@ -3089,8 +3089,8 @@ std::tuple GetTrtRuntimeVersion() { #endif } -std::string UpdateDllFlag(const char *name, const char *value) { - return paddle::UpdateDllFlag(name, value); +void UpdateDllFlag(const char *name, const char *value) { + paddle::UpdateDllFlag(name, value); } void ConvertToMixedPrecision(const std::string &model_file, diff --git a/paddle/fluid/inference/api/api.cc b/paddle/fluid/inference/api/api.cc index 93b88632984a8f339333426971c011ec68b6a7af..a15f8be18bf344b3e5de0c3415cf3ce56087b94a 100644 --- a/paddle/fluid/inference/api/api.cc +++ b/paddle/fluid/inference/api/api.cc @@ -14,13 +14,13 @@ #include -#include "gflags/gflags.h" #include "paddle/fluid/framework/commit.h" #include "paddle/fluid/framework/lod_tensor.h" #include "paddle/fluid/framework/scope.h" #include "paddle/fluid/inference/api/paddle_inference_api.h" #include "paddle/fluid/inference/api/paddle_pass_builder.h" #include "paddle/fluid/platform/enforce.h" +#include "paddle/utils/flags.h" namespace paddle { @@ -134,20 +134,18 @@ std::string get_version() { return ss.str(); } -std::string UpdateDllFlag(const char *name, const char *value) { +void UpdateDllFlag(const char *name, const char *value) { std::string ret; LOG(WARNING) << "The function \"UpdateDllFlag\" is only used to update the flag " "on the Windows shared library"; - ret = ::GFLAGS_NAMESPACE::SetCommandLineOption(name, value); + bool success = paddle::flags::SetFlagValue(name, value); PADDLE_ENFORCE_EQ( - ret.empty(), - false, + success, + true, platform::errors::InvalidArgument( "Fail to update flag: %s, please make sure the flag exists.", name)); - LOG(INFO) << ret; - return ret; } #ifdef PADDLE_WITH_CRYPTO diff --git a/paddle/fluid/inference/api/api_impl.cc b/paddle/fluid/inference/api/api_impl.cc index a2c2d099d771aa3a0748ffb3234c902a9517fd2e..d318042719a16cbd5c14ad63e340e30521f26428 100644 --- a/paddle/fluid/inference/api/api_impl.cc +++ b/paddle/fluid/inference/api/api_impl.cc @@ -26,7 +26,7 @@ limitations under the License. */ #include "paddle/fluid/platform/place.h" #include "paddle/fluid/platform/profiler.h" -DEFINE_bool(profile, false, "Turn on profiler for fluid"); // NOLINT +PD_DEFINE_bool(profile, false, "Turn on profiler for fluid"); // NOLINT namespace paddle { namespace { @@ -373,7 +373,6 @@ CreatePaddlePredictor( std::vector flags; if (config.fraction_of_gpu_memory >= 0.0f || config.fraction_of_gpu_memory <= 0.95f) { - flags.emplace_back("dummpy"); std::string flag = "--fraction_of_gpu_memory_to_use=" + num2str(config.fraction_of_gpu_memory); flags.push_back(flag); diff --git a/paddle/fluid/inference/api/demo_ci/onnxruntime_mobilenet_demo.cc b/paddle/fluid/inference/api/demo_ci/onnxruntime_mobilenet_demo.cc index fb5cee4e050346ab0e032fbcdff4e68a5052931f..04310139e5d02251266bd405c28edc35d1339b49 100644 --- a/paddle/fluid/inference/api/demo_ci/onnxruntime_mobilenet_demo.cc +++ b/paddle/fluid/inference/api/demo_ci/onnxruntime_mobilenet_demo.cc @@ -87,7 +87,7 @@ void Main() { } // namespace paddle int main(int argc, char** argv) { - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); paddle::demo::Main(); return 0; } diff --git a/paddle/fluid/inference/api/demo_ci/simple_on_word2vec.cc b/paddle/fluid/inference/api/demo_ci/simple_on_word2vec.cc index 66f2bc7056a66875648d36801bcf6b6b93e3f410..dca147e8353e300af9107210bfe777656c0afb6a 100644 --- a/paddle/fluid/inference/api/demo_ci/simple_on_word2vec.cc +++ b/paddle/fluid/inference/api/demo_ci/simple_on_word2vec.cc @@ -133,7 +133,7 @@ void MainThreads(int num_threads, bool use_gpu) { } // namespace paddle int main(int argc, char** argv) { - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); paddle::demo::Main(false /* use_gpu*/); paddle::demo::MainThreads(1, false /* use_gpu*/); paddle::demo::MainThreads(4, false /* use_gpu*/); diff --git a/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc b/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc index b6b20a901b2bb754cfdac8cad29f34fc4abe4bd7..b0f05d4f268c050767d581dabe559f2ad05a5315 100644 --- a/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc +++ b/paddle/fluid/inference/api/demo_ci/trt_mobilenet_demo.cc @@ -73,7 +73,7 @@ void Main() { } // namespace paddle int main(int argc, char** argv) { - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); paddle::demo::Main(); return 0; } diff --git a/paddle/fluid/inference/api/demo_ci/vis_demo.cc b/paddle/fluid/inference/api/demo_ci/vis_demo.cc index 7850b4edb10982bd259f71e20d04d26cc3a932c7..022c1249af001b7bb43760c5cfdb55eefd6258d3 100644 --- a/paddle/fluid/inference/api/demo_ci/vis_demo.cc +++ b/paddle/fluid/inference/api/demo_ci/vis_demo.cc @@ -28,9 +28,6 @@ DEFINE_string(data, "path of data; each line is a record, format is " "'\t MakeCipher( const std::string& config_file); diff --git a/paddle/fluid/inference/api/paddle_inference_api.h b/paddle/fluid/inference/api/paddle_inference_api.h index 18b8b6dfd43ea06a2ede35beb66ca79f20986b5d..0366a33a5f5e0d782c936c36c2997c65cd195525 100644 --- a/paddle/fluid/inference/api/paddle_inference_api.h +++ b/paddle/fluid/inference/api/paddle_inference_api.h @@ -235,7 +235,7 @@ PD_INFER_DECL int GetNumBytesOfDataType(DataType dtype); PD_INFER_DECL std::string GetVersion(); PD_INFER_DECL std::tuple GetTrtCompileVersion(); PD_INFER_DECL std::tuple GetTrtRuntimeVersion(); -PD_INFER_DECL std::string UpdateDllFlag(const char* name, const char* value); +PD_INFER_DECL void UpdateDllFlag(const char* name, const char* value); PD_INFER_DECL void ConvertToMixedPrecision( const std::string& model_file, diff --git a/paddle/fluid/inference/io.cc b/paddle/fluid/inference/io.cc index fe502a15798aa0211cce5346673996ea070115ac..000d4f5430ed4482a0ae93e62410c184bd499d17 100644 --- a/paddle/fluid/inference/io.cc +++ b/paddle/fluid/inference/io.cc @@ -29,12 +29,12 @@ limitations under the License. */ // phi #include "paddle/phi/kernels/declarations.h" -DEFINE_string(devices, // NOLINT - "", - "The devices to be used which is joined by comma."); -DEFINE_int32(math_num_threads, - 1, - "Number of threads used to run math functions."); +PD_DEFINE_string(devices, // NOLINT + "", + "The devices to be used which is joined by comma."); +PD_DEFINE_int32(math_num_threads, + 1, + "Number of threads used to run math functions."); namespace paddle { namespace inference { diff --git a/paddle/fluid/inference/paddle_inference.map b/paddle/fluid/inference/paddle_inference.map index 93d90238e340a08be2f71a7ec51e2db9f4d114e6..191f5934166c4e3959c2f1e9edb9a98cac754585 100644 --- a/paddle/fluid/inference/paddle_inference.map +++ b/paddle/fluid/inference/paddle_inference.map @@ -45,6 +45,7 @@ *paddle::RegisterSymbolsFor*; *paddle::from_blob*; *paddle::InitPhi*; + *paddle::flags*; /* ut needs the following symbol, we need to modify all the ut to hidden such symbols */ diff --git a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h index f08a8a75ba4067838642dfc1763db86e2a5ea138..b939dfaadc94da283c0b441480adace9ea6f8c84 100644 --- a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h +++ b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h @@ -31,7 +31,7 @@ namespace nvinfer1 { class ITensor; } // namespace nvinfer1 -DECLARE_bool(profile); +PD_DECLARE_bool(profile); namespace paddle { namespace inference { diff --git a/paddle/fluid/memory/allocation/allocator_facade_abs_flags_test.cc b/paddle/fluid/memory/allocation/allocator_facade_abs_flags_test.cc index 1e09c43c4f12f71eec914bba3bfd13caa7994118..21f734c7272666984126256f8d324f7843ba9b16 100644 --- a/paddle/fluid/memory/allocation/allocator_facade_abs_flags_test.cc +++ b/paddle/fluid/memory/allocation/allocator_facade_abs_flags_test.cc @@ -22,7 +22,7 @@ PHI_DECLARE_double(fraction_of_gpu_memory_to_use); PHI_DECLARE_double(fraction_of_cuda_pinned_memory_to_use); PHI_DECLARE_uint64(initial_gpu_memory_in_mb); PHI_DECLARE_uint64(reallocate_gpu_memory_in_mb); -DECLARE_int64(gpu_allocator_retry_time); +PD_DECLARE_int64(gpu_allocator_retry_time); #endif PHI_DECLARE_string(allocator_strategy); diff --git a/paddle/fluid/memory/allocation/allocator_facade_frac_flags_test.cc b/paddle/fluid/memory/allocation/allocator_facade_frac_flags_test.cc index 63e3eab3256c9fd14a72a1cb4d7bc5dee4d47713..049807f63888b68715e528095a8203e0cf56ef8a 100644 --- a/paddle/fluid/memory/allocation/allocator_facade_frac_flags_test.cc +++ b/paddle/fluid/memory/allocation/allocator_facade_frac_flags_test.cc @@ -22,7 +22,7 @@ PHI_DECLARE_double(fraction_of_gpu_memory_to_use); PHI_DECLARE_double(fraction_of_cuda_pinned_memory_to_use); PHI_DECLARE_uint64(initial_gpu_memory_in_mb); PHI_DECLARE_uint64(reallocate_gpu_memory_in_mb); -DECLARE_int64(gpu_allocator_retry_time); +PD_DECLARE_int64(gpu_allocator_retry_time); #endif PHI_DECLARE_string(allocator_strategy); diff --git a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_facade_test.cc b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_facade_test.cc index bfd05b6b323fed360375279133729add89b77956..15dc505517907fd1fbed4f38e20f361c8686c90c 100644 --- a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_facade_test.cc +++ b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_facade_test.cc @@ -26,7 +26,7 @@ #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) PHI_DECLARE_double(fraction_of_gpu_memory_to_use); PHI_DECLARE_double(fraction_of_cuda_pinned_memory_to_use); -DECLARE_int64(gpu_allocator_retry_time); +PD_DECLARE_int64(gpu_allocator_retry_time); #endif PHI_DECLARE_string(allocator_strategy); diff --git a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_test.cc b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_test.cc index 890ebde2aa3c9eb66f71557ff600222cd7308dd4..350cdbd9b4c248682267b1ae2a2970f4753d0cc0 100644 --- a/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_test.cc +++ b/paddle/fluid/memory/allocation/auto_growth_best_fit_allocator_test.cc @@ -19,8 +19,8 @@ #include "gtest/gtest.h" #include "paddle/fluid/memory/allocation/aligned_allocator.h" -DECLARE_bool(free_idle_chunk); -DECLARE_bool(free_when_no_cache_hit); +PD_DECLARE_bool(free_idle_chunk); +PD_DECLARE_bool(free_when_no_cache_hit); namespace paddle { namespace memory { diff --git a/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc b/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc index 09053ec5dede71b4267197e93d38db194951f503..1493913f5b2a8ff5f1aeef396cc6c531edc87c55 100644 --- a/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc +++ b/paddle/fluid/memory/allocation/naive_best_fit_allocator.cc @@ -41,7 +41,7 @@ PADDLE_DEFINE_EXPORTED_bool( PHI_DECLARE_double(fraction_of_gpu_memory_to_use); PHI_DECLARE_uint64(initial_gpu_memory_in_mb); PHI_DECLARE_uint64(reallocate_gpu_memory_in_mb); -DECLARE_bool(benchmark); +PD_DECLARE_bool(benchmark); namespace paddle { namespace memory { diff --git a/paddle/fluid/operators/pscore/CMakeLists.txt b/paddle/fluid/operators/pscore/CMakeLists.txt index 5a397699951ca08249e95321904c2a85f99acb8e..dea89806bc202c81be6041b27ee3aec81e2dc33f 100755 --- a/paddle/fluid/operators/pscore/CMakeLists.txt +++ b/paddle/fluid/operators/pscore/CMakeLists.txt @@ -37,15 +37,9 @@ else() ps_framework_proto framework_proto sendrecv_rpc - brpc - leveldb - ssl - crypto - protobuf + ${EXTERNAL_BRPC_DEPS} phi - glog zlib - snappy device_context) endif() diff --git a/paddle/fluid/operators/tdm_child_op.h b/paddle/fluid/operators/tdm_child_op.h index 0ebac190cda310398484b28b78bc615b389e6ab9..1a126b0a8853ccfaec87232d82c7cc3b23890075 100644 --- a/paddle/fluid/operators/tdm_child_op.h +++ b/paddle/fluid/operators/tdm_child_op.h @@ -21,9 +21,9 @@ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/framework/op_registry.h" #include "paddle/phi/core/mixed_vector.h" +#include "paddle/utils/flags.h" namespace paddle { namespace operators { diff --git a/paddle/fluid/operators/tdm_sampler_op.h b/paddle/fluid/operators/tdm_sampler_op.h index 4baff820784713afcfbd9ab28d74619feaa233b5..f3b55c4a5cc34b2944251e2df6c43addb1dfa037 100644 --- a/paddle/fluid/operators/tdm_sampler_op.h +++ b/paddle/fluid/operators/tdm_sampler_op.h @@ -21,10 +21,10 @@ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/operators/math/sampler.h" #include "paddle/phi/core/mixed_vector.h" +#include "paddle/utils/flags.h" namespace paddle { namespace operators { diff --git a/paddle/fluid/platform/cpu_info_test.cc b/paddle/fluid/platform/cpu_info_test.cc index 6f7ccd03cff2f2d0a3d55bcfc66d0860adaaf5d7..fef343112dc03028a00470ba8aea8d3ffaa047cd 100644 --- a/paddle/fluid/platform/cpu_info_test.cc +++ b/paddle/fluid/platform/cpu_info_test.cc @@ -15,10 +15,10 @@ #include -#include "gflags/gflags.h" #include "gtest/gtest.h" #include "paddle/fluid/string/printf.h" #include "paddle/phi/core/flags.h" +#include "paddle/utils/flags.h" PHI_DECLARE_double(fraction_of_cpu_memory_to_use); diff --git a/paddle/fluid/platform/cuda_graph_with_memory_pool.cc b/paddle/fluid/platform/cuda_graph_with_memory_pool.cc index 5799863f0aad70b9ef262dd2b3eb25f688bb93f4..d15f67e9965fa87bba3cb78c1f07061c40fed45d 100644 --- a/paddle/fluid/platform/cuda_graph_with_memory_pool.cc +++ b/paddle/fluid/platform/cuda_graph_with_memory_pool.cc @@ -19,7 +19,7 @@ #include "paddle/phi/backends/context_pool.h" #include "paddle/phi/core/flags.h" -DECLARE_bool(use_stream_safe_cuda_allocator); +PD_DECLARE_bool(use_stream_safe_cuda_allocator); PHI_DECLARE_bool(new_executor_use_cuda_graph); namespace paddle { diff --git a/paddle/fluid/platform/device/gpu/gpu_info.cc b/paddle/fluid/platform/device/gpu/gpu_info.cc index 10ba4ce6e1d127417bbd4dd1c3e5597ba7c684a0..ba2494a2f4891b6fc3bf8850fa2a01719b2a3cb4 100644 --- a/paddle/fluid/platform/device/gpu/gpu_info.cc +++ b/paddle/fluid/platform/device/gpu/gpu_info.cc @@ -20,7 +20,6 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/memory/memory.h" #include "paddle/fluid/platform/cuda_device_guard.h" #include "paddle/fluid/platform/enforce.h" @@ -32,6 +31,7 @@ limitations under the License. */ #include "paddle/fluid/platform/profiler/mem_tracing.h" #include "paddle/fluid/string/split.h" #include "paddle/phi/backends/gpu/gpu_info.h" +#include "paddle/utils/flags.h" #ifdef PADDLE_WITH_HIP #include "paddle/fluid/platform/dynload/miopen.h" diff --git a/paddle/fluid/platform/device/xpu/xpu_info.cc b/paddle/fluid/platform/device/xpu/xpu_info.cc index 3193d5d881989c3d13a557a06dab0b7e4a335f17..374166faeb6f7e83e8d86b07ac2fcfeec1f13f5e 100644 --- a/paddle/fluid/platform/device/xpu/xpu_info.cc +++ b/paddle/fluid/platform/device/xpu/xpu_info.cc @@ -14,7 +14,6 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/platform/device/device_wrapper.h" #include "paddle/fluid/platform/device/xpu/enforce_xpu.h" #include "paddle/fluid/platform/device/xpu/xpu_header.h" @@ -23,6 +22,7 @@ limitations under the License. */ #include "paddle/fluid/platform/monitor.h" #include "paddle/fluid/platform/place.h" #include "paddle/phi/backends/xpu/xpu_info.h" +#include "paddle/utils/flags.h" namespace paddle { namespace platform { diff --git a/paddle/fluid/platform/dynload/dynamic_loader.cc b/paddle/fluid/platform/dynload/dynamic_loader.cc index 40f69a87f37f867826ca2c153a76bdebdaae90e6..94a96b67cd6b1c43dd9fa69ef3934312a70a1b2b 100644 --- a/paddle/fluid/platform/dynload/dynamic_loader.cc +++ b/paddle/fluid/platform/dynload/dynamic_loader.cc @@ -16,8 +16,8 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "paddle/phi/backends/dynload/dynamic_loader.h" +#include "paddle/utils/flags.h" namespace paddle { namespace platform { diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h index 425d4939b565f5f49fad64cbc00c14af9fd29139..d9c9398461d5c721da69020b0ab32be2bb5213e0 100644 --- a/paddle/fluid/platform/enforce.h +++ b/paddle/fluid/platform/enforce.h @@ -60,10 +60,10 @@ limitations under the License. */ #endif #define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/fluid/platform/errors.h" #include "paddle/fluid/platform/macros.h" +#include "paddle/utils/flags.h" #include "paddle/fluid/string/printf.h" #include "paddle/fluid/string/to_string.h" diff --git a/paddle/fluid/platform/flags.h b/paddle/fluid/platform/flags.h index b08aececd2b816a9515424824d8e54a84498cae0..66d8f9557ef189e3751c9415731817fa7ffe6df3 100644 --- a/paddle/fluid/platform/flags.h +++ b/paddle/fluid/platform/flags.h @@ -23,7 +23,7 @@ #define __PADDLE_DEFINE_EXPORTED_FLAG( \ __name, __is_writable, __cpp_type, __gflag_type, __default_value, __doc) \ - DEFINE_##__gflag_type(__name, __default_value, __doc); /* NOLINT */ \ + PD_DEFINE_##__gflag_type(__name, __default_value, __doc); /* NOLINT */ \ struct __PaddleRegisterFlag_##__name { \ __PaddleRegisterFlag_##__name() { \ using FlagDeclaredType = \ diff --git a/paddle/fluid/platform/init.cc b/paddle/fluid/platform/init.cc index 2cc3dc7124a52acaeb87a3ebf1af0b92d7981d5d..4e0803c59ce59835524e2a7268d0a009ba05b04d 100644 --- a/paddle/fluid/platform/init.cc +++ b/paddle/fluid/platform/init.cc @@ -102,8 +102,8 @@ bool InitGflags(std::vector args) { << ", Init commandline: " << line; char **arr = argv.data(); - ::GFLAGS_NAMESPACE::AllowCommandLineReparsing(); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &arr, true); + paddle::flags::AllowUndefinedFlags(); + paddle::flags::ParseCommandLineFlags(&argc, &arr); successed = true; VLOG(1) << "After Parse: argc is " << argc; diff --git a/paddle/fluid/platform/init.h b/paddle/fluid/platform/init.h index 2d5c34002b4bb91803c85ab9607bb187d4a3847e..4ff99f2866e605333334a2f27c541b6dc8bab5c4 100644 --- a/paddle/fluid/platform/init.h +++ b/paddle/fluid/platform/init.h @@ -17,8 +17,8 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "glog/logging.h" +#include "paddle/utils/flags.h" namespace paddle { namespace framework { diff --git a/paddle/fluid/platform/profiler.cc b/paddle/fluid/platform/profiler.cc index 54d27cbd7b755730c785c77f090880fc27d3cc4c..efee8a264bc636e79cf531741f105747aa39add4 100644 --- a/paddle/fluid/platform/profiler.cc +++ b/paddle/fluid/platform/profiler.cc @@ -38,7 +38,9 @@ PADDLE_DEFINE_EXPORTED_bool(enable_rpc_profiler, false, "Enable rpc profiler or not."); -DEFINE_bool(enable_record_memory, false, "enable memory recorder"); // NOLINT +PD_DEFINE_bool(enable_record_memory, + false, + "enable memory recorder"); // NOLINT #if defined(_WIN32) && defined(PHI_SHARED) phi::ProfilerState phi::ProfilerHelper::g_state = phi::ProfilerState::kDisabled; diff --git a/paddle/fluid/platform/profiler/host_tracer.cc b/paddle/fluid/platform/profiler/host_tracer.cc index 6907ccd3b044bf68578167b485f5e1fda5ab93e8..56fe468838b836683dd854620f0a58bbb39287ff 100644 --- a/paddle/fluid/platform/profiler/host_tracer.cc +++ b/paddle/fluid/platform/profiler/host_tracer.cc @@ -17,17 +17,9 @@ #include "glog/logging.h" #include "paddle/fluid/framework/op_proto_maker.h" -#include "paddle/fluid/platform/flags.h" #include "paddle/fluid/platform/profiler/common_event.h" #include "paddle/fluid/platform/profiler/host_event_recorder.h" -// Used to filter events, works like glog VLOG(level). -// RecordEvent will works if host_trace_level >= level. -PADDLE_DEFINE_EXPORTED_int64(host_trace_level, - 1, - "RecordEvent will works " - "if host_trace_level >= level."); - namespace paddle { namespace platform { diff --git a/paddle/fluid/platform/profiler/profiler.cc b/paddle/fluid/platform/profiler/profiler.cc index 78e6443cbac6d60bd28fc7b7e0d6e2809140ad62..4f58b0e3ccef460fce8f0edfa6e6c97c964b8399 100644 --- a/paddle/fluid/platform/profiler/profiler.cc +++ b/paddle/fluid/platform/profiler/profiler.cc @@ -25,6 +25,7 @@ #include "paddle/fluid/platform/device/gpu/gpu_info.h" #endif #include "paddle/fluid/platform/enforce.h" +#include "paddle/fluid/platform/flags.h" #include "paddle/fluid/platform/profiler/cuda_tracer.h" #include "paddle/fluid/platform/profiler/custom_device/custom_tracer.h" #include "paddle/fluid/platform/profiler/extra_info.h" @@ -36,6 +37,13 @@ #include "paddle/phi/backends/device_manager.h" #endif +// Used to filter events, works like glog VLOG(level). +// RecordEvent will works if host_trace_level >= level. +PADDLE_DEFINE_EXPORTED_int64(host_trace_level, + 1, + "RecordEvent will works " + "if host_trace_level >= level."); + namespace paddle { namespace platform { diff --git a/paddle/fluid/platform/profiler/profiler.h b/paddle/fluid/platform/profiler/profiler.h index 28cf7a2d3857224c6c76b854585676154f1f135c..4ab98bab53050d80663824ddccc619ffa0c85229 100644 --- a/paddle/fluid/platform/profiler/profiler.h +++ b/paddle/fluid/platform/profiler/profiler.h @@ -27,7 +27,7 @@ #include "paddle/fluid/platform/profiler/event_python.h" #include "paddle/fluid/platform/profiler/tracer_base.h" -DECLARE_int64(host_trace_level); +PD_DECLARE_int64(host_trace_level); namespace paddle { namespace platform { diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index 42cdd9be3067c21d522af45396a667f3f7206daa..478efa7a5af9593b7cf05b0c23dd4166018cff04 100755 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -57,18 +57,7 @@ if(WITH_PSCORE) endif() endif() if(WITH_RPC) - set(PYBIND_DEPS - ${PYBIND_DEPS} - paddle_rpc - brpc - ssl - crypto - protobuf - zlib - leveldb - snappy - phi - glog) + set(PYBIND_DEPS ${PYBIND_DEPS} paddle_rpc ${EXTERNAL_BRPC_DEPS} zlib phi) endif() if(WITH_GPU OR WITH_ROCM) set(PYBIND_DEPS ${PYBIND_DEPS} dynload_cuda) diff --git a/paddle/fluid/pybind/eager_method.cc b/paddle/fluid/pybind/eager_method.cc index f667602a493105c53506720d40dd8d340d2d60d7..7d3037a076d572051c6e4a06a2358dcf44729576 100644 --- a/paddle/fluid/pybind/eager_method.cc +++ b/paddle/fluid/pybind/eager_method.cc @@ -67,7 +67,7 @@ typedef SSIZE_T ssize_t; #include "paddle/utils/pybind.h" PHI_DECLARE_bool(set_to_1d); -DECLARE_bool(use_stride_kernel); +PD_DECLARE_bool(use_stride_kernel); namespace paddle { namespace pybind { diff --git a/paddle/fluid/pybind/global_value_getter_setter.cc b/paddle/fluid/pybind/global_value_getter_setter.cc index 94e3ca1ba41bdbb944f8b031479d347c423b9fe0..7f8cac9ee1edf90053041b77b27c92743b4ff914 100644 --- a/paddle/fluid/pybind/global_value_getter_setter.cc +++ b/paddle/fluid/pybind/global_value_getter_setter.cc @@ -22,12 +22,12 @@ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/framework/python_headers.h" #include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/errors.h" #include "paddle/fluid/platform/macros.h" #include "paddle/phi/core/macros.h" +#include "paddle/utils/flags.h" #include "pybind11/stl.h" // FIXME(zengjinle): these 2 flags may be removed by the linker when compiling @@ -41,8 +41,8 @@ PADDLE_FORCE_LINK_FLAG(free_when_no_cache_hit); // NOTE: where are these 2 flags from? #ifdef PADDLE_WITH_DISTRIBUTE -DECLARE_int32(rpc_get_thread_num); -DECLARE_int32(rpc_prefetch_thread_num); +PD_DECLARE_int32(rpc_get_thread_num); +PD_DECLARE_int32(rpc_prefetch_thread_num); #endif namespace paddle { diff --git a/paddle/fluid/pybind/reader_py.cc b/paddle/fluid/pybind/reader_py.cc index 0ae52f7fdd74b0b9da021faae46ea8059281fe43..0e581e45b5970dba3a9b5ff77719388e8ce288a7 100644 --- a/paddle/fluid/pybind/reader_py.cc +++ b/paddle/fluid/pybind/reader_py.cc @@ -23,7 +23,6 @@ #include "Python.h" -#include "gflags/gflags.h" #include "paddle/fluid/framework/reader.h" #include "paddle/fluid/imperative/layer.h" #include "paddle/fluid/imperative/tracer.h" @@ -33,6 +32,7 @@ #include "paddle/fluid/platform/place.h" #include "paddle/phi/core/ddim.h" #include "paddle/phi/core/flags.h" +#include "paddle/utils/flags.h" #include "pybind11/stl.h" PHI_DECLARE_bool(reader_queue_speed_test_mode); diff --git a/paddle/fluid/string/pretty_log.h b/paddle/fluid/string/pretty_log.h index d161b2a912fcaa5bf31f03b28efa84fddad950da..8e34cbb8ab9dc09d64464fe41dc4139c1ee157a0 100644 --- a/paddle/fluid/string/pretty_log.h +++ b/paddle/fluid/string/pretty_log.h @@ -18,5 +18,5 @@ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/utils/string/pretty_log.h" diff --git a/paddle/phi/CMakeLists.txt b/paddle/phi/CMakeLists.txt index 83d8006957ac469519e58a5f15975a0a56e8f58a..cfbf8fec0adfdc05d3a90004922649af0dd25898 100644 --- a/paddle/phi/CMakeLists.txt +++ b/paddle/phi/CMakeLists.txt @@ -36,7 +36,7 @@ endif() set(PHI_DEPS phi_profiler_proto auto_parallel_proto - gflags + ${flags_dep} glog warpctc warprnnt diff --git a/paddle/phi/api/lib/api_gen_utils.cc b/paddle/phi/api/lib/api_gen_utils.cc index 73a1ed4c7fdc850de0d14f1092fc8f9e620ccbdd..80e169fae10c89069dda69318a591c6b570ce9ae 100644 --- a/paddle/phi/api/lib/api_gen_utils.cc +++ b/paddle/phi/api/lib/api_gen_utils.cc @@ -13,11 +13,11 @@ See the License for the specific language governing permissions and limitations under the License. */ #include "paddle/phi/api/lib/api_gen_utils.h" -#include "gflags/gflags.h" #include "paddle/phi/core/visit_type.h" #include "paddle/phi/kernels/strided_copy_kernel.h" +#include "paddle/utils/flags.h" -DECLARE_bool(use_stride_kernel); +PD_DECLARE_bool(use_stride_kernel); #include "glog/logging.h" diff --git a/paddle/phi/api/lib/data_transform.cc b/paddle/phi/api/lib/data_transform.cc index 3d717c6dbfe05975a8840d0b83481eb38e89b079..a8eb379e359fc49a2106d979e167e77596614c23 100644 --- a/paddle/phi/api/lib/data_transform.cc +++ b/paddle/phi/api/lib/data_transform.cc @@ -16,7 +16,6 @@ limitations under the License. */ #include "glog/logging.h" -#include "gflags/gflags.h" #include "paddle/fluid/platform/device_context.h" #include "paddle/phi/api/lib/kernel_dispatch.h" #include "paddle/phi/api/lib/utils/allocator.h" @@ -28,8 +27,9 @@ limitations under the License. */ #include "paddle/phi/kernels/cast_kernel.h" #include "paddle/phi/kernels/contiguous_kernel.h" #include "paddle/phi/kernels/transfer_layout_kernel.h" +#include "paddle/utils/flags.h" -DECLARE_bool(use_stride_kernel); +PD_DECLARE_bool(use_stride_kernel); namespace paddle { namespace experimental { diff --git a/paddle/phi/api/profiler/device_tracer.cc b/paddle/phi/api/profiler/device_tracer.cc index b16579491f49d5a2f545deb738020099c4d79833..e294130da7bab81169fe91bc9c24f40076814e9c 100644 --- a/paddle/phi/api/profiler/device_tracer.cc +++ b/paddle/phi/api/profiler/device_tracer.cc @@ -21,11 +21,11 @@ limitations under the License. */ #include #include // NOLINT -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/core/enforce.h" +#include "paddle/utils/flags.h" -DECLARE_bool(enable_host_event_recorder_hook); +PD_DECLARE_bool(enable_host_event_recorder_hook); namespace phi { diff --git a/paddle/phi/api/yaml/generator/api_gen.py b/paddle/phi/api/yaml/generator/api_gen.py index 50ba2078b045f9d0f906951d71bd4c28b324173a..5164ebda8403557a587b1bedc6c6145472e15618 100644 --- a/paddle/phi/api/yaml/generator/api_gen.py +++ b/paddle/phi/api/yaml/generator/api_gen.py @@ -360,7 +360,7 @@ def source_include(header_file_path): #include #include "glog/logging.h" -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/phi/api/lib/api_custom_impl.h" #include "paddle/phi/api/lib/api_gen_utils.h" @@ -379,8 +379,8 @@ def source_include(header_file_path): #include "paddle/phi/api/profiler/event_tracing.h" #include "paddle/phi/api/profiler/supplement_tracing.h" -DECLARE_bool(conv2d_disable_cudnn); -DECLARE_int32(low_precision_op_list); +PD_DECLARE_bool(conv2d_disable_cudnn); +PD_DECLARE_int32(low_precision_op_list); """ diff --git a/paddle/phi/api/yaml/generator/backward_api_gen.py b/paddle/phi/api/yaml/generator/backward_api_gen.py index 183904f642c986b5e6f7d878cbadffb30716e594..9347552dbb13422d2eaed5545c641ddfbd9e81ec 100644 --- a/paddle/phi/api/yaml/generator/backward_api_gen.py +++ b/paddle/phi/api/yaml/generator/backward_api_gen.py @@ -275,7 +275,7 @@ def source_include(header_file_path, fw_header_file_path): #include #include "glog/logging.h" -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/phi/api/lib/api_custom_impl.h" #include "paddle/phi/api/lib/api_gen_utils.h" @@ -290,8 +290,8 @@ def source_include(header_file_path, fw_header_file_path): #include "paddle/phi/api/profiler/event_tracing.h" #include "paddle/phi/api/profiler/supplement_tracing.h" -DECLARE_bool(conv2d_disable_cudnn); -DECLARE_int32(low_precision_op_list); +PD_DECLARE_bool(conv2d_disable_cudnn); +PD_DECLARE_int32(low_precision_op_list); """ diff --git a/paddle/phi/api/yaml/generator/dist_bw_api_gen.py b/paddle/phi/api/yaml/generator/dist_bw_api_gen.py index ac6f5f2e916eb39e1285e0de28b04a4eca09de0d..95a6f94706eeeec3d63dc5f7ad472080d4b9fc83 100644 --- a/paddle/phi/api/yaml/generator/dist_bw_api_gen.py +++ b/paddle/phi/api/yaml/generator/dist_bw_api_gen.py @@ -138,7 +138,7 @@ def source_include(header_file_path, fw_header_file_path): #include #include "glog/logging.h" -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/phi/api/lib/api_custom_impl.h" #include "paddle/phi/api/lib/api_gen_utils.h" @@ -153,8 +153,8 @@ def source_include(header_file_path, fw_header_file_path): #include "paddle/phi/api/profiler/event_tracing.h" #include "paddle/phi/api/profiler/supplement_tracing.h" -DECLARE_bool(conv2d_disable_cudnn); -DECLARE_int32(low_precision_op_list); +PD_DECLARE_bool(conv2d_disable_cudnn); +PD_DECLARE_int32(low_precision_op_list); """ diff --git a/paddle/phi/api/yaml/generator/intermediate_api_gen.py b/paddle/phi/api/yaml/generator/intermediate_api_gen.py index e8f9c1cf75152558b23040ecb7e121be52aff22b..5c3b6ddd32d5d93df962a3d8f593f037e1adaf99 100644 --- a/paddle/phi/api/yaml/generator/intermediate_api_gen.py +++ b/paddle/phi/api/yaml/generator/intermediate_api_gen.py @@ -36,7 +36,7 @@ def source_include(header_file_path): #include #include "glog/logging.h" -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/phi/api/lib/api_custom_impl.h" #include "paddle/phi/api/lib/api_gen_utils.h" @@ -56,7 +56,7 @@ def source_include(header_file_path): #include "paddle/phi/api/profiler/event_tracing.h" #include "paddle/phi/api/profiler/supplement_tracing.h" -DECLARE_int32(low_precision_op_list); +PD_DECLARE_int32(low_precision_op_list); """ diff --git a/paddle/phi/api/yaml/generator/sparse_api_gen.py b/paddle/phi/api/yaml/generator/sparse_api_gen.py index af28fcb0948c90c1d0f0758e634e902284bbb898..9a017725d68889a70bfa44966f3cac218a7eb7cf 100644 --- a/paddle/phi/api/yaml/generator/sparse_api_gen.py +++ b/paddle/phi/api/yaml/generator/sparse_api_gen.py @@ -426,7 +426,7 @@ def source_include(header_file_path): #include #include "glog/logging.h" -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/phi/api/lib/api_gen_utils.h" #include "paddle/phi/api/lib/data_transform.h" @@ -442,7 +442,7 @@ def source_include(header_file_path): #include "paddle/phi/infermeta/sparse/binary.h" #include "paddle/phi/infermeta/sparse/multiary.h" -DECLARE_int32(low_precision_op_list); +PD_DECLARE_int32(low_precision_op_list); """ diff --git a/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py b/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py index 67a8514bf19373667f0ccd4a3ac3c7ad3bf4cc74..064cf07d0dbf764efe57d1109a763829be9408ad 100644 --- a/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py +++ b/paddle/phi/api/yaml/generator/sparse_bw_api_gen.py @@ -121,7 +121,7 @@ def source_include(header_file_path): #include #include "glog/logging.h" -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/phi/api/include/sparse_api.h" #include "paddle/phi/api/lib/api_gen_utils.h" @@ -137,7 +137,7 @@ def source_include(header_file_path): #include "paddle/phi/infermeta/sparse/binary.h" #include "paddle/phi/infermeta/sparse/backward.h" -DECLARE_int32(low_precision_op_list); +PD_DECLARE_int32(low_precision_op_list); """ diff --git a/paddle/phi/api/yaml/generator/strings_api_gen.py b/paddle/phi/api/yaml/generator/strings_api_gen.py index ed0bfe42bd8b787a4d317c6dd0733692fdfb8d69..4e66bd5f2fdc6b8cc93a83269c206495c66e9bfc 100644 --- a/paddle/phi/api/yaml/generator/strings_api_gen.py +++ b/paddle/phi/api/yaml/generator/strings_api_gen.py @@ -329,8 +329,8 @@ def source_include(header_file_path): return f""" #include "{header_file_path}" -#include "gflags/gflags.h" #include "glog/logging.h" +#include "paddle/utils/flags.h" #include "paddle/phi/api/lib/api_gen_utils.h" #include "paddle/phi/core/kernel_context.h" @@ -340,7 +340,7 @@ def source_include(header_file_path): #include "paddle/phi/api/lib/kernel_dispatch.h" #include "paddle/phi/core/kernel_registry.h" -DECLARE_int32(low_precision_op_list); +PD_DECLARE_int32(low_precision_op_list); """ diff --git a/paddle/phi/api/yaml/generator/tensor_operants_gen.py b/paddle/phi/api/yaml/generator/tensor_operants_gen.py index f86efeaaefac8814f33e72a4e6449c05bd7fe35d..1ca80a8bd76e820d6c1093419427ffd19572806b 100644 --- a/paddle/phi/api/yaml/generator/tensor_operants_gen.py +++ b/paddle/phi/api/yaml/generator/tensor_operants_gen.py @@ -441,16 +441,16 @@ operants_manager_source_include = """// Generated by paddle/phi/api/yaml/generat #include "paddle/phi/api/include/operants_manager.h" -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/core/enforce.h" #include "paddle/phi/core/errors.h" +#include "paddle/utils/flags.h" """ operants_manager_source_start = """ -DECLARE_string(tensor_operants_mode); +PD_DECLARE_string(tensor_operants_mode); namespace paddle { diff --git a/paddle/phi/backends/cpu/cpu_info.cc b/paddle/phi/backends/cpu/cpu_info.cc index 8d1d757dfd598307bbf4b73e34de9ddc51002866..2eda0104877543139e5a9fd62f957bf23e22b969 100644 --- a/paddle/phi/backends/cpu/cpu_info.cc +++ b/paddle/phi/backends/cpu/cpu_info.cc @@ -35,9 +35,9 @@ limitations under the License. */ #include "paddle/phi/core/flags.h" -DECLARE_double(fraction_of_cpu_memory_to_use); -DECLARE_uint64(initial_cpu_memory_in_mb); -DECLARE_double(fraction_of_cuda_pinned_memory_to_use); +PD_DECLARE_double(fraction_of_cpu_memory_to_use); +PD_DECLARE_uint64(initial_cpu_memory_in_mb); +PD_DECLARE_double(fraction_of_cuda_pinned_memory_to_use); // If use_pinned_memory is true, CPUAllocator calls mlock, which // returns pinned and locked memory as staging areas for data exchange diff --git a/paddle/phi/backends/device_base.cc b/paddle/phi/backends/device_base.cc index 2b7d0411fedcabfe4276025f6760af5595a34352..5b6b8fcfc2fe9a33c1fb0be047e33ad31c0d2bbf 100644 --- a/paddle/phi/backends/device_base.cc +++ b/paddle/phi/backends/device_base.cc @@ -14,13 +14,13 @@ #include "paddle/phi/backends/device_base.h" -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/core/enforce.h" +#include "paddle/utils/flags.h" -DECLARE_double(fraction_of_gpu_memory_to_use); -DECLARE_uint64(initial_gpu_memory_in_mb); -DECLARE_uint64(reallocate_gpu_memory_in_mb); +PD_DECLARE_double(fraction_of_gpu_memory_to_use); +PD_DECLARE_uint64(initial_gpu_memory_in_mb); +PD_DECLARE_uint64(reallocate_gpu_memory_in_mb); constexpr static float fraction_reserve_gpu_memory = 0.05f; diff --git a/paddle/phi/backends/dynload/cudnn_frontend.h b/paddle/phi/backends/dynload/cudnn_frontend.h index 4d0b67ce2285c20d5cd5d573f5b3534f1940b971..ef680d7cba035e2476061c0d66cb932581d9c8c1 100644 --- a/paddle/phi/backends/dynload/cudnn_frontend.h +++ b/paddle/phi/backends/dynload/cudnn_frontend.h @@ -14,13 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once -#include "gflags/gflags.h" #include "glog/logging.h" +#include "paddle/utils/flags.h" #include "paddle/phi/backends/dynload/cudnn.h" #include "paddle/phi/backends/gpu/gpu_info.h" -DECLARE_bool(enable_cudnn_frontend); +PD_DECLARE_bool(enable_cudnn_frontend); // Redirect the CUDNN APIs in the cudnn_frontend namespace to // the functions in phi::dynload diff --git a/paddle/phi/backends/dynload/dynamic_loader.cc b/paddle/phi/backends/dynload/dynamic_loader.cc index 4621a9c3ddafa9dde2e02a231a627226479ff2da..6989f32b18e9e04ff588977dd293b51df8a75011 100644 --- a/paddle/phi/backends/dynload/dynamic_loader.cc +++ b/paddle/phi/backends/dynload/dynamic_loader.cc @@ -103,7 +103,7 @@ PHI_DEFINE_string(rccl_dir, #endif #ifdef PADDLE_WITH_XPU -DEFINE_string(xpti_dir, "", "Specify path for loading libxpti.so."); +PD_DEFINE_string(xpti_dir, "", "Specify path for loading libxpti.so."); #endif namespace phi { diff --git a/paddle/phi/backends/gpu/cuda/cudnn_helper.h b/paddle/phi/backends/gpu/cuda/cudnn_helper.h index 8b6c04090d88e29a382ea04cd5d221ad2d257fba..651a4247a12df0d7e0d8545dbc0f4d6e63d2f0cd 100644 --- a/paddle/phi/backends/gpu/cuda/cudnn_helper.h +++ b/paddle/phi/backends/gpu/cuda/cudnn_helper.h @@ -17,7 +17,6 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "paddle/phi/backends/dynload/cudnn.h" #include "paddle/phi/common/bfloat16.h" #include "paddle/phi/common/float16.h" @@ -26,8 +25,9 @@ limitations under the License. */ #include "paddle/phi/core/enforce.h" #include "paddle/phi/core/errors.h" #include "paddle/phi/core/macros.h" +#include "paddle/utils/flags.h" -DECLARE_bool(cudnn_deterministic); +PD_DECLARE_bool(cudnn_deterministic); namespace phi { namespace backends { diff --git a/paddle/phi/backends/gpu/gpu_info.cc b/paddle/phi/backends/gpu/gpu_info.cc index 417ff4c72e86c907266e0b5d83c150cd64f87921..f6ca9d4168b2c86f57e242e9bdca715ecb48d2f4 100644 --- a/paddle/phi/backends/gpu/gpu_info.cc +++ b/paddle/phi/backends/gpu/gpu_info.cc @@ -17,12 +17,12 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "glog/logging.h" +#include "paddle/utils/flags.h" #include "paddle/phi/common/memory_utils.h" -DECLARE_string(selected_gpus); +PD_DECLARE_string(selected_gpus); namespace phi { namespace backends { diff --git a/paddle/phi/backends/gpu/rocm/miopen_helper.h b/paddle/phi/backends/gpu/rocm/miopen_helper.h index 095f32ba460d014dc2db92bd896d0b8afbef3289..b8ce6e22e939be017b7b6c07739b54efac6e634b 100644 --- a/paddle/phi/backends/gpu/rocm/miopen_helper.h +++ b/paddle/phi/backends/gpu/rocm/miopen_helper.h @@ -17,7 +17,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/phi/backends/dynload/miopen.h" #include "paddle/phi/common/bfloat16.h" @@ -31,7 +31,7 @@ limitations under the License. */ // MIOPEN do not have epslion definition #define CUDNN_BN_MIN_EPSILON 1e-05 -DECLARE_bool(cudnn_deterministic); +PD_DECLARE_bool(cudnn_deterministic); namespace phi { namespace backends { diff --git a/paddle/phi/core/enforce.cc b/paddle/phi/core/enforce.cc index 69bcbf91ef463a2f41af45efa1e43d19da95b950..f47eb91e97d97e98e34ed3b11dca375faa7bb735 100644 --- a/paddle/phi/core/enforce.cc +++ b/paddle/phi/core/enforce.cc @@ -20,16 +20,16 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/common/scalar.h" #include "paddle/utils/blank.h" +#include "paddle/utils/flags.h" #ifdef PADDLE_WITH_CUDA #include "paddle/phi/core/external_error.pb.h" #endif // PADDLE_WITH_CUDA -DECLARE_int32(call_stack_level); +PD_DECLARE_int32(call_stack_level); namespace egr { class EagerVariable; diff --git a/paddle/phi/core/flags.cc b/paddle/phi/core/flags.cc index 2470981f6617c93ac6d6a2c972dd8f667cdc9c54..41d2dc8003bb6be623bc1409e28741489796d870 100644 --- a/paddle/phi/core/flags.cc +++ b/paddle/phi/core/flags.cc @@ -1056,19 +1056,21 @@ PHI_DEFINE_EXPORTED_uint64(executor_log_deps_every_microseconds, 0, "Enable new executor log deps every n microseconds"); -DEFINE_int32(record_pool_max_size, - 2000000, - "SlotRecordDataset slot record pool max size"); -DEFINE_int32(slotpool_thread_num, 1, "SlotRecordDataset slot pool thread num"); -DEFINE_bool(enable_slotpool_wait_release, // NOLINT - false, - "enable slotrecord object wait release, default false"); -DEFINE_bool(enable_slotrecord_reset_shrink, // NOLINT - false, - "enable slotrecord object reset shrink memory, default false"); -DEFINE_bool(enable_ins_parser_file, // NOLINT - false, - "enable parser ins file, default false"); +PD_DEFINE_int32(record_pool_max_size, + 2000000, + "SlotRecordDataset slot record pool max size"); +PD_DEFINE_int32(slotpool_thread_num, + 1, + "SlotRecordDataset slot pool thread num"); +PD_DEFINE_bool(enable_slotpool_wait_release, // NOLINT + false, + "enable slotrecord object wait release, default false"); +PD_DEFINE_bool(enable_slotrecord_reset_shrink, // NOLINT + false, + "enable slotrecord object reset shrink memory, default false"); +PD_DEFINE_bool(enable_ins_parser_file, // NOLINT + false, + "enable parser ins file, default false"); PHI_DEFINE_EXPORTED_bool( gpugraph_enable_hbm_table_collision_stat, false, diff --git a/paddle/phi/core/flags.h b/paddle/phi/core/flags.h index 278090ff97d26748887bd7399921b261ec43ca70..776e268b201c7c1830e491159a32bf331562782c 100644 --- a/paddle/phi/core/flags.h +++ b/paddle/phi/core/flags.h @@ -20,9 +20,8 @@ #include #include -#include "gflags/gflags.h" #include "paddle/phi/core/macros.h" - +#include "paddle/utils/flags.h" #include "paddle/utils/variant.h" #if defined(_WIN32) @@ -33,6 +32,7 @@ #define PHI_IMPORT_FLAG #endif // _WIN32 +#ifdef PADDLE_WITH_GFLAGS // We redefine the gflags' macro for exporting global variable // ----------------------------DECLARE FLAGS---------------------------- @@ -127,6 +127,23 @@ clstring& FLAGS_##name = *FLAGS_no##name; \ } /* NOLINT */ \ using fLS::FLAGS_##name +#else // PADDLE_WITH_GFLAGS +#define PHI_DEFINE_bool(name, val, txt) PD_DEFINE_bool(name, val, txt) +#define PHI_DEFINE_int32(name, val, txt) PD_DEFINE_int32(name, val, txt) +#define PHI_DEFINE_uint32(name, val, txt) PD_DEFINE_uint32(name, val, txt) +#define PHI_DEFINE_int64(name, val, txt) PD_DEFINE_int64(name, val, txt) +#define PHI_DEFINE_uint64(name, val, txt) PD_DEFINE_uint64(name, val, txt) +#define PHI_DEFINE_double(name, val, txt) PD_DEFINE_double(name, val, txt) +#define PHI_DEFINE_string(name, val, txt) PD_DEFINE_string(name, val, txt) + +#define PHI_DECLARE_bool(name) PD_DECLARE_bool(name) +#define PHI_DECLARE_int32(name) PD_DECLARE_int32(name) +#define PHI_DECLARE_uint32(name) PD_DECLARE_uint32(name) +#define PHI_DECLARE_int64(name) PD_DECLARE_int64(name) +#define PHI_DECLARE_uint64(name) PD_DECLARE_uint64(name) +#define PHI_DECLARE_double(name) PD_DECLARE_double(name) +#define PHI_DECLARE_string(name) PD_DECLARE_string(name) +#endif namespace phi { diff --git a/paddle/phi/core/kernel_factory.cc b/paddle/phi/core/kernel_factory.cc index 543fe4fef7611ab0317ef2b77bf3c77860d1a1ce..3ead251e332207d75bfb4fefd40e1e05fa44a15c 100644 --- a/paddle/phi/core/kernel_factory.cc +++ b/paddle/phi/core/kernel_factory.cc @@ -14,9 +14,9 @@ #include "paddle/phi/core/kernel_factory.h" -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/core/enforce.h" +#include "paddle/utils/flags.h" #if defined(PADDLE_WITH_XPU) #include "paddle/phi/backends/xpu/xpu_op_list.h" #include "paddle/phi/common/data_type.h" @@ -34,9 +34,9 @@ PADDLE_DEFINE_EXPORTED_bool( true, "Whether to use strdie kernel if op support stride."); -DECLARE_int32(low_precision_op_list); -DECLARE_bool(enable_api_kernel_fallback); -DECLARE_bool(run_kp_kernel); +PD_DECLARE_int32(low_precision_op_list); +PD_DECLARE_bool(enable_api_kernel_fallback); +PD_DECLARE_bool(run_kp_kernel); namespace phi { const static Kernel empty_kernel; // NOLINT diff --git a/paddle/phi/core/threadpool.cc b/paddle/phi/core/threadpool.cc index 5e146023c29cecd1ec5499c4305d95bd152252d4..7538087f4e855342cca9b642af2cb06146bad904 100644 --- a/paddle/phi/core/threadpool.cc +++ b/paddle/phi/core/threadpool.cc @@ -16,14 +16,14 @@ #include -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/core/enforce.h" +#include "paddle/utils/flags.h" -DECLARE_int32(dist_threadpool_size); -DEFINE_int32(io_threadpool_size, - 100, - "number of threads used for doing IO, default 100"); +PD_DECLARE_int32(dist_threadpool_size); +PD_DEFINE_int32(io_threadpool_size, + 100, + "number of threads used for doing IO, default 100"); namespace phi { diff --git a/paddle/phi/infermeta/unary.cc b/paddle/phi/infermeta/unary.cc index 6e04a149266ac19f06496b74a7027292aa56a320..4c952bb3cd2bc1ec5bc811dcf9ac8e87bcf0c725 100644 --- a/paddle/phi/infermeta/unary.cc +++ b/paddle/phi/infermeta/unary.cc @@ -17,7 +17,6 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "paddle/phi/common/data_type.h" #include "paddle/phi/common/type_traits.h" #include "paddle/phi/core/enforce.h" @@ -31,6 +30,7 @@ limitations under the License. */ #include "paddle/phi/kernels/funcs/unfold_functor.h" #include "paddle/phi/kernels/funcs/unsqueeze.h" #include "paddle/phi/kernels/impl/einsum_impl.h" +#include "paddle/utils/flags.h" namespace phi { diff --git a/paddle/phi/kernels/autotune/cache_cudnn_frontend.h b/paddle/phi/kernels/autotune/cache_cudnn_frontend.h index 095cedccb991c2bf0c34bef520180fd65188c733..4715efa1f776e534a771ca687e965b02d1930117 100644 --- a/paddle/phi/kernels/autotune/cache_cudnn_frontend.h +++ b/paddle/phi/kernels/autotune/cache_cudnn_frontend.h @@ -22,7 +22,7 @@ #include "paddle/phi/backends/dynload/cudnn_frontend.h" -DECLARE_int32(cudnn_cache_saturation_count); +PD_DECLARE_int32(cudnn_cache_saturation_count); namespace phi { namespace autotune { diff --git a/paddle/phi/kernels/autotune/switch_autotune.cc b/paddle/phi/kernels/autotune/switch_autotune.cc index 3742749b3bf0329da28c454d7c0222adea08c21d..e287705d08b79b897b4df4cccdcfc6e54463e1cd 100644 --- a/paddle/phi/kernels/autotune/switch_autotune.cc +++ b/paddle/phi/kernels/autotune/switch_autotune.cc @@ -14,10 +14,10 @@ #include "paddle/phi/kernels/autotune/switch_autotune.h" -#include "gflags/gflags.h" #include "glog/logging.h" +#include "paddle/utils/flags.h" -DECLARE_bool(use_autotune); +PD_DECLARE_bool(use_autotune); namespace phi { namespace autotune { diff --git a/paddle/phi/kernels/cpu/adam_kernel.cc b/paddle/phi/kernels/cpu/adam_kernel.cc index 083c9dab74001c5119565d315151059fc4e2527d..1a63b779b02a19eca28df8f8f33ef38752377b78 100644 --- a/paddle/phi/kernels/cpu/adam_kernel.cc +++ b/paddle/phi/kernels/cpu/adam_kernel.cc @@ -24,7 +24,7 @@ #include "paddle/phi/kernels/funcs/adam_functors.h" #include "paddle/phi/kernels/funcs/jit/kernels.h" -DECLARE_int32(inner_op_parallelism); +PD_DECLARE_int32(inner_op_parallelism); namespace phi { diff --git a/paddle/phi/kernels/funcs/blas/blas_impl.cu.h b/paddle/phi/kernels/funcs/blas/blas_impl.cu.h index 84eea97da9f510cdae08992db24bc374b7017cb0..6e4f7e22781f13ecd76825de2e075ec21ab7d605 100644 --- a/paddle/phi/kernels/funcs/blas/blas_impl.cu.h +++ b/paddle/phi/kernels/funcs/blas/blas_impl.cu.h @@ -17,8 +17,8 @@ #if defined(__NVCC__) #include #endif -#include "gflags/gflags.h" #include "glog/logging.h" +#include "paddle/utils/flags.h" #include "paddle/phi/backends/dynload/cublas.h" #include "paddle/phi/backends/gpu/gpu_context.h" diff --git a/paddle/phi/kernels/funcs/blas/blas_impl.hip.h b/paddle/phi/kernels/funcs/blas/blas_impl.hip.h index 805a718ab85ed443a29c182fbba19fdb0a45bf0e..224cf4d6cb49707e0f4085da5c11e759640bf444 100644 --- a/paddle/phi/kernels/funcs/blas/blas_impl.hip.h +++ b/paddle/phi/kernels/funcs/blas/blas_impl.hip.h @@ -14,13 +14,13 @@ #pragma once -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/phi/backends/dynload/rocblas.h" #include "paddle/phi/backends/gpu/gpu_context.h" #include "paddle/phi/kernels/funcs/math_function.h" -DECLARE_bool(enable_cublas_tensor_op_math); +PD_DECLARE_bool(enable_cublas_tensor_op_math); namespace phi { namespace funcs { diff --git a/paddle/phi/kernels/funcs/fused_gemm_epilogue.h b/paddle/phi/kernels/funcs/fused_gemm_epilogue.h index 6f4eb46bf4eb740e1cf8b671a9cbdbcc9cdab672..eb5f0fa540f8d36cf0ce928c6a464732cc6b32cf 100644 --- a/paddle/phi/kernels/funcs/fused_gemm_epilogue.h +++ b/paddle/phi/kernels/funcs/fused_gemm_epilogue.h @@ -26,7 +26,6 @@ limitations under the License. */ #if CUDA_VERSION >= 11060 -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/backends/all_context.h" #include "paddle/phi/backends/dynload/cublasLt.h" @@ -38,9 +37,10 @@ limitations under the License. */ #include "paddle/phi/core/enforce.h" #include "paddle/phi/core/scope_guard.h" #include "paddle/phi/kernels/funcs/blas/blaslt_impl.cu.h" +#include "paddle/utils/flags.h" #include "paddle/utils/optional.h" -DECLARE_int64(cublaslt_exhaustive_search_times); +PD_DECLARE_int64(cublaslt_exhaustive_search_times); namespace phi { namespace funcs { diff --git a/paddle/phi/kernels/funcs/jit/benchmark.cc b/paddle/phi/kernels/funcs/jit/benchmark.cc index ad7146db0d93a6d052fcaf87b01226fbe3ecb580..d5d1da7385797fbe615fed0c48724ae92f53414f 100644 --- a/paddle/phi/kernels/funcs/jit/benchmark.cc +++ b/paddle/phi/kernels/funcs/jit/benchmark.cc @@ -15,18 +15,18 @@ #include #include -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/api/profiler/device_tracer.h" #include "paddle/phi/common/place.h" #include "paddle/phi/core/dense_tensor.h" #include "paddle/phi/core/enforce.h" #include "paddle/phi/kernels/funcs/jit/kernels.h" +#include "paddle/utils/flags.h" -DEFINE_int32(burning, 10, "Burning times."); -DEFINE_int32(repeat, 3000, "Repeat times."); -DEFINE_int32(max_size, 1000, "The Max size would be tested."); -DEFINE_string(filter, "", "The Benchmark name would be run."); // NOLINT +PD_DEFINE_int32(burning, 10, "Burning times."); +PD_DEFINE_int32(repeat, 3000, "Repeat times."); +PD_DEFINE_int32(max_size, 1000, "The Max size would be tested."); +PD_DEFINE_string(filter, "", "The Benchmark name would be run."); // NOLINT class BenchJITKernel { public: @@ -546,7 +546,7 @@ BENCH_FP32_CPU(VBroadcast); // --max_size: the max size would be tested // --filter: the bench name would be run int main(int argc, char* argv[]) { - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + paddle::flags::ParseCommandLineFlags(&argc, &argv); google::InitGoogleLogging(argv[0]); LOG(INFO) << "Burning " << FLAGS_burning << " times, Repeat " << FLAGS_repeat << " times."; diff --git a/paddle/phi/kernels/funcs/jit/gen_base.h b/paddle/phi/kernels/funcs/jit/gen_base.h index dfad19eff3448226cc5119bc1b9ece6862ad862a..f96f08005ab033e1cc2fdbb9cd2865c349d89428 100644 --- a/paddle/phi/kernels/funcs/jit/gen_base.h +++ b/paddle/phi/kernels/funcs/jit/gen_base.h @@ -22,9 +22,9 @@ #include // for _aligned_malloc #endif -#include "gflags/gflags.h" #include "paddle/phi/core/flags.h" #include "paddle/phi/kernels/funcs/jit/kernel_base.h" +#include "paddle/utils/flags.h" PHI_DECLARE_bool(dump_jitcode); diff --git a/paddle/phi/kernels/funcs/jit/test.cc b/paddle/phi/kernels/funcs/jit/test.cc index 0dd5f6c6ba5aef8aa4669da9e2989cb5c2a5c04e..d388d95975cff97ac7580221d7205aeace203cd6 100644 --- a/paddle/phi/kernels/funcs/jit/test.cc +++ b/paddle/phi/kernels/funcs/jit/test.cc @@ -16,15 +16,15 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "glog/logging.h" #include "gtest/gtest.h" #include "paddle/phi/backends/cpu/cpu_info.h" #include "paddle/phi/common/place.h" #include "paddle/phi/core/enforce.h" #include "paddle/phi/kernels/funcs/jit/kernels.h" +#include "paddle/utils/flags.h" -DEFINE_double(acc, 1e-5, "Test accuracy threshold."); +PD_DEFINE_double(acc, 1e-5, "Test accuracy threshold."); template void RandomVec(const int n, diff --git a/paddle/phi/kernels/gpu/batch_norm_grad_kernel.cu b/paddle/phi/kernels/gpu/batch_norm_grad_kernel.cu index 0609bf945d9b38343212a51f4aae9635a0e6bd7b..31bad0987ba2707732244c8ceaa9ad50863af280 100644 --- a/paddle/phi/kernels/gpu/batch_norm_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/batch_norm_grad_kernel.cu @@ -34,7 +34,7 @@ #define LAUNCH_BOUNDS(BlockDim) #endif -DECLARE_bool(cudnn_batchnorm_spatial_persistent); +PD_DECLARE_bool(cudnn_batchnorm_spatial_persistent); namespace phi { template diff --git a/paddle/phi/kernels/gpu/batch_norm_kernel.cu b/paddle/phi/kernels/gpu/batch_norm_kernel.cu index 1a07e5f0d49098cae77b47dd0ee0dc67b5321cce..ad276ec6f1812bea8385a306417514ffbbd3a846 100644 --- a/paddle/phi/kernels/gpu/batch_norm_kernel.cu +++ b/paddle/phi/kernels/gpu/batch_norm_kernel.cu @@ -41,7 +41,7 @@ namespace cub = hipcub; #define LAUNCH_BOUNDS(BlockDim) #endif -DECLARE_bool(cudnn_batchnorm_spatial_persistent); +PD_DECLARE_bool(cudnn_batchnorm_spatial_persistent); namespace phi { diff --git a/paddle/phi/kernels/gpu/c_embedding_grad_kernel.cu b/paddle/phi/kernels/gpu/c_embedding_grad_kernel.cu index aaa5c6865be4746eb705471de35026019e9911ea..66057db357e50a8f350f447bdcbfdf447e4a3ed9 100644 --- a/paddle/phi/kernels/gpu/c_embedding_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/c_embedding_grad_kernel.cu @@ -14,7 +14,6 @@ #include "paddle/phi/kernels/c_embedding_kernel.h" -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/backends/gpu/gpu_context.h" @@ -22,8 +21,9 @@ #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/funcs/eigen/common.h" #include "paddle/phi/kernels/funcs/embedding_grad.h" +#include "paddle/utils/flags.h" -DECLARE_int64(embedding_deterministic); +PD_DECLARE_int64(embedding_deterministic); namespace phi { diff --git a/paddle/phi/kernels/gpu/embedding_grad_kernel.cu b/paddle/phi/kernels/gpu/embedding_grad_kernel.cu index 99ba12b1d62135c556fd64a139d66e17166c53dd..a7c75e64a462ad5a575ad88b2f1f1d676083b07a 100644 --- a/paddle/phi/kernels/gpu/embedding_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/embedding_grad_kernel.cu @@ -15,7 +15,6 @@ #include "paddle/phi/kernels/embedding_grad_kernel.h" #include "paddle/phi/kernels/funcs/embedding_grad.h" -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/backends/gpu/gpu_context.h" #include "paddle/phi/backends/gpu/gpu_primitives.h" @@ -26,8 +25,9 @@ #include "paddle/phi/core/mixed_vector.h" #include "paddle/phi/kernels/funcs/eigen/common.h" #include "paddle/phi/kernels/funcs/embedding_util.h" +#include "paddle/utils/flags.h" -DECLARE_int64(embedding_deterministic); +PD_DECLARE_int64(embedding_deterministic); namespace phi { diff --git a/paddle/phi/kernels/gpu/flash_attn_grad_kernel.cu b/paddle/phi/kernels/gpu/flash_attn_grad_kernel.cu index 60a1c54d72678e8d734ad5a4bbae03d4997ac734..7b76a5f458dddf35f069250aa937888ed70acb3e 100644 --- a/paddle/phi/kernels/gpu/flash_attn_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/flash_attn_grad_kernel.cu @@ -24,7 +24,7 @@ #include "paddle/phi/kernels/gpu/flash_attn_utils.h" #include "paddle/phi/kernels/reshape_kernel.h" -DECLARE_bool(cudnn_deterministic); +PD_DECLARE_bool(cudnn_deterministic); namespace phi { diff --git a/paddle/phi/kernels/gpu/flash_attn_kernel.cu b/paddle/phi/kernels/gpu/flash_attn_kernel.cu index bcf8791d3c17f695bbb378162c883d4a0b0a49a7..4bd5e28c09fedc2cc6b4fa79b11db32aabb7568e 100644 --- a/paddle/phi/kernels/gpu/flash_attn_kernel.cu +++ b/paddle/phi/kernels/gpu/flash_attn_kernel.cu @@ -24,7 +24,7 @@ #include "paddle/phi/kernels/gpu/flash_attn_utils.h" #include "paddle/phi/kernels/reshape_kernel.h" -DECLARE_bool(cudnn_deterministic); +PD_DECLARE_bool(cudnn_deterministic); namespace phi { diff --git a/paddle/phi/kernels/gpu/gelu_grad_kernel.cu b/paddle/phi/kernels/gpu/gelu_grad_kernel.cu index b1ffa921f912b760480d3a7a63a7f1db7ac9360c..2b847fb216bb9c710cf593df2cf072af176cf844 100644 --- a/paddle/phi/kernels/gpu/gelu_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/gelu_grad_kernel.cu @@ -21,7 +21,7 @@ #include "paddle/phi/kernels/funcs/broadcast_function.h" #include "paddle/phi/kernels/gpu/gelu_funcs.h" -DECLARE_bool(use_fast_math); +PD_DECLARE_bool(use_fast_math); namespace phi { diff --git a/paddle/phi/kernels/gpu/gelu_kernel.cu b/paddle/phi/kernels/gpu/gelu_kernel.cu index e0792c387d7510c7fdf7016bb7a0940462e13a66..8400b5d8cd5a5f2f28fdada5c8b604f3d067d81f 100644 --- a/paddle/phi/kernels/gpu/gelu_kernel.cu +++ b/paddle/phi/kernels/gpu/gelu_kernel.cu @@ -25,7 +25,7 @@ #include "paddle/phi/kernels/gpu/gelu_funcs.h" // clang-format on -DECLARE_bool(use_fast_math); +PD_DECLARE_bool(use_fast_math); namespace phi { diff --git a/paddle/phi/kernels/gpu/index_add_kernel.cu b/paddle/phi/kernels/gpu/index_add_kernel.cu index 9ba5bde7f61360fb40e20fe6647f900482df2acd..8fd15d5435f98bdae0489c4cd7465e33ece28391 100644 --- a/paddle/phi/kernels/gpu/index_add_kernel.cu +++ b/paddle/phi/kernels/gpu/index_add_kernel.cu @@ -14,15 +14,15 @@ #include "paddle/phi/kernels/index_add_kernel.h" -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/backends/gpu/gpu_info.h" #include "paddle/phi/backends/gpu/gpu_launch_config.h" #include "paddle/phi/backends/gpu/gpu_primitives.h" #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/core/utils/data_type.h" +#include "paddle/utils/flags.h" -DECLARE_bool(cudnn_deterministic); +PD_DECLARE_bool(cudnn_deterministic); namespace phi { diff --git a/paddle/phi/kernels/gpu/index_select_grad_kernel.cu b/paddle/phi/kernels/gpu/index_select_grad_kernel.cu index 64e3428cc9aaf82e624dfc90d699c5db990612eb..03f74888fca0f62099f97ff87a70f6dc3e473014 100644 --- a/paddle/phi/kernels/gpu/index_select_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/index_select_grad_kernel.cu @@ -14,7 +14,6 @@ #include "paddle/phi/kernels/index_select_grad_kernel.h" -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/backends/gpu/gpu_info.h" #include "paddle/phi/backends/gpu/gpu_launch_config.h" @@ -22,8 +21,9 @@ #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/core/utils/data_type.h" #include "paddle/phi/kernels/funcs/math_function.h" +#include "paddle/utils/flags.h" -DECLARE_bool(cudnn_deterministic); +PD_DECLARE_bool(cudnn_deterministic); namespace phi { diff --git a/paddle/phi/kernels/gpu/layer_norm_kernel.cu b/paddle/phi/kernels/gpu/layer_norm_kernel.cu index c5bb0c288f260968c7f163ccbc8db63c2ef45ba8..eb85d9ac826d0aa278b65882491c9d6330649fde 100644 --- a/paddle/phi/kernels/gpu/layer_norm_kernel.cu +++ b/paddle/phi/kernels/gpu/layer_norm_kernel.cu @@ -13,13 +13,13 @@ // limitations under the License. #include "paddle/phi/kernels/layer_norm_kernel.h" -#include "gflags/gflags.h" #include "paddle/phi/backends/gpu/gpu_context.h" #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/funcs/layer_norm_impl.cu.h" #include "paddle/phi/kernels/funcs/layer_norm_util.h" +#include "paddle/utils/flags.h" -DECLARE_bool(use_fast_math); +PD_DECLARE_bool(use_fast_math); namespace phi { diff --git a/paddle/phi/kernels/gpu/randperm_kernel.cu b/paddle/phi/kernels/gpu/randperm_kernel.cu index 2ad512701e097f07fc698c151410fb296e170d12..4c6597b93f91fdea8ba26ab4511c428d3b78ecfc 100644 --- a/paddle/phi/kernels/gpu/randperm_kernel.cu +++ b/paddle/phi/kernels/gpu/randperm_kernel.cu @@ -26,7 +26,6 @@ namespace cub = hipcub; #endif -#include "gflags/gflags.h" #include "paddle/phi/backends/gpu/gpu_launch_config.h" #include "paddle/phi/common/amp_type_traits.h" #include "paddle/phi/common/memory_utils.h" @@ -34,6 +33,7 @@ namespace cub = hipcub; #include "paddle/phi/kernels/empty_kernel.h" #include "paddle/phi/kernels/funcs/for_range.h" #include "paddle/phi/kernels/randint_kernel.h" +#include "paddle/utils/flags.h" namespace phi { diff --git a/paddle/phi/kernels/gpu/uniform_inplace_kernel.cu b/paddle/phi/kernels/gpu/uniform_inplace_kernel.cu index 653a64b127a254596ea9c8da8a29fa6e21e5854b..87bae7fe5647fa334be3a8f926638807d1aefb21 100644 --- a/paddle/phi/kernels/gpu/uniform_inplace_kernel.cu +++ b/paddle/phi/kernels/gpu/uniform_inplace_kernel.cu @@ -16,11 +16,11 @@ limitations under the License. */ #include -#include "gflags/gflags.h" #include "paddle/phi/common/amp_type_traits.h" #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/funcs/distribution_helper.h" #include "paddle/phi/kernels/funcs/index_impl.cu.h" +#include "paddle/utils/flags.h" namespace phi { diff --git a/paddle/phi/kernels/gpu/uniform_kernel.cu b/paddle/phi/kernels/gpu/uniform_kernel.cu index 04217db0a74c1a4362eee79b3a2227216c268a42..2a514947bb71771df13834edcce052a996284caa 100644 --- a/paddle/phi/kernels/gpu/uniform_kernel.cu +++ b/paddle/phi/kernels/gpu/uniform_kernel.cu @@ -16,10 +16,10 @@ #include -#include "gflags/gflags.h" #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/funcs/distribution_helper.h" #include "paddle/phi/kernels/funcs/index_impl.cu.h" +#include "paddle/utils/flags.h" namespace phi { diff --git a/paddle/phi/kernels/gpudnn/conv_gpudnn_info.h b/paddle/phi/kernels/gpudnn/conv_gpudnn_info.h index 9b7b35f4357cd41fc50ab9e7e7d244260d7efa66..0dd3d7f0cdea1bae0e961496ca7f2f30802bcf4e 100644 --- a/paddle/phi/kernels/gpudnn/conv_gpudnn_info.h +++ b/paddle/phi/kernels/gpudnn/conv_gpudnn_info.h @@ -20,9 +20,9 @@ limitations under the License. */ #include "paddle/phi/backends/gpu/gpu_dnn.h" -DECLARE_int64(conv_workspace_size_limit); -DECLARE_bool(cudnn_exhaustive_search); -DECLARE_int64(cudnn_exhaustive_search_times); +PD_DECLARE_int64(conv_workspace_size_limit); +PD_DECLARE_bool(cudnn_exhaustive_search); +PD_DECLARE_int64(cudnn_exhaustive_search_times); namespace phi { diff --git a/paddle/phi/kernels/impl/conv_cudnn_impl.h b/paddle/phi/kernels/impl/conv_cudnn_impl.h index c918eeec831219515dd2d79ee917603a8289c3ca..81a7c7726203ea2f0bd3013b13420ac5e524c0de 100644 --- a/paddle/phi/kernels/impl/conv_cudnn_impl.h +++ b/paddle/phi/kernels/impl/conv_cudnn_impl.h @@ -30,9 +30,9 @@ #include "paddle/phi/kernels/funcs/batch_norm_utils.h" #include "paddle/phi/kernels/funcs/padding.h" -DECLARE_bool(cudnn_deterministic); -DECLARE_int64(conv_workspace_size_limit); -DECLARE_bool(cudnn_exhaustive_search); +PD_DECLARE_bool(cudnn_deterministic); +PD_DECLARE_int64(conv_workspace_size_limit); +PD_DECLARE_bool(cudnn_exhaustive_search); namespace phi { diff --git a/paddle/phi/kernels/impl/einsum_impl.h b/paddle/phi/kernels/impl/einsum_impl.h index d297e786845bc16a834048b037ed74e8e201ed9a..e32f64f347f4c62208ae1ed41a587c23dc966602 100644 --- a/paddle/phi/kernels/impl/einsum_impl.h +++ b/paddle/phi/kernels/impl/einsum_impl.h @@ -15,8 +15,8 @@ #include -#include "gflags/gflags.h" #include "glog/logging.h" +#include "paddle/utils/flags.h" #include "paddle/phi/core/dense_tensor.h" #include "paddle/phi/kernels/diagonal_kernel.h" @@ -27,7 +27,7 @@ #include "paddle/phi/kernels/transpose_kernel.h" #include "paddle/utils/string/string_helper.h" -DECLARE_bool(einsum_opt); +PD_DECLARE_bool(einsum_opt); namespace phi { diff --git a/paddle/phi/kernels/legacy/gpu/uniform_kernel.cu b/paddle/phi/kernels/legacy/gpu/uniform_kernel.cu index 609238435c96f1e035bf311cf059f0ddae89879d..abf51cf61f2b5c3e8bc057557fdbaf4d31de9e77 100644 --- a/paddle/phi/kernels/legacy/gpu/uniform_kernel.cu +++ b/paddle/phi/kernels/legacy/gpu/uniform_kernel.cu @@ -16,10 +16,10 @@ #include -#include "gflags/gflags.h" #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/funcs/distribution_helper.h" #include "paddle/phi/kernels/funcs/index_impl.cu.h" +#include "paddle/utils/flags.h" namespace phi { diff --git a/paddle/phi/kernels/selected_rows/cpu/adam_kernel.cc b/paddle/phi/kernels/selected_rows/cpu/adam_kernel.cc index c4732db041edf7370235da4c328586e962332559..b93975c188b01e42aab9de716696d30c07ee1918 100644 --- a/paddle/phi/kernels/selected_rows/cpu/adam_kernel.cc +++ b/paddle/phi/kernels/selected_rows/cpu/adam_kernel.cc @@ -14,8 +14,8 @@ #include "paddle/phi/kernels/selected_rows/adam_kernel.h" -#include "gflags/gflags.h" #include "glog/logging.h" +#include "paddle/utils/flags.h" #include "paddle/phi/backends/cpu/cpu_context.h" #include "paddle/phi/core/kernel_registry.h" @@ -24,7 +24,7 @@ #include "paddle/phi/kernels/funcs/adam_functors.h" #include "paddle/phi/kernels/funcs/selected_rows_functor.h" -DECLARE_int32(inner_op_parallelism); +PD_DECLARE_int32(inner_op_parallelism); namespace phi { namespace sr { diff --git a/paddle/testing/paddle_gtest_main.cc b/paddle/testing/paddle_gtest_main.cc index b22f0c3a450497bfc03bbffee69d3b638c3fbf2c..6701693545e49e92dd007ce5dd65546f05521a07 100644 --- a/paddle/testing/paddle_gtest_main.cc +++ b/paddle/testing/paddle_gtest_main.cc @@ -12,15 +12,15 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ -#include "gflags/gflags.h" #include "gtest/gtest.h" #include "paddle/fluid/framework/phi_utils.h" #include "paddle/fluid/memory/allocation/allocator_strategy.h" #include "paddle/fluid/platform/init.h" #include "paddle/phi/core/flags.h" +#include "paddle/utils/flags.h" #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) -DECLARE_bool(enable_gpu_memory_usage_log); +PD_DECLARE_bool(enable_gpu_memory_usage_log); #endif int main(int argc, char** argv) { // NOLINT @@ -32,11 +32,8 @@ int main(int argc, char** argv) { // NOLINT } std::vector envs; - std::vector undefok; #if defined(PADDLE_WITH_DISTRIBUTE) && !defined(PADDLE_WITH_PSLIB) - std::string str_max_body_size; - if (::GFLAGS_NAMESPACE::GetCommandLineOption("max_body_size", - &str_max_body_size)) { + if (paddle::flags::FindFlag("max_body_size")) { setenv("FLAGS_max_body_size", "2147483647", 1); envs.push_back("max_body_size"); } @@ -45,18 +42,8 @@ int main(int argc, char** argv) { // NOLINT const auto& flag_map = phi::GetExportedFlagInfoMap(); for (const auto& pair : flag_map) { const std::string& name = pair.second.name; - // NOTE(zhiqiu): some names may not linked in some tests, so add to - // `undefok`. - // One way to handle that is to check each flag item by item, and put it in - // `envs` or `undefok`; - // another way is to add all flags to `envs` and `undeok`, basically it is - // not a good design, - // but it can simplify the procedure of creating new flag and seems no side - // effects. - // see details: https://gflags.github.io/gflags/#special if (pair.second.is_writable) { // means public envs.push_back(name); - undefok.push_back(name); } } @@ -72,20 +59,8 @@ int main(int argc, char** argv) { // NOLINT VLOG(1) << "gtest env_string:" << env_string; } - char* undefok_str = nullptr; - if (!undefok.empty()) { - std::string undefok_string = "--undefok="; - for (auto t : undefok) { - undefok_string += t + ","; - } - undefok_string = undefok_string.substr(0, undefok_string.length() - 1); - undefok_str = strdup(undefok_string.c_str()); - new_argv.push_back(undefok_str); - VLOG(1) << "gtest undefok_string:" << undefok_string; - } - #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) - if (strstr(undefok_str, "enable_gpu_memory_usage_log")) { + if (strstr(env_str, "enable_gpu_memory_usage_log")) { VLOG(1) << "Set FLAGS_enable_gpu_memory_usage_log to true"; FLAGS_enable_gpu_memory_usage_log = true; } @@ -93,8 +68,8 @@ int main(int argc, char** argv) { // NOLINT int new_argc = static_cast(new_argv.size()); char** new_argv_address = new_argv.data(); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags( - &new_argc, &new_argv_address, false); + paddle::flags::AllowUndefinedFlags(); + paddle::flags::ParseCommandLineFlags(&new_argc, &new_argv_address); paddle::framework::InitMemoryMethod(); paddle::framework::InitDevices(); paddle::framework::InitDefaultKernelSignatureMap(); @@ -102,6 +77,5 @@ int main(int argc, char** argv) { // NOLINT int ret = RUN_ALL_TESTS(); if (env_str) free(env_str); - if (undefok_str) free(undefok_str); return ret; } diff --git a/paddle/utils/CMakeLists.txt b/paddle/utils/CMakeLists.txt index dbb53a3ac368a2f6042335f607ba17f3dab5b5c1..bb177e7578bb40c4efc31c209ae71def86836008 100644 --- a/paddle/utils/CMakeLists.txt +++ b/paddle/utils/CMakeLists.txt @@ -23,3 +23,11 @@ if(NOT ((NOT WITH_PYTHON) AND ON_INFER)) SRCS pybind.cc DEPS phi) endif() + +if(NOT WITH_GFLAGS) + cc_library(paddle_flags SRCS flags_native.cc) + cc_test( + flags_native_test + SRCS flags_native_test.cc + DEPS paddle_flags) +endif() diff --git a/paddle/utils/flags.h b/paddle/utils/flags.h new file mode 100644 index 0000000000000000000000000000000000000000..5a019f1439b434082b34ddd307c4b471ca46a950 --- /dev/null +++ b/paddle/utils/flags.h @@ -0,0 +1,77 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#ifdef PADDLE_WITH_GFLAGS +#include "gflags/gflags.h" +#else +#include "paddle/utils/flags_native.h" +#endif + +#ifdef PADDLE_WITH_GFLAGS +#define PD_DEFINE_bool(name, val, txt) DEFINE_bool(name, val, txt) +#define PD_DEFINE_int32(name, val, txt) DEFINE_int32(name, val, txt) +#define PD_DEFINE_uint32(name, val, txt) DEFINE_uint32(name, val, txt) +#define PD_DEFINE_int64(name, val, txt) DEFINE_int64(name, val, txt) +#define PD_DEFINE_uint64(name, val, txt) DEFINE_uint64(name, val, txt) +#define PD_DEFINE_double(name, val, txt) DEFINE_double(name, val, txt) +#define PD_DEFINE_string(name, val, txt) DEFINE_string(name, val, txt) + +#define PD_DECLARE_bool(name) DECLARE_bool(name) +#define PD_DECLARE_int32(name) DECLARE_int32(name) +#define PD_DECLARE_uint32(name) DECLARE_uint32(name) +#define PD_DECLARE_int64(name) DECLARE_int64(name) +#define PD_DECLARE_uint64(name) DECLARE_uint64(name) +#define PD_DECLARE_double(name) DECLARE_double(name) +#define PD_DECLARE_string(name) DECLARE_string(name) +#endif + +namespace paddle { +namespace flags { + +#ifdef PADDLE_WITH_GFLAGS +inline void ParseCommandLineFlags(int* argc, char*** argv) { + gflags::ParseCommandLineFlags(argc, argv, true); +} +#else +using paddle::flags::ParseCommandLineFlags; +#endif + +#ifdef PADDLE_WITH_GFLAGS +inline bool SetFlagValue(const char* name, const char* value) { + std::string ret = gflags::SetCommandLineOption(name, value); + return ret.empty() ? false : true; +} +#else +using paddle::flags::SetFlagValue; +#endif + +#ifdef PADDLE_WITH_GFLAGS +inline bool FindFlag(const char* name) { + std::string value; + return gflags::GetCommandLineOption(name, &value); +} +#else +using paddle::flags::FindFlag; +#endif + +#ifdef PADDLE_WITH_GFLAGS +inline void AllowUndefinedFlags() { gflags::AllowCommandLineReparsing(); } +#else +using paddle::flags::AllowUndefinedFlags; +#endif + +} // namespace flags +} // namespace paddle diff --git a/paddle/utils/flags_native.cc b/paddle/utils/flags_native.cc new file mode 100644 index 0000000000000000000000000000000000000000..05d90d8adf21cc7ff3181fc7afc4271de49f40a9 --- /dev/null +++ b/paddle/utils/flags_native.cc @@ -0,0 +1,484 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/utils/flags_native.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace paddle { +namespace flags { + +std::stringstream& ErrorStream() { + static std::stringstream err_ss; + return err_ss; +} + +inline void exit_with_errors() { + std::cerr << ErrorStream().str(); + exit(-1); +} + +#define LOG_FLAG_ERROR(message) \ + ErrorStream() << "paddle flags error: " << message << " (at " << __FILE__ \ + << ":" << __LINE__ << ")" << std::endl + +#define LOG_FLAG_FATAL_ERROR(message) \ + LOG_FLAG_ERROR(message); \ + exit_with_errors() + +enum class FlagType : uint8_t { + BOOL = 0, + INT32 = 1, + UINT32 = 2, + INT64 = 3, + UINT64 = 4, + DOUBLE = 5, + STRING = 6, + UNDEFINED = 7, +}; + +class Flag { + public: + Flag(std::string name, + std::string description, + std::string file, + FlagType type, + const void* default_value, + void* value) + : name_(name), + description_(description), + file_(file), + type_(type), + default_value_(default_value), + value_(value) {} + ~Flag() = default; + + // Summary: --name_: type_, description_ (default: default_value_) + std::string Summary() const; + + void SetValueFromString(const std::string& value); + + private: + friend class FlagRegistry; + + const std::string name_; + const std::string description_; + const std::string file_; + const FlagType type_; + const void* default_value_; + void* value_; +}; + +class FlagRegistry { + public: + static FlagRegistry* Instance() { + static FlagRegistry* global_registry_ = new FlagRegistry(); + return global_registry_; + } + + void RegisterFlag(Flag* flag); + + bool SetFlagValue(const std::string& name, const std::string& value); + + bool HasFlag(const std::string& name) const; + + void PrintAllFlagHelp(std::ostream& os) const; + + private: + FlagRegistry() = default; + + std::map flags_; + + struct FlagCompare { + bool operator()(const Flag* flag1, const Flag* flag2) const { + return flag1->name_ < flag2->name_; + } + }; + + std::map> flags_by_file_; + + std::mutex mutex_; +}; + +template +struct FlagTypeTraits { + static constexpr FlagType Type = FlagType::UNDEFINED; +}; + +#define DEFINE_FLAG_TYPE_TRAITS(type, flag_type) \ + template <> \ + struct FlagTypeTraits { \ + static constexpr FlagType Type = flag_type; \ + } + +DEFINE_FLAG_TYPE_TRAITS(bool, FlagType::BOOL); +DEFINE_FLAG_TYPE_TRAITS(int32_t, FlagType::INT32); +DEFINE_FLAG_TYPE_TRAITS(uint32_t, FlagType::UINT32); +DEFINE_FLAG_TYPE_TRAITS(int64_t, FlagType::INT64); +DEFINE_FLAG_TYPE_TRAITS(uint64_t, FlagType::UINT64); +DEFINE_FLAG_TYPE_TRAITS(double, FlagType::DOUBLE); +DEFINE_FLAG_TYPE_TRAITS(std::string, FlagType::STRING); + +#undef DEFINE_FLAG_TYPE_TRAITS + +template +FlagRegisterer::FlagRegisterer(std::string name, + std::string help, + std::string file, + const T* default_value, + T* value) { + FlagType type = FlagTypeTraits::Type; + Flag* flag = new Flag(name, help, file, type, default_value, value); + FlagRegistry::Instance()->RegisterFlag(flag); +} + +// Instantiate FlagRegisterer for supported types. +#define INSTANTIATE_FLAG_REGISTERER(type) \ + template FlagRegisterer::FlagRegisterer(std::string name, \ + std::string help, \ + std::string file, \ + const type* default_value, \ + type* value) + +INSTANTIATE_FLAG_REGISTERER(bool); +INSTANTIATE_FLAG_REGISTERER(int32_t); +INSTANTIATE_FLAG_REGISTERER(uint32_t); +INSTANTIATE_FLAG_REGISTERER(int64_t); +INSTANTIATE_FLAG_REGISTERER(uint64_t); +INSTANTIATE_FLAG_REGISTERER(double); +INSTANTIATE_FLAG_REGISTERER(std::string); + +#undef INSTANTIATE_FLAG_REGISTERER + +std::string FlagType2String(FlagType type) { + switch (type) { + case FlagType::BOOL: + return "bool"; + case FlagType::INT32: + return "int32"; + case FlagType::UINT32: + return "uint32"; + case FlagType::INT64: + return "int64"; + case FlagType::UINT64: + return "uint64"; + case FlagType::DOUBLE: + return "double"; + case FlagType::STRING: + return "string"; + default: + return "undefined"; + } +} + +std::string Value2String(const void* value, FlagType type) { + switch (type) { + case FlagType::BOOL: { + const bool* val = static_cast(value); + return *val ? "true" : "false"; + } + case FlagType::INT32: { + const int32_t* val = static_cast(value); + return std::to_string(*val); + } + case FlagType::UINT32: { + const uint32_t* val = static_cast(value); + return std::to_string(*val); + } + case FlagType::INT64: { + const int64_t* val = static_cast(value); + return std::to_string(*val); + } + case FlagType::UINT64: { + const uint64_t* val = static_cast(value); + return std::to_string(*val); + } + case FlagType::DOUBLE: { + const double* val = static_cast(value); + return std::to_string(*val); + } + case FlagType::STRING: { + const std::string* val = static_cast(value); + return *val; + } + default: + LOG_FLAG_FATAL_ERROR("flag type is undefined."); + return ""; + } +} + +std::string Flag::Summary() const { + return "--" + name_ + ": " + FlagType2String(type_) + ", " + description_ + + " (default: " + Value2String(default_value_, type_) + ")"; +} + +void Flag::SetValueFromString(const std::string& value) { + try { + switch (type_) { + case FlagType::BOOL: { + bool* val = static_cast(value_); + if (value == "true" || value == "True" || value == "TRUE" || + value == "1") { + *val = true; + } else if (value == "false" || value == "False" || value == "FALSE" || + value == "0") { + *val = false; + } else { + throw std::invalid_argument( + ", please use [true, True, TRUE, 1] or [false, False, FALSE, " + "0]."); + } + break; + } + case FlagType::INT32: { + int32_t* val = static_cast(value_); + *val = std::stoi(value); + break; + } + case FlagType::UINT32: { + uint32_t* val = static_cast(value_); + *val = std::stoul(value); + break; + } + case FlagType::INT64: { + int64_t* val = static_cast(value_); + *val = std::stoll(value); + break; + } + case FlagType::UINT64: { + uint64_t* val = static_cast(value_); + *val = std::stoull(value); + break; + } + case FlagType::DOUBLE: { + double* val = static_cast(value_); + *val = std::stod(value); + break; + } + case FlagType::STRING: { + std::string* val = static_cast(value_); + *val = value; + break; + } + default: { + LOG_FLAG_FATAL_ERROR("flag type is undefined."); + } + } + } catch (const std::exception& e) { + std::string error_msg = "value: \"" + value + "\" is invalid for " + + FlagType2String(type_) + " flag \"" + name_ + "\""; + if (type_ == FlagType::BOOL) { + error_msg += e.what(); + } else { + error_msg += "."; + } + LOG_FLAG_ERROR(error_msg); + } +} + +void FlagRegistry::RegisterFlag(Flag* flag) { + auto iter = flags_.find(flag->name_); + if (iter != flags_.end()) { + LOG_FLAG_FATAL_ERROR("illegal RegisterFlag, flag \"" + flag->name_ + + "\" has been defined in " + iter->second->file_); + } else { + std::lock_guard lock(mutex_); + flags_[flag->name_] = flag; + flags_by_file_[flag->file_].insert(flag); + } +} + +bool FlagRegistry::SetFlagValue(const std::string& name, + const std::string& value) { + if (HasFlag(name)) { + std::lock_guard lock(mutex_); + flags_[name]->SetValueFromString(value); + return true; + } else { + LOG_FLAG_ERROR("illegal SetFlagValue, flag \"" + name + + "\" is not defined."); + return false; + } +} + +bool FlagRegistry::HasFlag(const std::string& name) const { + return flags_.find(name) != flags_.end(); +} + +void FlagRegistry::PrintAllFlagHelp(std::ostream& os) const { + for (const auto& iter : flags_by_file_) { + os << std::endl << "Flags defined in " << iter.first << ":" << std::endl; + for (const auto& flag : iter.second) { + os << " " << flag->Summary() << std::endl; + } + } + os << std::endl; +} + +void PrintAllFlagHelp(bool to_file, const std::string& file_path) { + if (to_file) { + std::ofstream fout(file_path); + FlagRegistry::Instance()->PrintAllFlagHelp(fout); + } else { + FlagRegistry::Instance()->PrintAllFlagHelp(std::cout); + } +} + +bool SetFlagValue(const std::string& name, const std::string& value) { + return FlagRegistry::Instance()->SetFlagValue(name, value); +} + +bool FindFlag(const std::string& name) { + return FlagRegistry::Instance()->HasFlag(name); +} + +bool GetValueFromEnv(const std::string& name, std::string* value) { + const char* env_var = std::getenv(name.c_str()); + if (env_var == nullptr) { + return false; + } + *value = std::string(env_var); + return true; +} + +void SetFlagsFromEnv(const std::vector& flags, bool error_fatal) { + bool success = true; + for (const std::string& flag_name : flags) { + std::string env_var_name = std::string("FLAGS_") + flag_name; + std::string env_var_value; + if (GetValueFromEnv(env_var_name, &env_var_value)) { + success = + FlagRegistry::Instance()->SetFlagValue(flag_name, env_var_value); + } else if (error_fatal) { + LOG_FLAG_ERROR("environment variable \"" + env_var_name + + "\" is not set."); + success = false; + } + } + if (error_fatal && !success) { + exit_with_errors(); + } +} + +static bool allow_undefined_flags = false; + +void AllowUndefinedFlags() { allow_undefined_flags = true; } + +void ParseCommandLineFlags(int* pargc, char*** pargv) { + assert(*pargc > 0); + size_t argv_num = *pargc - 1; + std::vector argvs(*pargv + 1, *pargv + *pargc); + + std::string arg_format_help = + "please follow the formats: \"--help(h)\", \"--name=value\"" + " or \"--name value\"."; + for (size_t i = 0; i < argv_num; i++) { + const std::string& argv = argvs[i]; + + if (argv.size() < 2 || argv[0] != '-') { + LOG_FLAG_FATAL_ERROR("invalid commandline argument: \"" + argv + "\", " + + arg_format_help); + } + + // parse arg name and value + size_t hyphen_num = argv[1] == '-' ? 2 : 1; + std::string name, value; + size_t split_pos = argv.find('='); + if (split_pos == std::string::npos) { + // the argv format is "--name" or "--name value" + name = argv.substr(hyphen_num); + if (name.empty()) { + LOG_FLAG_FATAL_ERROR("invalid commandline argument: \"" + argv + + "\", " + arg_format_help); + } + + // print help message + if (name == "help" || name == "h") { + FlagRegistry::Instance()->PrintAllFlagHelp(std::cout); + exit(1); + } + + // get the value from next argv. + if (++i == argv_num) { + LOG_FLAG_FATAL_ERROR("expected value of flag \"" + name + + "\" but found none."); + } else { + value = argvs[i]; + } + } else { + // the argv format is "--name=value" + if (split_pos == hyphen_num || split_pos == argv.size() - 1) { + LOG_FLAG_FATAL_ERROR("invalid commandline argument: \"" + argv + + "\", " + arg_format_help); + } + name = argv.substr(hyphen_num, split_pos - hyphen_num); + value = argv.substr(split_pos + 1); + } + + // special case for flag value enclosed in "" + if (value[0] == '"') { + value = value.substr(1); + if (value.back() == '"') { + value.pop_back(); + } else { + while (i < argv_num) { + value += " "; + value += argvs[++i]; + if (value.back() == '"') { + break; + } + } + if (value.back() == '"') { + value.pop_back(); + } else { + LOG_FLAG_FATAL_ERROR("unexperted end of flag \"" + name + + "\" value while looking for matching `\"'"); + } + } + } + + if (name == "fromenv" || name == "tryfromenv") { + // Value of --fromenv or --tryfromenv should be + // a comma separated list of env var names. + std::vector env_flag_names; + for (size_t start_pos = 0, end_pos = 0; + start_pos < value.size() && end_pos != std::string::npos; + start_pos = end_pos + 1) { + end_pos = value.find(',', start_pos); + env_flag_names.push_back(value.substr(start_pos, end_pos - start_pos)); + } + if (name == "fromenv") { + SetFlagsFromEnv(env_flag_names, true); + } else { + SetFlagsFromEnv(env_flag_names, false); + } + continue; + } + + FlagRegistry::Instance()->SetFlagValue(name, value); + } + if (!allow_undefined_flags && !ErrorStream().str().empty()) { + exit_with_errors(); + } +} + +} // namespace flags +} // namespace paddle diff --git a/paddle/utils/flags_native.h b/paddle/utils/flags_native.h new file mode 100644 index 0000000000000000000000000000000000000000..04814a4f679507167ddae196621d29d0118a34a5 --- /dev/null +++ b/paddle/utils/flags_native.h @@ -0,0 +1,131 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include + +// This is a simple commandline flags tool for paddle, which is inspired by +// gflags but only implements the following necessary features: +// 1. Define or declare a flag. +// 2. Parse commandline flags. +// 3. Other utility functions. + +namespace paddle { +namespace flags { +/** + * @brief Parse commandline flags. + * + * It recieves commandline arguments passed in argc and argv from main function, + * argv[0] is the program name, and argv[1:] are the commandline arguments + * which matching the format "--name=value" or "--name value". After parsing, + * the corresponding flag value will be reset. + */ +void ParseCommandLineFlags(int* argc, char*** argv); + +/** + * @brief Allow undefined flags in ParseCommandLineFlags() + */ +void AllowUndefinedFlags(); + +/** + * @brief Set flags from environment variables. + * + * It recieves a list of flags name, and will find the corresponding environment + * variables named "FLAGS_name", if found, it will set the environment variable + * values to the flags. If error_fatal is true, the program will exit when the + * environment variable is not set or the flag is not defined, that is the same + * effect as using commandline argument "--fromenv=var_name1,var_name2,...". + * Otherwise, the errors above will be ignored, that is the same effect as using + * commandline argument "--tryfromenv=var_name1,var_name2,...". + */ +void SetFlagsFromEnv(const std::vector& flags, bool error_fatal); + +/** + * @brief Set Single flag value, return true if success. + */ +bool SetFlagValue(const std::string& name, const std::string& value); + +/** + * @brief Find flag by name, return true if found. + */ +bool FindFlag(const std::string& name); + +/** + * @brief Print all registered flags' help message. If to_file is true, + * write help message to file. + */ +void PrintAllFlagHelp(bool to_file = false, + const std::string& file_name = "all_flags.txt"); +} // namespace flags +} // namespace paddle + +// ----------------------------DECLARE FLAGS---------------------------- +#define PD_DECLARE_VARIABLE(type, name) \ + namespace paddle { \ + namespace flags { \ + extern type FLAGS_##name; \ + } \ + } \ + using paddle::flags::FLAGS_##name + +#define PD_DECLARE_bool(name) PD_DECLARE_VARIABLE(bool, name) +#define PD_DECLARE_int32(name) PD_DECLARE_VARIABLE(int32_t, name) +#define PD_DECLARE_uint32(name) PD_DECLARE_VARIABLE(uint32_t, name) +#define PD_DECLARE_int64(name) PD_DECLARE_VARIABLE(int64_t, name) +#define PD_DECLARE_uint64(name) PD_DECLARE_VARIABLE(uint64_t, name) +#define PD_DECLARE_double(name) PD_DECLARE_VARIABLE(double, name) +#define PD_DECLARE_string(name) PD_DECLARE_VARIABLE(std::string, name) + +namespace paddle { +namespace flags { +class FlagRegisterer { + public: + template + FlagRegisterer(std::string name, + std::string description, + std::string file, + const T* default_value, + T* value); +}; +} // namespace flags +} // namespace paddle + +// ----------------------------DEFINE FLAGS---------------------------- +#define PD_DEFINE_VARIABLE(type, name, default_value, description) \ + namespace paddle { \ + namespace flags { \ + static const type FLAGS_##name##_default = default_value; \ + type FLAGS_##name = default_value; \ + /* Register FLAG */ \ + static ::paddle::flags::FlagRegisterer flag_##name##_registerer( \ + #name, description, __FILE__, &FLAGS_##name##_default, &FLAGS_##name); \ + } \ + } \ + using paddle::flags::FLAGS_##name + +#define PD_DEFINE_bool(name, val, txt) PD_DEFINE_VARIABLE(bool, name, val, txt) +#define PD_DEFINE_int32(name, val, txt) \ + PD_DEFINE_VARIABLE(int32_t, name, val, txt) +#define PD_DEFINE_uint32(name, val, txt) \ + PD_DEFINE_VARIABLE(uint32_t, name, val, txt) +#define PD_DEFINE_int64(name, val, txt) \ + PD_DEFINE_VARIABLE(int64_t, name, val, txt) +#define PD_DEFINE_uint64(name, val, txt) \ + PD_DEFINE_VARIABLE(uint64_t, name, val, txt) +#define PD_DEFINE_double(name, val, txt) \ + PD_DEFINE_VARIABLE(double, name, val, txt) +#define PD_DEFINE_string(name, val, txt) \ + PD_DEFINE_VARIABLE(std::string, name, val, txt) diff --git a/paddle/utils/flags_native_test.cc b/paddle/utils/flags_native_test.cc new file mode 100644 index 0000000000000000000000000000000000000000..2b26a8cd1e9dbb35c5fde7aca4646ca4098a89f1 --- /dev/null +++ b/paddle/utils/flags_native_test.cc @@ -0,0 +1,104 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/utils/flags_native.h" + +#include +#include "gtest/gtest.h" + +PD_DEFINE_int32(paddle_test_int32, 1, "test int32 flag"); +PD_DEFINE_uint32(paddle_test_uint32, 2, "test uint32 flag"); +PD_DEFINE_string(paddle_test_string, "raw", "test string flag"); + +using namespace paddle::flags; // NOLINT + +void SplitCommandlineArg(const std::string& commandline, + int* argc, + char*** argv) { + static std::vector args; + args.clear(); + for (size_t start_pos = 0, end_pos = 0; + start_pos < commandline.size() && end_pos != std::string::npos; + start_pos = end_pos + 1) { + end_pos = commandline.find(' ', start_pos); + args.push_back(commandline.substr(start_pos, end_pos - start_pos)); + } + *argc = args.size(); + *argv = new char*[*argc]; + for (size_t i = 0; i < args.size(); i++) { + (*argv)[i] = const_cast(args[i].c_str()); + } +} + +TEST(flags_native_test, ParseCommandLineFlags) { + uint32_t test_int32 = 2; + ASSERT_EQ(FLAGS_paddle_test_int32, 1); + ASSERT_EQ(FLAGS_paddle_test_uint32, test_int32); + ASSERT_EQ(FLAGS_paddle_test_string, "raw"); + + // Construct commandline arguments input + std::string commandline = + "test --paddle_test_int32=3 --paddle_test_uint32=\"4\" " + "--paddle_test_string \"modified string\""; + int argc; + char** argv; + SplitCommandlineArg(commandline, &argc, &argv); + + // Parse commandline flags and check + ParseCommandLineFlags(&argc, &argv); + delete argv; + + test_int32 = 4; + ASSERT_EQ(FLAGS_paddle_test_int32, 3); + ASSERT_EQ(FLAGS_paddle_test_uint32, test_int32); + ASSERT_EQ(FLAGS_paddle_test_string, "modified string"); + + // test FindFlag and SetFlagValue + ASSERT_TRUE(FindFlag("paddle_test_int32")); + + SetFlagValue("paddle_test_int32", "9"); + ASSERT_EQ(FLAGS_paddle_test_int32, 9); +} + +#if defined(_POSIX_C_SOURCE) && \ + _POSIX_C_SOURCE >= 200112L // environment for use setenv +bool SetEnvVar(const std::string& var_name, const std::string& var_value) { + int res = setenv(var_name.c_str(), var_value.c_str(), 1); + return res == 0; +} + +PD_DEFINE_bool(paddle_test_env_bool, false, "test env bool flag"); +PD_DEFINE_double(paddle_test_env_double, 3.14, "test env double flag"); + +TEST(flags_native_test, SetFlagsFromEnv) { + ASSERT_EQ(FLAGS_paddle_test_env_bool, false); + ASSERT_EQ(FLAGS_paddle_test_env_double, 3.14); + + ASSERT_TRUE(SetEnvVar("FLAGS_paddle_test_env_bool", "true")); + ASSERT_TRUE(SetEnvVar("FLAGS_paddle_test_env_double", "2.71")); + + std::string commandline = + "test --fromenv=paddle_test_env_bool,paddle_test_env_double"; + int argc; + char** argv; + SplitCommandlineArg(commandline, &argc, &argv); + ParseCommandLineFlags(&argc, &argv); + delete argv; + + ASSERT_EQ(FLAGS_paddle_test_env_bool, true); + ASSERT_EQ(FLAGS_paddle_test_env_double, 2.71); +} +#endif + +TEST(flags_native_test, PrintAllFlagHelp) { PrintAllFlagHelp(); } diff --git a/paddle/utils/string/pretty_log.cc b/paddle/utils/string/pretty_log.cc index 44bbbd0cc1bb62717da6a068454118fc4f9a58be..bb84ad201848ce7db810018804cab2a26c74d261 100644 --- a/paddle/utils/string/pretty_log.cc +++ b/paddle/utils/string/pretty_log.cc @@ -14,9 +14,9 @@ #include "paddle/utils/string/pretty_log.h" -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" -DEFINE_bool(color, true, "Whether to turn on pretty log"); // NOLINT +PD_DEFINE_bool(color, true, "Whether to turn on pretty log"); // NOLINT namespace paddle { namespace string {} // namespace string diff --git a/paddle/utils/string/pretty_log.h b/paddle/utils/string/pretty_log.h index 9de7ce24abd72d96708d970afe1ee41d476e8c27..546bf1eec7d1da1f5308844f545f6d903a970e39 100644 --- a/paddle/utils/string/pretty_log.h +++ b/paddle/utils/string/pretty_log.h @@ -18,10 +18,10 @@ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/utils/string/printf.h" -DECLARE_bool(color); +PD_DECLARE_bool(color); namespace paddle { diff --git a/test/cpp/fluid/benchmark/op_tester.cc b/test/cpp/fluid/benchmark/op_tester.cc index 0ab90f03999da698d5950bd154637fcef22e9869..6f68ab23a45669f8d9f7a6f0cab1cae7046a2e53 100644 --- a/test/cpp/fluid/benchmark/op_tester.cc +++ b/test/cpp/fluid/benchmark/op_tester.cc @@ -16,7 +16,6 @@ limitations under the License. */ #include -#include "gflags/gflags.h" #include "gtest/gtest.h" #include "paddle/fluid/framework/op_info.h" #include "paddle/fluid/framework/op_registry.h" @@ -25,6 +24,7 @@ limitations under the License. */ #include "paddle/fluid/platform/profiler.h" #include "paddle/fluid/platform/timer.h" #include "paddle/fluid/pybind/pybind.h" +#include "paddle/utils/flags.h" // phi #include "paddle/phi/kernels/declarations.h" @@ -33,8 +33,8 @@ namespace paddle { namespace operators { namespace benchmark { -DEFINE_string(op_config_list, "", "Path of op config file."); // NOLINT -DEFINE_int32(specified_config_id, -1, "Test the specified op config."); +PD_DEFINE_string(op_config_list, "", "Path of op config file."); // NOLINT +PD_DEFINE_int32(specified_config_id, -1, "Test the specified op config."); void OpTester::Init(const std::string &filename) { Init(OpTesterConfig(filename)); @@ -57,13 +57,13 @@ void OpTester::Init(const OpTesterConfig &config) { } if (config_.device_id >= 0) { - place_ = paddle::platform::CUDAPlace(config_.device_id); + place_ = ::paddle::platform::CUDAPlace(config_.device_id); } else { - place_ = paddle::platform::CPUPlace(); + place_ = ::paddle::platform::CPUPlace(); } framework::InitDevices(); - scope_ = std::make_unique(); + scope_ = std::make_unique<::paddle::framework::Scope>(); op_ = framework::OpRegistry::CreateOp(op_desc_); CreateVariables(scope_.get()); @@ -318,7 +318,7 @@ void OpTester::SetupTensor(phi::DenseTensor *tensor, } if (!platform::is_cpu_place(place_)) { - paddle::framework::TensorCopySync(cpu_tensor, place_, tensor); + ::paddle::framework::TensorCopySync(cpu_tensor, place_, tensor); } } diff --git a/test/cpp/fluid/mkldnn/test_mkldnn_cpu_quantize_pass.cc b/test/cpp/fluid/mkldnn/test_mkldnn_cpu_quantize_pass.cc index 6f1ac7e56f3045067a939baa9ed037a077aaa3d0..d6ba2f7a0c23220fcc639890b4a5deb89579227a 100644 --- a/test/cpp/fluid/mkldnn/test_mkldnn_cpu_quantize_pass.cc +++ b/test/cpp/fluid/mkldnn/test_mkldnn_cpu_quantize_pass.cc @@ -29,7 +29,7 @@ using std::pair; using std::string; using std::unordered_map; -DEFINE_bool(enable_mkldnn, true, "Enable MKLDNN"); +PD_DEFINE_bool(enable_mkldnn, true, "Enable MKLDNN"); namespace paddle { namespace pass { diff --git a/test/cpp/fluid/pscore/CMakeLists.txt b/test/cpp/fluid/pscore/CMakeLists.txt index c19df6b46960562bc60d7f1f99e20d33008e7db2..07d3efaa31110275c75fd9953becc7a2434b8083 100644 --- a/test/cpp/fluid/pscore/CMakeLists.txt +++ b/test/cpp/fluid/pscore/CMakeLists.txt @@ -18,7 +18,7 @@ if(WITH_ARM_BRPC) framework_proto sendrecv_rpc arm_brpc - gflags + ${flags_dep} glog snappy device_context) @@ -35,15 +35,9 @@ else() ps_framework_proto framework_proto sendrecv_rpc - brpc - leveldb - ssl - crypto - protobuf - gflags - glog + ${EXTERNAL_BRPC_DEPS} + ${flags_dep} zlib - snappy device_context) endif() diff --git a/test/cpp/fluid/pscore/switch_server_test.cc b/test/cpp/fluid/pscore/switch_server_test.cc index a5e6fff4804af491f357156e218f107ebc1d41e1..5ea2e28d4543b8ec51a4c0b8dd0c3008749cef12 100644 --- a/test/cpp/fluid/pscore/switch_server_test.cc +++ b/test/cpp/fluid/pscore/switch_server_test.cc @@ -21,18 +21,18 @@ limitations under the License. */ #include #include // NOLINT -#include "gflags/gflags.h" #include "gtest/gtest.h" #include "paddle/fluid/distributed/ps/service/heter_client.h" #include "paddle/fluid/distributed/ps/service/heter_server.h" +#include "paddle/utils/flags.h" namespace framework = paddle::framework; namespace platform = paddle::platform; namespace distributed = paddle::distributed; -DEFINE_string(switch_addr_inner, "127.0.0.1:6000", "addr of inner cluster"); -DEFINE_string(switch_addr_heter, "127.0.0.1:6100", "add of inter cluster"); -DEFINE_string(peer_switch_addr, "127.0.0.1:7100", "add of inter cluster"); +PD_DEFINE_string(switch_addr_inner, "127.0.0.1:6000", "addr of inner cluster"); +PD_DEFINE_string(switch_addr_heter, "127.0.0.1:6100", "add of inter cluster"); +PD_DEFINE_string(peer_switch_addr, "127.0.0.1:7100", "add of inter cluster"); void StartSwitchServer( std::shared_ptr& switch_server_ptr, // NOLINT @@ -61,7 +61,7 @@ int main(int argc, char* argv[]) { framework::ProgramDesc program; exe.Prepare(program, 0); // solve undefined symbol: tensor_table.cc - google::ParseCommandLineFlags(&argc, &argv, true); + paddle::flags::ParseCommandLineFlags(&argc, &argv); std::string switch_a_endpoint(FLAGS_switch_addr_inner); std::string switch_a_endpoint_inter(FLAGS_switch_addr_heter); diff --git a/test/cpp/inference/api/analysis_predictor_tester.cc b/test/cpp/inference/api/analysis_predictor_tester.cc index 2f794a4c784fa4288725cbaf2390b1a6dd30922c..35c07c3a83790ca1939ba8c65fc41b8ae168b12e 100644 --- a/test/cpp/inference/api/analysis_predictor_tester.cc +++ b/test/cpp/inference/api/analysis_predictor_tester.cc @@ -31,7 +31,7 @@ #include "paddle/phi/backends/cpu/cpu_info.h" #include "test/cpp/inference/api/tester_helper.h" -DEFINE_string(dirname, "", "dirname to tests."); +PD_DEFINE_string(dirname, "", "dirname to tests."); namespace paddle { diff --git a/test/cpp/inference/api/analyzer_bfloat16_image_classification_tester.cc b/test/cpp/inference/api/analyzer_bfloat16_image_classification_tester.cc index f6d4d8e8f769fb1f9953216089e4c8ab162087d2..fe7d2a3a6f6cf90c8e5db073bae201a7980baae1 100644 --- a/test/cpp/inference/api/analyzer_bfloat16_image_classification_tester.cc +++ b/test/cpp/inference/api/analyzer_bfloat16_image_classification_tester.cc @@ -16,7 +16,7 @@ limitations under the License. */ #include "paddle/phi/backends/cpu/cpu_info.h" #include "test/cpp/inference/api/tester_helper.h" -DEFINE_bool(enable_mkldnn, true, "Enable MKLDNN"); +PD_DEFINE_bool(enable_mkldnn, true, "Enable MKLDNN"); namespace paddle { namespace inference { diff --git a/test/cpp/inference/api/analyzer_dam_tester.cc b/test/cpp/inference/api/analyzer_dam_tester.cc index 21276c71f63ec3829dd036869ba160bb69ab8b61..d17f8670adcf434dd83c0be52ba3d26acc20b75c 100644 --- a/test/cpp/inference/api/analyzer_dam_tester.cc +++ b/test/cpp/inference/api/analyzer_dam_tester.cc @@ -130,7 +130,7 @@ void PrepareInputs(std::vector *input_slots, auto one_batch = data->NextBatch(); PADDLE_ENFORCE( !one_batch.response.empty(), - paddle::platform::errors::Fatal("The response of one batch is empty.")); + ::paddle::platform::errors::Fatal("The response of one batch is empty.")); int size = one_batch.response[0].size(); CHECK_EQ(size, kMaxTurnLen); // turn tensor assignment @@ -228,17 +228,17 @@ void profile(bool use_mkldnn = false) { if (FLAGS_num_threads == 1 && !FLAGS_test_all_data) { PADDLE_ENFORCE_GT(outputs.size(), 0, - paddle::platform::errors::Fatal( + ::paddle::platform::errors::Fatal( "The size of outputs should be greater than 0.")); auto output = outputs.back(); PADDLE_ENFORCE_GT(output.size(), 0, - paddle::platform::errors::Fatal( + ::paddle::platform::errors::Fatal( "The size of output should be greater than 0.")); size_t size = GetSize(output[0]); PADDLE_ENFORCE_GT(size, 0, - paddle::platform::errors::Fatal( + ::paddle::platform::errors::Fatal( "The size of output should be greater than 0.")); float *result = static_cast(output[0].data.data()); for (size_t i = 0; i < size; i++) { diff --git a/test/cpp/inference/api/analyzer_detect_functional_mkldnn_tester.cc b/test/cpp/inference/api/analyzer_detect_functional_mkldnn_tester.cc index 389e0c9648d29189af6ae2531c30929b0ecace89..8d5627fce3cd8bbb2d70687427afb3cc7b6c5d22 100644 --- a/test/cpp/inference/api/analyzer_detect_functional_mkldnn_tester.cc +++ b/test/cpp/inference/api/analyzer_detect_functional_mkldnn_tester.cc @@ -20,8 +20,8 @@ limitations under the License. */ #include "paddle/phi/common/place.h" #include "test/cpp/inference/api/tester_helper.h" -DEFINE_string(infer_shape, "", "data shape file"); -DEFINE_int32(sample, 20, "number of sample"); +PD_DEFINE_string(infer_shape, "", "data shape file"); +PD_DEFINE_int32(sample, 20, "number of sample"); namespace paddle { namespace inference { diff --git a/test/cpp/inference/api/analyzer_detect_tester.cc b/test/cpp/inference/api/analyzer_detect_tester.cc index 62b97d635b5e23effb1f01ac7b2c96c7d70b4f30..72e498521cd6a02929d85d9488b4be3e67dfc692 100644 --- a/test/cpp/inference/api/analyzer_detect_tester.cc +++ b/test/cpp/inference/api/analyzer_detect_tester.cc @@ -19,8 +19,8 @@ limitations under the License. */ #include "test/cpp/inference/api/tester_helper.h" -DEFINE_string(infer_shape, "", "data shape file"); -DEFINE_int32(sample, 20, "number of sample"); +PD_DEFINE_string(infer_shape, "", "data shape file"); +PD_DEFINE_int32(sample, 20, "number of sample"); namespace paddle { namespace inference { diff --git a/test/cpp/inference/api/analyzer_image_classification_tester.cc b/test/cpp/inference/api/analyzer_image_classification_tester.cc index 93b8f92c797d9981f51359707a42765641b3533b..1df6d4488614d46f2238de5d91149b6b1e1f8f65 100644 --- a/test/cpp/inference/api/analyzer_image_classification_tester.cc +++ b/test/cpp/inference/api/analyzer_image_classification_tester.cc @@ -17,7 +17,7 @@ limitations under the License. */ #include "test/cpp/inference/api/tester_helper.h" -DEFINE_bool(disable_mkldnn_fc, false, "Disable usage of MKL-DNN's FC op"); +PD_DEFINE_bool(disable_mkldnn_fc, false, "Disable usage of MKL-DNN's FC op"); namespace paddle { namespace inference { diff --git a/test/cpp/inference/api/analyzer_int8_image_classification_tester.cc b/test/cpp/inference/api/analyzer_int8_image_classification_tester.cc index 57552ccb82e60226b441a2e9d8a33ef58cba99d1..77c12dcfe0f524fb051715184a9d05f55a864f54 100644 --- a/test/cpp/inference/api/analyzer_int8_image_classification_tester.cc +++ b/test/cpp/inference/api/analyzer_int8_image_classification_tester.cc @@ -18,7 +18,7 @@ limitations under the License. */ #include "paddle/fluid/inference/api/paddle_analysis_config.h" #include "test/cpp/inference/api/tester_helper.h" -DEFINE_bool(enable_mkldnn, true, "Enable MKLDNN"); +PD_DEFINE_bool(enable_mkldnn, true, "Enable MKLDNN"); namespace paddle { namespace inference { @@ -53,7 +53,7 @@ TEST(Analyzer_int8_image_classification, quantization) { // prepare warmup batch from input data read earlier // warmup batch size can be different than batch size std::shared_ptr> warmup_data = - paddle::inference::GetWarmupData(input_slots_all); + ::paddle::inference::GetWarmupData(input_slots_all); // INT8 implies FC oneDNN passes to be used q_cfg.pass_builder()->AppendPass("fc_mkldnn_pass"); diff --git a/test/cpp/inference/api/analyzer_int8_object_detection_tester.cc b/test/cpp/inference/api/analyzer_int8_object_detection_tester.cc index a3b13dd691cd7b3d46e259fca7f33efd7ead25c0..311fb0946ca0068b3bcc8b2932e79bdfa2d65b80 100644 --- a/test/cpp/inference/api/analyzer_int8_object_detection_tester.cc +++ b/test/cpp/inference/api/analyzer_int8_object_detection_tester.cc @@ -18,7 +18,7 @@ limitations under the License. */ #include "paddle/fluid/inference/api/paddle_analysis_config.h" #include "test/cpp/inference/api/tester_helper.h" -DEFINE_bool(enable_mkldnn, true, "Enable MKLDNN"); +PD_DEFINE_bool(enable_mkldnn, true, "Enable MKLDNN"); // setting iterations to 0 means processing the whole dataset namespace paddle { @@ -153,7 +153,7 @@ std::shared_ptr> GetWarmupData( PADDLE_ENFORCE_LE( static_cast(num_images), iterations * test_data_batch_size, - paddle::platform::errors::Fatal( + ::paddle::platform::errors::Fatal( "The requested quantization warmup data size " + std::to_string(num_images) + " is bigger than all test data size.")); @@ -247,9 +247,9 @@ std::shared_ptr> GetWarmupData( PADDLE_ENFORCE_EQ( static_cast(num_objects), static_cast(objects_accum), - paddle::platform::errors::Fatal("The requested num of objects " + - std::to_string(num_objects) + - " is the same as objects_accum.")); + ::paddle::platform::errors::Fatal("The requested num of objects " + + std::to_string(num_objects) + + " is the same as objects_accum.")); auto warmup_data = std::make_shared>(4); (*warmup_data)[0] = std::move(images); diff --git a/test/cpp/inference/api/analyzer_lac_tester.cc b/test/cpp/inference/api/analyzer_lac_tester.cc index fb82bbc3b2d947e397d408042528d82387a3303f..9bdb819e5fbd684b69f829c3f446686714d72e3e 100644 --- a/test/cpp/inference/api/analyzer_lac_tester.cc +++ b/test/cpp/inference/api/analyzer_lac_tester.cc @@ -99,10 +99,10 @@ void GetOneBatch(std::vector *input_slots, input_tensor.name = "word"; input_tensor.dtype = PaddleDType::INT64; TensorAssignData(&input_tensor, {one_batch.data}, one_batch.lod); - PADDLE_ENFORCE_EQ( - batch_size, - static_cast(one_batch.lod.size() - 1), - paddle::platform::errors::Fatal("The lod size of one batch is invaild.")); + PADDLE_ENFORCE_EQ(batch_size, + static_cast(one_batch.lod.size() - 1), + ::paddle::platform::errors::Fatal( + "The lod size of one batch is invaild.")); input_slots->assign({input_tensor}); } @@ -145,19 +145,19 @@ TEST(Analyzer_LAC, profile) { 15, 44, 38, 39, 14, 15, 44, 22, 23, 23, 23, 23, 23, 23, 23}; PADDLE_ENFORCE_GT(outputs.size(), 0, - paddle::platform::errors::Fatal( + ::paddle::platform::errors::Fatal( "The size of output should be greater than 0.")); auto output = outputs.back(); PADDLE_ENFORCE_EQ(output.size(), 1UL, - paddle::platform::errors::Fatal( + ::paddle::platform::errors::Fatal( "The size of output should be equal to 1.")); size_t size = GetSize(output[0]); size_t batch1_size = sizeof(lac_ref_data) / sizeof(int64_t); PADDLE_ENFORCE_GE( size, batch1_size, - paddle::platform::errors::Fatal("The size of batch is invaild.")); + ::paddle::platform::errors::Fatal("The size of batch is invaild.")); int64_t *pdata = static_cast(output[0].data.data()); for (size_t i = 0; i < batch1_size; ++i) { EXPECT_EQ(pdata[i], lac_ref_data[i]); diff --git a/test/cpp/inference/api/analyzer_mmp_tester.cc b/test/cpp/inference/api/analyzer_mmp_tester.cc index a432d5c10b2295e3b07c65429690463b422cbc5b..92345fc8950a5362e819fe0ce81d5c30aa118f4b 100644 --- a/test/cpp/inference/api/analyzer_mmp_tester.cc +++ b/test/cpp/inference/api/analyzer_mmp_tester.cc @@ -18,8 +18,8 @@ #include "test/cpp/inference/api/tester_helper.h" // Here add missing commands -DEFINE_string(infer_model2, "", "model path"); -DEFINE_string(infer_model3, "", "model path"); +PD_DEFINE_string(infer_model2, "", "model path"); +PD_DEFINE_string(infer_model3, "", "model path"); namespace paddle { namespace inference { @@ -100,12 +100,12 @@ void compare(bool use_mkldnn = false) { xx2_output.begin(), [](const float& l, const float& r) { return fabs(l - r) < 1e-4; }); - PADDLE_ENFORCE_EQ( - result, - true, - paddle::platform::errors::Fatal("Results of model run independently " - "differs from results of the same model " - "run as a sequence of models")); + PADDLE_ENFORCE_EQ(result, + true, + ::paddle::platform::errors::Fatal( + "Results of model run independently " + "differs from results of the same model " + "run as a sequence of models")); } TEST(Analyzer_mmp, compare) { compare(); } diff --git a/test/cpp/inference/api/analyzer_quant_image_classification_tester.cc b/test/cpp/inference/api/analyzer_quant_image_classification_tester.cc index 69b627275cd056b17a91cde1b10386bc5e8fbe6b..e9b841ec772abc18d6a295515616e936e26b83e2 100644 --- a/test/cpp/inference/api/analyzer_quant_image_classification_tester.cc +++ b/test/cpp/inference/api/analyzer_quant_image_classification_tester.cc @@ -18,7 +18,7 @@ limitations under the License. */ #include "paddle/fluid/inference/api/paddle_analysis_config.h" #include "test/cpp/inference/api/tester_helper.h" -DEFINE_bool(enable_mkldnn, true, "Enable MKLDNN"); +PD_DEFINE_bool(enable_mkldnn, true, "Enable MKLDNN"); namespace paddle { namespace inference { diff --git a/test/cpp/inference/api/analyzer_rnn1_tester.cc b/test/cpp/inference/api/analyzer_rnn1_tester.cc index 7f0f11f5515bddac1c8c8769bd7c4b1335cc751d..c5c7df887dd5ed2316e9992a63e58c7020c2ae01 100644 --- a/test/cpp/inference/api/analyzer_rnn1_tester.cc +++ b/test/cpp/inference/api/analyzer_rnn1_tester.cc @@ -14,7 +14,7 @@ #include "test/cpp/inference/api/tester_helper.h" -DEFINE_bool(with_precision_check, true, "turn on test"); +PD_DEFINE_bool(with_precision_check, true, "turn on test"); namespace paddle { namespace inference { diff --git a/test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h b/test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h index 3c0ad4b5f823d80a54e0fd6a93739fcd60fa8835..0d75eacbbdf4e66e967f026dc8eb9c3ca5315e01 100644 --- a/test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h +++ b/test/cpp/inference/api/analyzer_seq_pool1_tester_helper.h @@ -65,7 +65,7 @@ struct DataRecord { PADDLE_ENFORCE_EQ( slot_data.size() % 11, 0UL, - paddle::platform::errors::Fatal( + ::paddle::platform::errors::Fatal( "line %d, %s should be divisible", num_lines, name)); datasets[name].emplace_back(std::move(slot_data)); } @@ -73,19 +73,19 @@ struct DataRecord { PADDLE_ENFORCE_EQ( num_samples * num_slots, static_cast(num_lines), - paddle::platform::errors::Fatal("num samples should be divisible")); + ::paddle::platform::errors::Fatal("num samples should be divisible")); PADDLE_ENFORCE_GT(num_samples, 0UL, - paddle::platform::errors::Fatal( + ::paddle::platform::errors::Fatal( "The num of samples should be greater than 0.")); } void Prepare(int bs) { for (auto it = datasets.begin(); it != datasets.end(); ++it) { - PADDLE_ENFORCE_EQ( - it->second.size(), - num_samples, - paddle::platform::errors::Fatal("size of each slot should be equal")); + PADDLE_ENFORCE_EQ(it->second.size(), + num_samples, + ::paddle::platform::errors::Fatal( + "size of each slot should be equal")); } size_t num_batches = num_samples / bs; EXPECT_GT(num_batches, 0UL); @@ -110,7 +110,7 @@ struct DataRecord { PADDLE_ENFORCE_EQ( len * 11, datas[id].size(), - paddle::platform::errors::Fatal( + ::paddle::platform::errors::Fatal( "%s %d size should be divisible", slot.name, id)); lod[k + 1] = lod[k] + len; } diff --git a/test/cpp/inference/api/analyzer_vis_tester.cc b/test/cpp/inference/api/analyzer_vis_tester.cc index cf79d26847f452640f77e0ff8454e6bc243c11d4..d8b15393ad60137e25bf52a3cccf22a8651f6a3b 100644 --- a/test/cpp/inference/api/analyzer_vis_tester.cc +++ b/test/cpp/inference/api/analyzer_vis_tester.cc @@ -64,7 +64,7 @@ void SetInput(std::vector> *inputs) { PADDLE_ENFORCE_EQ( FLAGS_test_all_data, 0, - paddle::platform::errors::Fatal("Only have single batch of data.")); + ::paddle::platform::errors::Fatal("Only have single batch of data.")); std::string line; std::ifstream file(FLAGS_infer_data); std::getline(file, line); @@ -107,7 +107,7 @@ void profile(bool use_mkldnn = false) { PADDLE_ENFORCE_GT(outputs.size(), 0, - paddle::platform::errors::Fatal( + ::paddle::platform::errors::Fatal( "The size of output should be greater than 0.")); auto &output = outputs.back().front(); size_t numel = output.data.length() / PaddleDtypeSize(output.dtype); diff --git a/test/cpp/inference/api/api_impl_tester.cc b/test/cpp/inference/api/api_impl_tester.cc index 934ff06535054d3089cddec6ed57aab579f87913..78e908189cc1d4028b37368ffe35451f9cd1203a 100644 --- a/test/cpp/inference/api/api_impl_tester.cc +++ b/test/cpp/inference/api/api_impl_tester.cc @@ -17,9 +17,9 @@ limitations under the License. */ #include // NOLINT -#include "gflags/gflags.h" #include "paddle/fluid/framework/convert_utils.h" #include "paddle/fluid/inference/api/api_impl.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/test_helper.h" #ifdef __clang__ @@ -28,10 +28,10 @@ limitations under the License. */ #define ACC_DIFF 1e-3 #endif -DEFINE_string(word2vec_dirname, - "", - "Directory of the word2vec inference model."); -DEFINE_string(book_dirname, "", "Directory of the book inference model."); +PD_DEFINE_string(word2vec_dirname, + "", + "Directory of the word2vec inference model."); +PD_DEFINE_string(book_dirname, "", "Directory of the book inference model."); namespace paddle { @@ -67,11 +67,11 @@ NativeConfig GetConfig() { return config; } -void MainWord2Vec(const paddle::PaddlePlace& place) { +void MainWord2Vec(const ::paddle::PaddlePlace& place) { NativeConfig config = GetConfig(); auto predictor = CreatePaddlePredictor(config); - config.use_gpu = paddle::gpu_place_used(place); - config.use_xpu = paddle::xpu_place_used(place); + config.use_gpu = ::paddle::gpu_place_used(place); + config.use_xpu = ::paddle::xpu_place_used(place); phi::DenseTensor first_word, second_word, third_word, fourth_word; framework::LoD lod{{0, 1}}; @@ -105,7 +105,7 @@ void MainWord2Vec(const paddle::PaddlePlace& place) { cpu_feeds.push_back(&fourth_word); framework::FetchType output1; - std::vector cpu_fetchs1; + std::vector<::paddle::framework::FetchType*> cpu_fetchs1; cpu_fetchs1.push_back(&output1); TestInference(config.model_dir, cpu_feeds, cpu_fetchs1); @@ -118,12 +118,12 @@ void MainWord2Vec(const paddle::PaddlePlace& place) { } } -void MainImageClassification(const paddle::PaddlePlace& place) { +void MainImageClassification(const ::paddle::PaddlePlace& place) { int batch_size = 2; bool repeat = false; NativeConfig config = GetConfig(); - config.use_gpu = paddle::gpu_place_used(place); - config.use_xpu = paddle::xpu_place_used(place); + config.use_gpu = ::paddle::gpu_place_used(place); + config.use_xpu = ::paddle::xpu_place_used(place); config.model_dir = FLAGS_book_dirname + "/image_classification_resnet.inference.model"; @@ -163,10 +163,10 @@ void MainImageClassification(const paddle::PaddlePlace& place) { } } -void MainThreadsWord2Vec(const paddle::PaddlePlace& place) { +void MainThreadsWord2Vec(const ::paddle::PaddlePlace& place) { NativeConfig config = GetConfig(); - config.use_gpu = paddle::gpu_place_used(place); - config.use_xpu = paddle::xpu_place_used(place); + config.use_gpu = ::paddle::gpu_place_used(place); + config.use_xpu = ::paddle::xpu_place_used(place); auto main_predictor = CreatePaddlePredictor(config); // prepare inputs data and reference results @@ -186,7 +186,7 @@ void MainThreadsWord2Vec(const paddle::PaddlePlace& place) { // get reference result of each job std::vector ref_feeds; - std::vector ref_fetches(1, &refs[i]); + std::vector<::paddle::framework::FetchType*> ref_fetches(1, &refs[i]); for (auto& word : jobs[i]) { ref_feeds.push_back(&word); } @@ -225,12 +225,12 @@ void MainThreadsWord2Vec(const paddle::PaddlePlace& place) { } } -void MainThreadsImageClassification(const paddle::PaddlePlace& place) { +void MainThreadsImageClassification(const ::paddle::PaddlePlace& place) { constexpr int num_jobs = 4; // each job run 1 batch constexpr int batch_size = 1; NativeConfig config = GetConfig(); - config.use_gpu = paddle::gpu_place_used(place); - config.use_xpu = paddle::xpu_place_used(place); + config.use_gpu = ::paddle::gpu_place_used(place); + config.use_xpu = ::paddle::xpu_place_used(place); config.model_dir = FLAGS_book_dirname + "/image_classification_resnet.inference.model"; @@ -280,53 +280,53 @@ void MainThreadsImageClassification(const paddle::PaddlePlace& place) { } TEST(inference_api_native, word2vec_cpu) { - MainWord2Vec(paddle::PaddlePlace::kCPU); + MainWord2Vec(::paddle::PaddlePlace::kCPU); } TEST(inference_api_native, word2vec_cpu_threads) { - MainThreadsWord2Vec(paddle::PaddlePlace::kCPU); + MainThreadsWord2Vec(::paddle::PaddlePlace::kCPU); } TEST(inference_api_native, image_classification_cpu) { - MainImageClassification(paddle::PaddlePlace::kCPU); + MainImageClassification(::paddle::PaddlePlace::kCPU); } TEST(inference_api_native, image_classification_cpu_threads) { - MainThreadsImageClassification(paddle::PaddlePlace::kCPU); + MainThreadsImageClassification(::paddle::PaddlePlace::kCPU); } #ifdef PADDLE_WITH_XPU TEST(inference_api_native, word2vec_xpu) { - MainWord2Vec(paddle::PaddlePlace::kXPU); + MainWord2Vec(::paddle::PaddlePlace::kXPU); } TEST(inference_api_native, image_classification_xpu) { - MainImageClassification(paddle::PaddlePlace::kXPU); + MainImageClassification(::paddle::PaddlePlace::kXPU); } #endif #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) TEST(inference_api_native, word2vec_gpu) { - MainWord2Vec(paddle::PaddlePlace::kGPU); + MainWord2Vec(::paddle::PaddlePlace::kGPU); } // Turn off temporarily for the unstable result. // TEST(inference_api_native, word2vec_gpu_threads) { -// MainThreadsWord2Vec(paddle::PaddlePlace::kGPU); +// MainThreadsWord2Vec(::paddle::PaddlePlace::kGPU); // } TEST(inference_api_native, image_classification_gpu) { - MainImageClassification(paddle::PaddlePlace::kGPU); + MainImageClassification(::paddle::PaddlePlace::kGPU); } // Turn off temporarily for the unstable result. // TEST(inference_api_native, image_classification_gpu_threads) { -// MainThreadsImageClassification(paddle::PaddlePlace::kGPU); +// MainThreadsImageClassification(::paddle::PaddlePlace::kGPU); // } #endif #ifdef PADDLE_WITH_DNNL TEST(inference_api_native, image_classification_cpu_onednn) { FLAGS_use_mkldnn = true; - MainImageClassification(paddle::PaddlePlace::kCPU); + MainImageClassification(::paddle::PaddlePlace::kCPU); } TEST(inference_api_native, word2vec_cpu_onednn) { FLAGS_use_mkldnn = true; - MainWord2Vec(paddle::PaddlePlace::kCPU); + MainWord2Vec(::paddle::PaddlePlace::kCPU); } #endif diff --git a/test/cpp/inference/api/ipu_multi_model_profile.cc b/test/cpp/inference/api/ipu_multi_model_profile.cc index 3c5b1af1594e68e8abfe861c2d42ecbbf35fad50..d5b8ce2532244b7aa5398da570e2e100db63f06f 100644 --- a/test/cpp/inference/api/ipu_multi_model_profile.cc +++ b/test/cpp/inference/api/ipu_multi_model_profile.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/tester_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/ipu_resnet50_fp16_test.cc b/test/cpp/inference/api/ipu_resnet50_fp16_test.cc index 99f0d58926dbdc3f6ecf66ddbf34374ce0158877..1e3ddb51cb8235a00d7385538208e34aec94a5d6 100644 --- a/test/cpp/inference/api/ipu_resnet50_fp16_test.cc +++ b/test/cpp/inference/api/ipu_resnet50_fp16_test.cc @@ -17,7 +17,7 @@ limitations under the License. */ #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/tester_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/ipu_resnet50_test.cc b/test/cpp/inference/api/ipu_resnet50_test.cc index 5a414bf9415bc741417d2b808b6f8ca08e03572f..d45cd9bf49ae25acb511ab48d1100ea884913d94 100644 --- a/test/cpp/inference/api/ipu_resnet50_test.cc +++ b/test/cpp/inference/api/ipu_resnet50_test.cc @@ -14,7 +14,7 @@ limitations under the License. */ #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/tester_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/ipu_word2vec_sample.cc b/test/cpp/inference/api/ipu_word2vec_sample.cc index ba8f28ee5e19fa508d38dd5f818e3083420740a1..e43d03c5108599e3cde131c334c9cfc336895627 100644 --- a/test/cpp/inference/api/ipu_word2vec_sample.cc +++ b/test/cpp/inference/api/ipu_word2vec_sample.cc @@ -24,11 +24,11 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/fluid/inference/api/paddle_inference_api.h" +#include "paddle/utils/flags.h" -DEFINE_string(infer_model, "", "Directory of the inference model."); +PD_DEFINE_string(infer_model, "", "Directory of the inference model."); using paddle_infer::Config; using paddle_infer::CreatePredictor; @@ -70,7 +70,7 @@ void inference(std::string model_path, } int main(int argc, char *argv[]) { - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + ::paddle::flags::ParseCommandLineFlags(&argc, &argv); std::vector ipu_result; std::vector cpu_result; inference(FLAGS_infer_model, true, &ipu_result); diff --git a/test/cpp/inference/api/lite_mul_model_test.cc b/test/cpp/inference/api/lite_mul_model_test.cc index e600a3bab916c952581f09661ee485aaf2a390bd..3fa8e545a57da48c6973d4e92a1ed34cb8649125 100644 --- a/test/cpp/inference/api/lite_mul_model_test.cc +++ b/test/cpp/inference/api/lite_mul_model_test.cc @@ -19,7 +19,7 @@ limitations under the License. */ #include // NOLINT #include // NOLINT -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/tester_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/lite_resnet50_test.cc b/test/cpp/inference/api/lite_resnet50_test.cc index e35e28388234d437c4c797e7deb709f671775ef4..dce9a8932fe32e4563062d8c5ed71ad227a74861 100644 --- a/test/cpp/inference/api/lite_resnet50_test.cc +++ b/test/cpp/inference/api/lite_resnet50_test.cc @@ -17,7 +17,7 @@ limitations under the License. */ #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/tester_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/mkldnn_quantizer_tester.cc b/test/cpp/inference/api/mkldnn_quantizer_tester.cc index 5e699a8b4c6a58f525ea16e9e4b2917ef2faf682..8edad9fe27127aea0c0a304150a4d4e5c25df196 100644 --- a/test/cpp/inference/api/mkldnn_quantizer_tester.cc +++ b/test/cpp/inference/api/mkldnn_quantizer_tester.cc @@ -19,7 +19,7 @@ #include "paddle/fluid/inference/api/mkldnn_quantizer.h" #include "paddle/fluid/inference/api/paddle_inference_api.h" -DEFINE_string(dirname, "", "dirname to tests."); +PD_DEFINE_string(dirname, "", "dirname to tests."); namespace paddle { diff --git a/test/cpp/inference/api/paddle_infer_api_copy_tensor_tester.cc b/test/cpp/inference/api/paddle_infer_api_copy_tensor_tester.cc index 4674b77091a513c226dfa092a7cd4f4b4586248c..56b1b5b5d7fe06837b5e33b65004549a0d96495f 100644 --- a/test/cpp/inference/api/paddle_infer_api_copy_tensor_tester.cc +++ b/test/cpp/inference/api/paddle_infer_api_copy_tensor_tester.cc @@ -19,10 +19,10 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/fluid/inference/api/paddle_infer_contrib.h" #include "paddle/fluid/platform/float16.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle_infer { diff --git a/test/cpp/inference/api/paddle_infer_api_errors_tester.cc b/test/cpp/inference/api/paddle_infer_api_errors_tester.cc index c716115ce2a4c99e18e33a6e28fda89a3ce50218..4fc1f6f0843c0d55d2f92789d6bca012d4d4a292 100644 --- a/test/cpp/inference/api/paddle_infer_api_errors_tester.cc +++ b/test/cpp/inference/api/paddle_infer_api_errors_tester.cc @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "gflags/gflags.h" #include "glog/logging.h" #include "gtest/gtest.h" #include "paddle/fluid/inference/api/paddle_infer_contrib.h" #include "paddle/fluid/platform/enforce.h" +#include "paddle/utils/flags.h" namespace paddle_infer { namespace contrib { diff --git a/test/cpp/inference/api/paddle_infer_api_test.cc b/test/cpp/inference/api/paddle_infer_api_test.cc index c59ac40e5e5ecb9485503280ecc0f867105d693d..e53473a520d8030f280f3424747793df19a0481a 100644 --- a/test/cpp/inference/api/paddle_infer_api_test.cc +++ b/test/cpp/inference/api/paddle_infer_api_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/tester_helper.h" namespace paddle_infer { diff --git a/test/cpp/inference/api/tester_helper.h b/test/cpp/inference/api/tester_helper.h index fe016abdaee02b0fc243dd052ac10f3bebad62dd..a204c31cfd306bd581e9da46851200d99429b3e8 100644 --- a/test/cpp/inference/api/tester_helper.h +++ b/test/cpp/inference/api/tester_helper.h @@ -40,83 +40,87 @@ #include "test/cpp/inference/api/config_printer.h" #include "test/cpp/inference/test_helper.h" -DEFINE_string(model_name, "", "model name"); -DEFINE_string(infer_model, "", "model path"); -DEFINE_string(fp32_model, "", "FP32 model path"); -DEFINE_string(int8_model, "", "INT8 model path"); -DEFINE_string(infer_data, "", "data file"); -DEFINE_string(refer_result, "", "reference result for comparison"); -DEFINE_int32(batch_size, 1, "batch size"); -DEFINE_bool(ernie_large, false, "Test ernie large"); -DEFINE_bool(with_accuracy_layer, - true, - "Calculate the accuracy while label is in the input"); -DEFINE_bool(enable_fp32, true, "Enable FP32 type prediction"); -DEFINE_bool(enable_bf16, false, "Enable BF16 type prediction"); -DEFINE_bool(enable_int8_ptq, - false, - "Enable INT8 post-training quantization prediction"); -DEFINE_bool(enable_int8_qat, - false, - "Enable INT8 quant-aware training prediction"); -DEFINE_int32(warmup_batch_size, 100, "batch size for quantization warmup"); +PD_DEFINE_string(model_name, "", "model name"); +PD_DEFINE_string(infer_model, "", "model path"); +PD_DEFINE_string(fp32_model, "", "FP32 model path"); +PD_DEFINE_string(int8_model, "", "INT8 model path"); +PD_DEFINE_string(infer_data, "", "data file"); +PD_DEFINE_string(refer_result, "", "reference result for comparison"); +PD_DEFINE_int32(batch_size, 1, "batch size"); +PD_DEFINE_bool(ernie_large, false, "Test ernie large"); +PD_DEFINE_bool(with_accuracy_layer, + true, + "Calculate the accuracy while label is in the input"); +PD_DEFINE_bool(enable_fp32, true, "Enable FP32 type prediction"); +PD_DEFINE_bool(enable_bf16, false, "Enable BF16 type prediction"); +PD_DEFINE_bool(enable_int8_ptq, + false, + "Enable INT8 post-training quantization prediction"); +PD_DEFINE_bool(enable_int8_qat, + false, + "Enable INT8 quant-aware training prediction"); +PD_DEFINE_int32(warmup_batch_size, 100, "batch size for quantization warmup"); // setting iterations to 0 means processing the whole dataset -DEFINE_int32(iterations, 0, "number of batches to process"); -DEFINE_int32(repeat, 1, "Running the inference program repeat times."); -DEFINE_bool(test_all_data, false, "Test the all dataset in data file."); -DEFINE_int32(num_threads, 1, "Running the inference program in multi-threads."); -DEFINE_bool(use_analysis, - true, - "Running the inference program in analysis mode."); -DEFINE_bool(record_benchmark, - false, - "Record benchmark after profiling the model"); -DEFINE_double(accuracy, 1e-3, "Result Accuracy."); -DEFINE_double(quantized_accuracy, 1e-2, "Result Quantized Accuracy."); -DEFINE_bool(zero_copy, false, "Use ZeroCopy to speedup Feed/Fetch."); -DEFINE_bool(warmup, - false, - "Use warmup to calculate elapsed_time more accurately. " - "To reduce CI time, it sets false in default."); -DEFINE_int32(warmup_iters, 1, "Number of batches to process during warmup."); - -DEFINE_bool(enable_profile, false, "Turn on profiler for fluid"); -DEFINE_int32(cpu_num_threads, 1, "Number of threads for each paddle instance."); -DEFINE_bool(fuse_multi_gru, - false, - "Running the inference program with multi_gru_fuse_pass"); +PD_DEFINE_int32(iterations, 0, "number of batches to process"); +PD_DEFINE_int32(repeat, 1, "Running the inference program repeat times."); +PD_DEFINE_bool(test_all_data, false, "Test the all dataset in data file."); +PD_DEFINE_int32(num_threads, + 1, + "Running the inference program in multi-threads."); +PD_DEFINE_bool(use_analysis, + true, + "Running the inference program in analysis mode."); +PD_DEFINE_bool(record_benchmark, + false, + "Record benchmark after profiling the model"); +PD_DEFINE_double(accuracy, 1e-3, "Result Accuracy."); +PD_DEFINE_double(quantized_accuracy, 1e-2, "Result Quantized Accuracy."); +PD_DEFINE_bool(zero_copy, false, "Use ZeroCopy to speedup Feed/Fetch."); +PD_DEFINE_bool(warmup, + false, + "Use warmup to calculate elapsed_time more accurately. " + "To reduce CI time, it sets false in default."); +PD_DEFINE_int32(warmup_iters, 1, "Number of batches to process during warmup."); + +PD_DEFINE_bool(enable_profile, false, "Turn on profiler for fluid"); +PD_DEFINE_int32(cpu_num_threads, + 1, + "Number of threads for each paddle instance."); +PD_DEFINE_bool(fuse_multi_gru, + false, + "Running the inference program with multi_gru_fuse_pass"); // ipu related -DEFINE_int32(ipu_micro_batch_size, 1, "micro batch size"); -DEFINE_int32(ipu_device_num, 1, "device num"); -DEFINE_bool(ipu_enable_pipelining, false, "enable pipelining"); -DEFINE_int32(ipu_batches_per_step, - 1, - "the number of batches per run in pipelining"); -DEFINE_bool(ipu_enable_fp16, false, "enable fp16"); -DEFINE_int32(ipu_replica_num, 1, "replica num"); -DEFINE_double(ipu_available_memory_proportion, - 1.0, - "available memory proportion"); -DEFINE_bool(ipu_enable_half_partial, false, "enable half partial"); +PD_DEFINE_int32(ipu_micro_batch_size, 1, "micro batch size"); +PD_DEFINE_int32(ipu_device_num, 1, "device num"); +PD_DEFINE_bool(ipu_enable_pipelining, false, "enable pipelining"); +PD_DEFINE_int32(ipu_batches_per_step, + 1, + "the number of batches per run in pipelining"); +PD_DEFINE_bool(ipu_enable_fp16, false, "enable fp16"); +PD_DEFINE_int32(ipu_replica_num, 1, "replica num"); +PD_DEFINE_double(ipu_available_memory_proportion, + 1.0, + "available memory proportion"); +PD_DEFINE_bool(ipu_enable_half_partial, false, "enable half partial"); namespace paddle { namespace inference { -using paddle::framework::proto::VarType; -using float16 = paddle::platform::float16; +using ::paddle::framework::proto::VarType; +using float16 = ::paddle::platform::float16; template -constexpr paddle::PaddleDType GetPaddleDType(); +constexpr ::paddle::PaddleDType GetPaddleDType(); template <> -constexpr paddle::PaddleDType GetPaddleDType() { - return paddle::PaddleDType::INT64; +constexpr ::paddle::PaddleDType GetPaddleDType() { + return ::paddle::PaddleDType::INT64; } template <> -constexpr paddle::PaddleDType GetPaddleDType() { - return paddle::PaddleDType::FLOAT32; +constexpr ::paddle::PaddleDType GetPaddleDType() { + return ::paddle::PaddleDType::FLOAT32; } void PrintConfig(const PaddlePredictor::Config *config, bool use_analysis) { @@ -521,7 +525,7 @@ void PredictionWarmUp(PaddlePredictor *predictor, PrintTime( batch_size, 1, num_threads, tid, batch_latency, iterations, data_type); if (FLAGS_enable_profile) { - paddle::platform::ResetProfiler(); + ::paddle::platform::ResetProfiler(); } } @@ -749,7 +753,7 @@ float CompareAccuracyOne( if (output_slots[i][compared_idx].lod.size() > 0) throw std::invalid_argument("CompareAccuracy: output has nonempty LoD."); - if (output_slots[i][compared_idx].dtype != paddle::PaddleDType::FLOAT32) + if (output_slots[i][compared_idx].dtype != ::paddle::PaddleDType::FLOAT32) throw std::invalid_argument( "CompareAccuracy: output is of a wrong type."); @@ -1156,7 +1160,7 @@ static bool CompareTensor(const phi::DenseTensor &a, return true; } -void ConvertFP32toFP16(paddle::PaddleTensor &tensor // NOLINT +void ConvertFP32toFP16(::paddle::PaddleTensor &tensor // NOLINT ) { int num = 1; for (auto dim : tensor.shape) { @@ -1177,7 +1181,7 @@ void ConvertFP32toFP16(paddle::PaddleTensor &tensor // NOLINT tensor.dtype = PaddleDType::FLOAT16; } -void ConvertFP16toFP32(paddle::PaddleTensor &tensor // NOLINT +void ConvertFP16toFP32(::paddle::PaddleTensor &tensor // NOLINT ) { int num = 1; for (auto dim : tensor.shape) { diff --git a/test/cpp/inference/api/trt_cascade_rcnn_test.cc b/test/cpp/inference/api/trt_cascade_rcnn_test.cc index 86759c33e472ee85fe3854f2363765e4e6998ade..710e6481d018ca429b18e65bb8be60a85bfcf937 100644 --- a/test/cpp/inference/api/trt_cascade_rcnn_test.cc +++ b/test/cpp/inference/api/trt_cascade_rcnn_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_dynamic_shape_ernie_fp16_serialize_deserialize_test.cc b/test/cpp/inference/api/trt_dynamic_shape_ernie_fp16_serialize_deserialize_test.cc index 806950ca8d6fda953e109867ea32d7a960855119..34ddb8fa3c330fc848beb4d8035a6abcca928a23 100644 --- a/test/cpp/inference/api/trt_dynamic_shape_ernie_fp16_serialize_deserialize_test.cc +++ b/test/cpp/inference/api/trt_dynamic_shape_ernie_fp16_serialize_deserialize_test.cc @@ -23,7 +23,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.cc b/test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.cc index bcf82d66f781ed2c441de219dfbbe55b175f937b..7946b7c2428b2d3719bd1a1793694ec4b6c31f10 100644 --- a/test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.cc +++ b/test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.cc @@ -23,7 +23,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h b/test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h index 09b20d23e9708d1401735543951325cbcff1e1c4..e046181dbf094ccb5906d2321a501b706050f535 100644 --- a/test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h +++ b/test/cpp/inference/api/trt_dynamic_shape_ernie_serialize_deserialize_test.h @@ -25,7 +25,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_dynamic_shape_ernie_test.cc b/test/cpp/inference/api/trt_dynamic_shape_ernie_test.cc index 8abf7224a135e40388e4f5e8a00c59b4ee8ec8d2..fd31613c2b6289f0f95512e7193ebd2e1580614e 100644 --- a/test/cpp/inference/api/trt_dynamic_shape_ernie_test.cc +++ b/test/cpp/inference/api/trt_dynamic_shape_ernie_test.cc @@ -14,7 +14,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "paddle/fluid/inference/tensorrt/helper.h" #include "test/cpp/inference/api/trt_test_helper.h" diff --git a/test/cpp/inference/api/trt_dynamic_shape_test.cc b/test/cpp/inference/api/trt_dynamic_shape_test.cc index 505c0cdf083c0be3eaac9e9eede7dd0ceb813c20..8f284d75b7e3ceff9642d4a2e69dbfa061326f11 100644 --- a/test/cpp/inference/api/trt_dynamic_shape_test.cc +++ b/test/cpp/inference/api/trt_dynamic_shape_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_dynamic_shape_transformer_prune_test.cc b/test/cpp/inference/api/trt_dynamic_shape_transformer_prune_test.cc index 2e71da39b5f453792809c185f0f49363b1452e2b..ff8c60df00559586e2a49348dc9313bd9761ceb6 100644 --- a/test/cpp/inference/api/trt_dynamic_shape_transformer_prune_test.cc +++ b/test/cpp/inference/api/trt_dynamic_shape_transformer_prune_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_fc_prelu_test.cc b/test/cpp/inference/api/trt_fc_prelu_test.cc index a1ef33407596adb3ae345cca56a161eb2356e7eb..5f10c12bf3dd13e8b27aef4c7f5f53b43b3d18ca 100644 --- a/test/cpp/inference/api/trt_fc_prelu_test.cc +++ b/test/cpp/inference/api/trt_fc_prelu_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_instance_norm_converter_test.cc b/test/cpp/inference/api/trt_instance_norm_converter_test.cc index b58ddb2d919d8e7bf500429030569d1f4c570969..fc78219a9db6d1ca6eb5a80d6341e7c17059c6c0 100644 --- a/test/cpp/inference/api/trt_instance_norm_converter_test.cc +++ b/test/cpp/inference/api/trt_instance_norm_converter_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_mark_trt_engine_outputs_test.cc b/test/cpp/inference/api/trt_mark_trt_engine_outputs_test.cc index 9c6a87a6d161a7d102d2ee46d26ca6bd1f89dd49..d34d640cfaf40eb82432886e3d6899d51eb1d3f9 100644 --- a/test/cpp/inference/api/trt_mark_trt_engine_outputs_test.cc +++ b/test/cpp/inference/api/trt_mark_trt_engine_outputs_test.cc @@ -12,7 +12,6 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_mobilenet_test.cc b/test/cpp/inference/api/trt_mobilenet_test.cc index 7cae99e0d3479a552876b36b8bae728bc67b7c4b..670eaa7b1169e05b44bd9dc2b797ddd0edde7e91 100644 --- a/test/cpp/inference/api/trt_mobilenet_test.cc +++ b/test/cpp/inference/api/trt_mobilenet_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_quant_int8_test.cc b/test/cpp/inference/api/trt_quant_int8_test.cc index f40b2197fb2292cdcb43dc80559498f208023651..46c1fb7c9f742b5107f2bfd697948bf5cb9790c6 100644 --- a/test/cpp/inference/api/trt_quant_int8_test.cc +++ b/test/cpp/inference/api/trt_quant_int8_test.cc @@ -17,7 +17,7 @@ limitations under the License. */ #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_quant_int8_yolov3_r50_test.cc b/test/cpp/inference/api/trt_quant_int8_yolov3_r50_test.cc index ce058a1275c92647f6e45a942eb468531645da7b..412aeae6ed75ad98ef11ed65af7457abc2b7a52a 100644 --- a/test/cpp/inference/api/trt_quant_int8_yolov3_r50_test.cc +++ b/test/cpp/inference/api/trt_quant_int8_yolov3_r50_test.cc @@ -14,7 +14,7 @@ limitations under the License. */ #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_rebind_stream_test.cc b/test/cpp/inference/api/trt_rebind_stream_test.cc index 3a42af93427c505c680f3e6e0a42b951522db7a1..8c8e78a51907e305f4946ca791c30f15f04e7909 100644 --- a/test/cpp/inference/api/trt_rebind_stream_test.cc +++ b/test/cpp/inference/api/trt_rebind_stream_test.cc @@ -16,8 +16,8 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "paddle/fluid/inference/api/paddle_inference_api.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/tester_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_resnet50_test.cc b/test/cpp/inference/api/trt_resnet50_test.cc index 8dde6a0f5dd8ceeff65ac64eebe1f6fff5d6ee5d..085b64ef882b5697ddad999f8da41406a5e61b08 100644 --- a/test/cpp/inference/api/trt_resnet50_test.cc +++ b/test/cpp/inference/api/trt_resnet50_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_resnext_test.cc b/test/cpp/inference/api/trt_resnext_test.cc index a80058468d576870ba53de3ed60536f97187ae28..65e09d3532d86b5ef13d272ca21302d33f83dcff 100644 --- a/test/cpp/inference/api/trt_resnext_test.cc +++ b/test/cpp/inference/api/trt_resnext_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_split_converter_test.cc b/test/cpp/inference/api/trt_split_converter_test.cc index ca41ac5681e4e10c0f689b1577ab5883e856bb5c..8d87b98f6e34bf3370bf4f80b1b17cfc89120fe7 100644 --- a/test/cpp/inference/api/trt_split_converter_test.cc +++ b/test/cpp/inference/api/trt_split_converter_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/trt_test_helper.h" namespace paddle { diff --git a/test/cpp/inference/api/trt_test_helper.h b/test/cpp/inference/api/trt_test_helper.h index db446e64054c03ab23030ab98cc9c8a9354cd1f7..dccbb589bdb0b2e09279db63f8d8fc92f5b8870a 100644 --- a/test/cpp/inference/api/trt_test_helper.h +++ b/test/cpp/inference/api/trt_test_helper.h @@ -17,17 +17,17 @@ limitations under the License. */ #include #include -#include "gflags/gflags.h" #include "glog/logging.h" #include "gtest/gtest.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/tester_helper.h" namespace paddle { namespace inference { -DEFINE_bool(use_tensorrt, true, "Test the performance of TensorRT engine."); -DEFINE_string(prog_filename, "", "Name of model file."); -DEFINE_string(param_filename, "", "Name of parameters file."); +PD_DEFINE_bool(use_tensorrt, true, "Test the performance of TensorRT engine."); +PD_DEFINE_string(prog_filename, "", "Name of model file."); +PD_DEFINE_string(param_filename, "", "Name of parameters file."); template void SetConfig(ConfigType* config, diff --git a/test/cpp/inference/api/xpu_config_resnet50_test.cc b/test/cpp/inference/api/xpu_config_resnet50_test.cc index ce3796e420970d13ea55f9d50e84ec6ca6deae10..d118eef9e88fd48d2d086484495654566282a030 100644 --- a/test/cpp/inference/api/xpu_config_resnet50_test.cc +++ b/test/cpp/inference/api/xpu_config_resnet50_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/tester_helper.h" namespace paddle_infer { diff --git a/test/cpp/inference/api/xpu_runtime_config_resnet50_test.cc b/test/cpp/inference/api/xpu_runtime_config_resnet50_test.cc index b9ab6ea68d7b5c7187f9d22ab8bc8f669cde8d47..94de193c89513c19bb972a0e2538182c06bd0ea3 100644 --- a/test/cpp/inference/api/xpu_runtime_config_resnet50_test.cc +++ b/test/cpp/inference/api/xpu_runtime_config_resnet50_test.cc @@ -15,7 +15,7 @@ limitations under the License. */ #include #include #include -#include "gflags/gflags.h" +#include "paddle/utils/flags.h" #include "test/cpp/inference/api/tester_helper.h" #include "xpu/runtime.h" #include "xpu/xdnn.h" diff --git a/test/cpp/inference/infer_ut/test_LeViT.cc b/test/cpp/inference/infer_ut/test_LeViT.cc index 056371b0ae662a76445c06162b07979b4bf8eaee..ed30f04e7301bdb6210f48ad7b5961a5512bef67 100644 --- a/test/cpp/inference/infer_ut/test_LeViT.cc +++ b/test/cpp/inference/infer_ut/test_LeViT.cc @@ -248,6 +248,6 @@ TEST(tensorrt_tester_LeViT, multi_stream_thread4_trt_fp32_bz2) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); return RUN_ALL_TESTS(); } diff --git a/test/cpp/inference/infer_ut/test_det_mv3_db.cc b/test/cpp/inference/infer_ut/test_det_mv3_db.cc index a407251ccba6668455c9fbe81c1d022b999d6529..474d0701225f57ad1fb27940ae32f5181afd2e03 100644 --- a/test/cpp/inference/infer_ut/test_det_mv3_db.cc +++ b/test/cpp/inference/infer_ut/test_det_mv3_db.cc @@ -183,6 +183,6 @@ TEST(mkldnn_tester_det_mv3_db, multi_thread2_mkl_fp32_bz2) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); return RUN_ALL_TESTS(); } diff --git a/test/cpp/inference/infer_ut/test_ernie_text_cls.cc b/test/cpp/inference/infer_ut/test_ernie_text_cls.cc index 2ffeb604230f7bbd44ec9bb58487b69d851329cb..ddf2dbc49d8979548114037bc5b7788e9c370329 100644 --- a/test/cpp/inference/infer_ut/test_ernie_text_cls.cc +++ b/test/cpp/inference/infer_ut/test_ernie_text_cls.cc @@ -134,6 +134,6 @@ TEST(mkldnn_tester_ernie_text_cls, multi_thread4_mkl_fp32_bz2) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); return RUN_ALL_TESTS(); } diff --git a/test/cpp/inference/infer_ut/test_ernie_xnli_int8.cc b/test/cpp/inference/infer_ut/test_ernie_xnli_int8.cc index e3030d08021ce02cf69018a5a19e52964157432d..4e9c96c530a1007c1e74b70f3a2cd6cd0c833b81 100644 --- a/test/cpp/inference/infer_ut/test_ernie_xnli_int8.cc +++ b/test/cpp/inference/infer_ut/test_ernie_xnli_int8.cc @@ -192,7 +192,7 @@ TEST(tensorrt_tester_ernie_xnli, oss_varlen_truth_data_int8) { int main(int argc, char **argv) { ::testing::InitGoogleTest(&argc, argv); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); #if IS_TRT_VERSION_GE(7200) return RUN_ALL_TESTS(); diff --git a/test/cpp/inference/infer_ut/test_mobilnetv1.cc b/test/cpp/inference/infer_ut/test_mobilnetv1.cc index 582c34e1b0b47b09072aaef9047ef3eba2ed3e23..2660cc5cbd5d655c1eb20f54f1603d55ca6b0586 100644 --- a/test/cpp/inference/infer_ut/test_mobilnetv1.cc +++ b/test/cpp/inference/infer_ut/test_mobilnetv1.cc @@ -81,6 +81,6 @@ TEST(tensorrt_tester_mobilenetv1, tuned_dynamic_trt_fp32_bz2) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); return RUN_ALL_TESTS(); } diff --git a/test/cpp/inference/infer_ut/test_ppyolo_mbv3.cc b/test/cpp/inference/infer_ut/test_ppyolo_mbv3.cc index a075192a58054bdff2787369576981d0bf93dcf4..407e7c87dc97c6a85c2a5cec47e87a9216a4a4c0 100644 --- a/test/cpp/inference/infer_ut/test_ppyolo_mbv3.cc +++ b/test/cpp/inference/infer_ut/test_ppyolo_mbv3.cc @@ -155,6 +155,6 @@ TEST(DISABLED_mkldnn_tester_ppyolo_mbv3, multi_thread4_mkl_bz2) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); return RUN_ALL_TESTS(); } diff --git a/test/cpp/inference/infer_ut/test_ppyolov2_r50vd.cc b/test/cpp/inference/infer_ut/test_ppyolov2_r50vd.cc index ad91cb5bc9e85563c4cfc1006ca41e808fd71614..c90256fb8b8c12ab4727069490d4fd60dd4b431e 100644 --- a/test/cpp/inference/infer_ut/test_ppyolov2_r50vd.cc +++ b/test/cpp/inference/infer_ut/test_ppyolov2_r50vd.cc @@ -154,6 +154,6 @@ TEST(mkldnn_tester_ppyolov2_r50vd, multi_thread2_mkl_bz2) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); return RUN_ALL_TESTS(); } diff --git a/test/cpp/inference/infer_ut/test_resnet50.cc b/test/cpp/inference/infer_ut/test_resnet50.cc index 1c9b70564929f2f87ed8f9941aa628eae94f1ec3..50b0b71c2e1da86dfe58aebbc3fa8a2ca963688e 100644 --- a/test/cpp/inference/infer_ut/test_resnet50.cc +++ b/test/cpp/inference/infer_ut/test_resnet50.cc @@ -242,6 +242,6 @@ TEST(DISABLED_tensorrt_tester_resnet50, profile_multi_thread_trt_fp32) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); return RUN_ALL_TESTS(); } diff --git a/test/cpp/inference/infer_ut/test_resnet50_quant.cc b/test/cpp/inference/infer_ut/test_resnet50_quant.cc index 452cf31b311392cc4e8f22c6d152f36c21f39169..c2c44102f56b69828ea8486be689c5f1cd08e2b9 100644 --- a/test/cpp/inference/infer_ut/test_resnet50_quant.cc +++ b/test/cpp/inference/infer_ut/test_resnet50_quant.cc @@ -171,6 +171,6 @@ TEST(DISABLED_tensorrt_tester_resnet50_quant, multi_thread_multi_instance) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); return RUN_ALL_TESTS(); } diff --git a/test/cpp/inference/infer_ut/test_yolov3.cc b/test/cpp/inference/infer_ut/test_yolov3.cc index 4dc35af4959354cb3e3f7e6e04c3f25efd4dd539..4a3e65ba5a574cb41a0879e7e359df47c731eba5 100644 --- a/test/cpp/inference/infer_ut/test_yolov3.cc +++ b/test/cpp/inference/infer_ut/test_yolov3.cc @@ -154,6 +154,6 @@ TEST(test_yolov3, multi_thread4_mkl_bz2) { int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); - ::GFLAGS_NAMESPACE::ParseCommandLineFlags(&argc, &argv, true); + gflags::ParseCommandLineFlags(&argc, &argv, true); return RUN_ALL_TESTS(); } diff --git a/test/cpp/phi/api/scale_api.h b/test/cpp/phi/api/scale_api.h index 571ab0defbce74d3ea83a05e27956c79d4db36a8..104034d5effe99ea8e1a1853cf6455b5fdfe18bf 100644 --- a/test/cpp/phi/api/scale_api.h +++ b/test/cpp/phi/api/scale_api.h @@ -14,7 +14,6 @@ #pragma once -#include "gflags/gflags.h" #include "glog/logging.h" #include "paddle/phi/api/include/tensor.h" #include "paddle/phi/api/lib/kernel_dispatch.h" @@ -26,6 +25,7 @@ #include "paddle/phi/core/meta_tensor.h" #include "paddle/phi/infermeta/unary.h" #include "paddle/phi/kernels/scale_kernel.h" +#include "paddle/utils/flags.h" PHI_DECLARE_int32(low_precision_op_list); namespace paddle { diff --git a/test/cpp/prim/test_static_prim.cc b/test/cpp/prim/test_static_prim.cc index e26f54a44be43087701f800bfea6e43324865b47..d4f5dcb8998ae7f720fc1ffe5bb46e4dc6294c07 100644 --- a/test/cpp/prim/test_static_prim.cc +++ b/test/cpp/prim/test_static_prim.cc @@ -28,7 +28,7 @@ #include "paddle/phi/core/flags.h" #include "paddle/phi/core/kernel_registry.h" -DECLARE_bool(prim_enabled); +PD_DECLARE_bool(prim_enabled); PHI_DECLARE_string(tensor_operants_mode); PD_DECLARE_KERNEL(full, CPU, ALL_LAYOUT);