From c984cd858378d532beed54557efb095285bc3a40 Mon Sep 17 00:00:00 2001 From: chenjian Date: Mon, 21 Feb 2022 12:49:10 +0800 Subject: [PATCH] Update record interface using part2 (#39694) * fix RecordEvent interface * modify default level to 4 * update interface use * add const default trace level * update record event interface using * update record event interface using * update operator.cc * update part2 * update part1 * fix include profiler.h header in ps server * fix include profiler.h header in ps server * fix profiler.h header --- paddle/fluid/framework/details/scope_buffered_monitor.cc | 8 +++++--- .../details/scope_buffered_ssa_graph_executor.cc | 8 +++++--- .../framework/details/sparse_all_reduce_op_handle.cc | 7 +++++-- .../framework/details/threaded_ssa_graph_executor.cc | 5 +++-- paddle/fluid/framework/executor.cc | 1 + paddle/fluid/framework/ir/cost_model.h | 1 + paddle/fluid/framework/new_executor/interpretercore.cc | 8 ++++++-- .../new_executor/workqueue/nonblocking_threadpool.h | 3 ++- paddle/fluid/framework/parallel_executor.cc | 1 + paddle/fluid/framework/tensor_util.cc | 2 +- paddle/fluid/imperative/basic_engine.cc | 3 ++- paddle/fluid/imperative/layer.cc | 3 ++- paddle/fluid/imperative/partial_grad_engine.cc | 2 +- paddle/fluid/imperative/tracer.cc | 4 +++- paddle/fluid/inference/tensorrt/plugin/trt_plugin.h | 2 +- paddle/fluid/inference/tests/api/tester_helper.h | 2 +- 16 files changed, 40 insertions(+), 20 deletions(-) diff --git a/paddle/fluid/framework/details/scope_buffered_monitor.cc b/paddle/fluid/framework/details/scope_buffered_monitor.cc index 5881148190..57faf0e75b 100644 --- a/paddle/fluid/framework/details/scope_buffered_monitor.cc +++ b/paddle/fluid/framework/details/scope_buffered_monitor.cc @@ -13,7 +13,7 @@ // limitations under the License. #include "paddle/fluid/framework/details/scope_buffered_monitor.h" -#include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" namespace paddle { namespace framework { @@ -91,7 +91,8 @@ void ScopeBufferedMonitor::Apply(const std::function &callback, bool has_fetch) { std::unique_ptr pre_local_exec_scopes_event( new platform::RecordEvent( - "ScopeBufferedMonitor::pre_local_exec_scopes_process")); + "ScopeBufferedMonitor::pre_local_exec_scopes_process", + platform::TracerEventType::UserDefined, 2)); for (size_t scope_id = 0; scope_id < local_exec_scopes_.size(); ++scope_id) { pre_local_exec_scopes_.at(scope_id).clear(); auto scopes = local_exec_scopes_.at(scope_id)->kids(); @@ -105,7 +106,8 @@ void ScopeBufferedMonitor::Apply(const std::function &callback, std::unique_ptr post_local_exec_scopes_event( new platform::RecordEvent( - "ScopeBufferedMonitor::post_local_exec_scopes_process")); + "ScopeBufferedMonitor::post_local_exec_scopes_process", + platform::TracerEventType::UserDefined, 2)); for (size_t scope_id = 0; scope_id < local_exec_scopes_.size(); ++scope_id) { post_local_exec_scopes_.at(scope_id).clear(); auto scopes = local_exec_scopes_.at(scope_id)->kids(); diff --git a/paddle/fluid/framework/details/scope_buffered_ssa_graph_executor.cc b/paddle/fluid/framework/details/scope_buffered_ssa_graph_executor.cc index 5d271d06b6..d496301297 100644 --- a/paddle/fluid/framework/details/scope_buffered_ssa_graph_executor.cc +++ b/paddle/fluid/framework/details/scope_buffered_ssa_graph_executor.cc @@ -23,7 +23,7 @@ #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/framework/variable_helper.h" #include "paddle/fluid/platform/cuda_graph_with_memory_pool.h" -#include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" namespace paddle { namespace framework { @@ -75,7 +75,8 @@ FetchResultType ScopeBufferedSSAGraphExecutor::Run( #endif if (drop_scope_counter_ == 0) { - platform::RecordEvent e("InitLocalVars"); + platform::RecordEvent e("InitLocalVars", + platform::TracerEventType::UserDefined, 2); InitVariables(); } @@ -164,7 +165,8 @@ void ScopeBufferedSSAGraphExecutor::InitVariables() { } void ScopeBufferedSSAGraphExecutor::DropLocalExeScopes(bool need_wait) { - platform::RecordEvent drop_scope_event("DropLocalExeScopes"); + platform::RecordEvent drop_scope_event( + "DropLocalExeScopes", platform::TracerEventType::UserDefined, 2); drop_scope_counter_ = 0; if (need_wait) { for (auto &p : places_) { diff --git a/paddle/fluid/framework/details/sparse_all_reduce_op_handle.cc b/paddle/fluid/framework/details/sparse_all_reduce_op_handle.cc index f0de723c20..d198eb1459 100644 --- a/paddle/fluid/framework/details/sparse_all_reduce_op_handle.cc +++ b/paddle/fluid/framework/details/sparse_all_reduce_op_handle.cc @@ -25,7 +25,7 @@ #include "paddle/fluid/memory/malloc.h" #include "paddle/fluid/platform/cuda_device_guard.h" #include "paddle/fluid/platform/device/gpu/gpu_info.h" -#include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" DECLARE_bool(sync_nccl_allreduce); @@ -66,7 +66,8 @@ SparseAllReduceOpHandle::SparseAllReduceOpHandle( } void SparseAllReduceOpHandle::RunImplEncoded() { - platform::RecordEvent record_event(Name()); + platform::RecordEvent record_event(Name(), + platform::TracerEventType::UserDefined, 2); auto in_var_handles = DynamicCast(this->Inputs()); auto out_var_handles = DynamicCast(this->Outputs()); @@ -279,6 +280,8 @@ bool SparseAllReduceOpHandle::IsEncoded() { } void SparseAllReduceOpHandle::RunImpl() { + platform::RecordEvent record_event( + Name(), platform::TracerEventType::Communication, 1); if (!IsEncoded()) { AllReduceOpHandle::RunImpl(); return; diff --git a/paddle/fluid/framework/details/threaded_ssa_graph_executor.cc b/paddle/fluid/framework/details/threaded_ssa_graph_executor.cc index c8a6cd25f0..39683c9a0d 100644 --- a/paddle/fluid/framework/details/threaded_ssa_graph_executor.cc +++ b/paddle/fluid/framework/details/threaded_ssa_graph_executor.cc @@ -15,7 +15,7 @@ #include "paddle/fluid/framework/details/threaded_ssa_graph_executor.h" #include "paddle/fluid/framework/ir/graph_helper.h" -#include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" #if defined PADDLE_WITH_PSCORE #include "paddle/fluid/distributed/ps/service/communicator/communicator.h" @@ -56,7 +56,8 @@ ThreadedSSAGraphExecutor::ThreadedSSAGraphExecutor( inline FetchResultType ThreadedSSAGraphExecutor::RunImpl( const std::vector &fetch_tensors, bool return_merged) { std::unique_ptr event( - new platform::RecordEvent("ThreadedSSAGraphExecutorPrepare")); + new platform::RecordEvent("ThreadedSSAGraphExecutorPrepare", + platform::TracerEventType::UserDefined, 2)); std::unique_ptr op_deps = op_deps_futures_.get(); CopyOpDeps(); diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc index 4e6a4d5360..48850d4624 100644 --- a/paddle/fluid/framework/executor.cc +++ b/paddle/fluid/framework/executor.cc @@ -22,6 +22,7 @@ limitations under the License. */ #include "paddle/fluid/operators/controlflow/while_op_helper.h" #include "paddle/fluid/platform/place.h" #include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" #ifdef PADDLE_WITH_MKLDNN #include "paddle/fluid/platform/mkldnn_helper.h" #endif diff --git a/paddle/fluid/framework/ir/cost_model.h b/paddle/fluid/framework/ir/cost_model.h index 41567df2cb..20d9be7e95 100644 --- a/paddle/fluid/framework/ir/cost_model.h +++ b/paddle/fluid/framework/ir/cost_model.h @@ -26,6 +26,7 @@ #include "paddle/fluid/framework/ir/node.h" #include "paddle/fluid/framework/program_desc.h" #include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" #include "paddle/fluid/platform/variant.h" namespace paddle { diff --git a/paddle/fluid/framework/new_executor/interpretercore.cc b/paddle/fluid/framework/new_executor/interpretercore.cc index a73aeddae8..766a3b9e49 100644 --- a/paddle/fluid/framework/new_executor/interpretercore.cc +++ b/paddle/fluid/framework/new_executor/interpretercore.cc @@ -388,7 +388,9 @@ void InterpreterCore::RunInstruction(const Instruction& instr_node) { : global_scope_->GetMutableScope(); auto op_with_kernel = dynamic_cast(op); { - platform::RecordEvent infershape_event("InferShape"); + platform::RecordEvent infershape_event( + "InferShape", platform::TracerEventType::OperatorInner, 1, + platform::EventRole::kInnerOp); // If it is OperatorBase, InferShape do nothing. if (op_with_kernel != nullptr) op_with_kernel->Info().infer_shape_( @@ -408,7 +410,9 @@ void InterpreterCore::RunInstruction(const Instruction& instr_node) { } } { - platform::RecordEvent compute_event("Compute"); + platform::RecordEvent compute_event( + "Compute", platform::TracerEventType::OperatorInner, 1, + platform::EventRole::kInnerOp); if (op_with_kernel == nullptr) { instr_node.OpBase()->Run(*local_scope, place_); } else { diff --git a/paddle/fluid/framework/new_executor/workqueue/nonblocking_threadpool.h b/paddle/fluid/framework/new_executor/workqueue/nonblocking_threadpool.h index 2ad76562c1..7b3916bafc 100644 --- a/paddle/fluid/framework/new_executor/workqueue/nonblocking_threadpool.h +++ b/paddle/fluid/framework/new_executor/workqueue/nonblocking_threadpool.h @@ -408,7 +408,8 @@ class ThreadPoolTempl { ec_.Notify(true); return false; } - platform::RecordEvent("SleepWaitForWork"); + platform::RecordEvent("SleepWaitForWork", + platform::TracerEventType::UserDefined, 2); ec_.CommitWait(waiter); blocked_--; return true; diff --git a/paddle/fluid/framework/parallel_executor.cc b/paddle/fluid/framework/parallel_executor.cc index 1a826f6bdd..5b913ff2d2 100644 --- a/paddle/fluid/framework/parallel_executor.cc +++ b/paddle/fluid/framework/parallel_executor.cc @@ -39,6 +39,7 @@ limitations under the License. */ #include "paddle/fluid/platform/cuda_graph_with_memory_pool.h" #include "paddle/fluid/platform/event.h" #include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) #include "paddle/fluid/platform/cuda_device_guard.h" diff --git a/paddle/fluid/framework/tensor_util.cc b/paddle/fluid/framework/tensor_util.cc index 83161fa679..1eb5727298 100644 --- a/paddle/fluid/framework/tensor_util.cc +++ b/paddle/fluid/framework/tensor_util.cc @@ -23,7 +23,7 @@ limitations under the License. */ #include "paddle/fluid/framework/data_type.h" #include "paddle/fluid/framework/tensor_util.h" #include "paddle/fluid/platform/complex.h" -#include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" #include "paddle/phi/core/dense_tensor.h" diff --git a/paddle/fluid/imperative/basic_engine.cc b/paddle/fluid/imperative/basic_engine.cc index 13b2982b53..97a188e5c9 100644 --- a/paddle/fluid/imperative/basic_engine.cc +++ b/paddle/fluid/imperative/basic_engine.cc @@ -410,7 +410,8 @@ void BasicEngine::Execute() { auto& inplace_grad_name_map = shared_cur_node->InplaceGradNameMap(); for (auto& cur_op : *shared_cur_node) { - platform::RecordEvent op_type_record_event(cur_op.Type()); + platform::RecordEvent op_type_record_event( + cur_op.Type(), platform::TracerEventType::Operator, 1); ++op_num; diff --git a/paddle/fluid/imperative/layer.cc b/paddle/fluid/imperative/layer.cc index 34c46f79f5..f3bd85ff29 100644 --- a/paddle/fluid/imperative/layer.cc +++ b/paddle/fluid/imperative/layer.cc @@ -233,7 +233,8 @@ void VarBase::ClearGradient(bool set_to_zero) { grad_t->mutable_value()->clear(); } } else { - platform::RecordEvent record_event("ClearGradient"); + platform::RecordEvent record_event( + "ClearGradient", platform::TracerEventType::UserDefined, 2); auto* grad_t = grad_var_->MutableVar()->GetMutable(); if (grad_t->IsInitialized()) { diff --git a/paddle/fluid/imperative/partial_grad_engine.cc b/paddle/fluid/imperative/partial_grad_engine.cc index e231d3c180..f1d0c8afdd 100644 --- a/paddle/fluid/imperative/partial_grad_engine.cc +++ b/paddle/fluid/imperative/partial_grad_engine.cc @@ -30,7 +30,7 @@ #include "paddle/fluid/imperative/op_base.h" #include "paddle/fluid/imperative/tracer.h" #include "paddle/fluid/platform/device_context.h" -#include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" #include "paddle/fluid/string/string_helper.h" #include "paddle/phi/kernels/funcs/math_function.h" diff --git a/paddle/fluid/imperative/tracer.cc b/paddle/fluid/imperative/tracer.cc index a669ff9d5a..1c9cc538ff 100644 --- a/paddle/fluid/imperative/tracer.cc +++ b/paddle/fluid/imperative/tracer.cc @@ -22,6 +22,7 @@ #include "paddle/fluid/platform/denormal.h" #include "paddle/fluid/platform/device/device_wrapper.h" #include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" #include "paddle/fluid/string/string_helper.h" DECLARE_bool(use_mkldnn); @@ -171,7 +172,8 @@ void Tracer::TraceOp(const std::string& type, const NameVarMap& ins, const std::map& inplace_map, paddle::framework::AttributeMap* passed_default_attrs_, bool use_default_attr_map) { - platform::RecordEvent op_type_record_event(type); + platform::RecordEvent op_type_record_event( + type, platform::TracerEventType::Operator, 2); platform::ScopedFlushDenormal flush; VLOG(1) << "Trace Op: " << type; if (FLAGS_use_mkldnn) { diff --git a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h index 6b2925a068..9210cd48d0 100644 --- a/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h +++ b/paddle/fluid/inference/tensorrt/plugin/trt_plugin.h @@ -24,7 +24,7 @@ #include "paddle/fluid/inference/tensorrt/helper.h" #include "paddle/fluid/inference/tensorrt/plugin/trt_plugin_utils.h" #include "paddle/fluid/platform/enforce.h" -#include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" namespace nvinfer1 { class ITensor; diff --git a/paddle/fluid/inference/tests/api/tester_helper.h b/paddle/fluid/inference/tests/api/tester_helper.h index cc168ed793..77fab0a86f 100644 --- a/paddle/fluid/inference/tests/api/tester_helper.h +++ b/paddle/fluid/inference/tests/api/tester_helper.h @@ -38,7 +38,7 @@ #include "paddle/fluid/inference/tests/api/config_printer.h" #include "paddle/fluid/inference/tests/test_helper.h" #include "paddle/fluid/inference/utils/benchmark.h" -#include "paddle/fluid/platform/profiler.h" +#include "paddle/fluid/platform/profiler/event_tracing.h" DEFINE_string(model_name, "", "model name"); DEFINE_string(infer_model, "", "model path"); -- GitLab