From 406f1b9650e072abf04ac7572b24d89399d98343 Mon Sep 17 00:00:00 2001 From: liutiexing <74819124+liutiexing@users.noreply.github.com> Date: Mon, 28 Feb 2022 11:25:55 +0800 Subject: [PATCH] Update host tracer (#39975) * add align for WorkQueue * add spinlock * merge develop * merge * Add EventsWaiter * Revert "Add EventsWaiter" This reverts commit e206173aa9be7401b83a53581627bfaf557c8fb2. * update HostTracer * fix * update * update Co-authored-by: liutiexing --- .../details/fast_threaded_ssa_graph_executor.cc | 2 ++ paddle/fluid/platform/os_info.cc | 4 +--- paddle/fluid/platform/os_info.h | 3 ++- paddle/fluid/platform/profiler/host_tracer.cc | 11 ++++++++++- paddle/fluid/platform/profiler/host_tracer.h | 8 ++++---- .../fluid/platform/profiler/trace_event_collector.h | 11 +++++++++++ 6 files changed, 30 insertions(+), 9 deletions(-) diff --git a/paddle/fluid/framework/details/fast_threaded_ssa_graph_executor.cc b/paddle/fluid/framework/details/fast_threaded_ssa_graph_executor.cc index 1cf69a1a3d6..1b2b2476289 100644 --- a/paddle/fluid/framework/details/fast_threaded_ssa_graph_executor.cc +++ b/paddle/fluid/framework/details/fast_threaded_ssa_graph_executor.cc @@ -231,6 +231,8 @@ void FastThreadedSSAGraphExecutor::RunOpAsync( OpHandleBase *op, const std::shared_ptr> &complete_q) { ++remaining_; + platform::RecordEvent("WorkQueue::AddTask", + platform::TracerEventType::UserDefined, 10 /*level*/); this->pool_->enqueue([=] { std::deque op_queue; op_queue.push_front(op); diff --git a/paddle/fluid/platform/os_info.cc b/paddle/fluid/platform/os_info.cc index 58d37783d05..36dd7891d55 100644 --- a/paddle/fluid/platform/os_info.cc +++ b/paddle/fluid/platform/os_info.cc @@ -95,8 +95,6 @@ std::unordered_map GetAllThreadIds() { return res; } -static constexpr const char* kDefaultThreadName = "unset"; - std::string GetCurrentThreadName() { const auto& thread_name = internal::ThreadDataRegistry::GetInstance() @@ -112,7 +110,7 @@ std::unordered_map GetAllThreadNames() { bool SetCurrentThreadName(const std::string& name) { auto& instance = internal::ThreadDataRegistry::GetInstance(); const auto& cur_name = instance.GetCurrentThreadData(); - if (!cur_name.empty() || cur_name == kDefaultThreadName) { + if (!cur_name.empty() || name.empty() || name == kDefaultThreadName) { return false; } instance.SetCurrentThreadData(name); diff --git a/paddle/fluid/platform/os_info.h b/paddle/fluid/platform/os_info.h index 7f607aaec97..ef894fd3dc2 100644 --- a/paddle/fluid/platform/os_info.h +++ b/paddle/fluid/platform/os_info.h @@ -57,7 +57,8 @@ ThreadId GetCurrentThreadId(); // create/destory when using it. std::unordered_map GetAllThreadIds(); -// Returns 'unset' if SetCurrentThreadName is never called. +static constexpr const char* kDefaultThreadName = "unset"; +// Returns kDefaultThreadName if SetCurrentThreadName is never called. std::string GetCurrentThreadName(); // Return the map from StdTid to ThreadName diff --git a/paddle/fluid/platform/profiler/host_tracer.cc b/paddle/fluid/platform/profiler/host_tracer.cc index 2172fe4d1e3..3f97113aecb 100644 --- a/paddle/fluid/platform/profiler/host_tracer.cc +++ b/paddle/fluid/platform/profiler/host_tracer.cc @@ -26,6 +26,9 @@ void ProcessHostEvents(const HostEventSection& host_events, TraceEventCollector* collector) { for (const auto& thr_sec : host_events.thr_sections) { uint64_t tid = thr_sec.thread_id; + if (thr_sec.thread_name != kDefaultThreadName) { + collector->AddThreadName(tid, thr_sec.thread_name); + } for (const auto& evt : thr_sec.events) { HostTraceEvent event; event.name = evt.name; @@ -41,12 +44,18 @@ void ProcessHostEvents(const HostEventSection& host_events, } // namespace +void HostTracer::PrepareTracing() { + // warm up + HostTraceLevel::GetInstance().SetLevel(options_.trace_level); + state_ = TracerState::READY; +} + void HostTracer::StartTracing() { PADDLE_ENFORCE_EQ( state_ == TracerState::READY || state_ == TracerState::STOPED, true, platform::errors::PreconditionNotMet("TracerState must be READY")); HostEventRecorder::GetInstance().GatherEvents(); - HostTraceLevel::GetInstance().SetLevel(trace_level_); + HostTraceLevel::GetInstance().SetLevel(options_.trace_level); state_ = TracerState::STARTED; } diff --git a/paddle/fluid/platform/profiler/host_tracer.h b/paddle/fluid/platform/profiler/host_tracer.h index b6c10e558b7..d05e829357f 100644 --- a/paddle/fluid/platform/profiler/host_tracer.h +++ b/paddle/fluid/platform/profiler/host_tracer.h @@ -45,9 +45,9 @@ struct HostTracerOptions { class HostTracer : public TracerBase { public: - explicit HostTracer(const HostTracerOptions& options) { - trace_level_ = options.trace_level; - } + explicit HostTracer(const HostTracerOptions& options) : options_(options) {} + + void PrepareTracing() override; void StartTracing() override; @@ -56,7 +56,7 @@ class HostTracer : public TracerBase { void CollectTraceData(TraceEventCollector* collector) override; private: - uint32_t trace_level_; + HostTracerOptions options_; }; } // namespace platform diff --git a/paddle/fluid/platform/profiler/trace_event_collector.h b/paddle/fluid/platform/profiler/trace_event_collector.h index 30b32220d9f..cc85a178d14 100644 --- a/paddle/fluid/platform/profiler/trace_event_collector.h +++ b/paddle/fluid/platform/profiler/trace_event_collector.h @@ -15,6 +15,8 @@ limitations under the License. */ #pragma once #include +#include +#include #include "paddle/fluid/platform/profiler/trace_event.h" namespace paddle { @@ -32,6 +34,10 @@ class TraceEventCollector { device_events_.push_back(event); } + void AddThreadName(uint64_t tid, const std::string& name) { + thread_names_[tid] = name; + } + const std::list& HostEvents() const { return host_events_; } const std::list& RuntimeEvents() const { @@ -42,7 +48,12 @@ class TraceEventCollector { return device_events_; } + const std::unordered_map& ThreadNames() const { + return thread_names_; + } + private: + std::unordered_map thread_names_; std::list host_events_; std::list runtime_events_; std::list device_events_; -- GitLab