fix

paddle/fluid/operators/distributed/parameter_prefetch.cc

@@ -195,8 +195,8 @@ void prefetchs(const std::vector<std::string> &id_var_names,
                const std::vector<std::string> &endpoints,
                const framework::ExecutionContext &context,
                const framework::Scope &scope) {
-  platform::RecordEvent record_event("Distributed_lookup_table::prefetchs",
-                                     platform::EventRole::kInnerOp);
+  platform::PushEvent("Distributed_lookup_table::prefetchs",
+                      platform::EventRole::kInnerOp);
   auto vec_dim_1 = 0;
   auto vec_dim_0 = 0;
   framework::Variable *var = scope.FindVar(persistable_var_name);
@@ -304,6 +304,8 @@ void prefetchs(const std::vector<std::string> &id_var_names,
 #endif
     }
   }
+  platform::PopEvent("Distributed_lookup_table::prefetchs",
+                     platform::EventRole::kInnerOp);
 }
 
 };  // namespace distributed

paddle/fluid/operators/distributed/parameter_recv.cc

@@ -43,8 +43,8 @@ using DDim = framework::DDim;
 template <typename T>
 void RecvSelectedRows(const CommContext &rpc_ctx,
                       const framework::Scope &scope) {
-  platform::RecordEvent record_event("ParameterRecv::RecvSelectedRows",
-                                     platform::EventRole::kInnerOp);
+  platform::RecordEvent record_event_grpc("ParameterRecv::RecvSelectedRows",
+                                          platform::EventRole::kInnerOp);
   platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance();
   auto cpu_place = platform::CPUPlace();
   auto &cpu_ctx = *pool.Get(cpu_place);
@@ -114,8 +114,8 @@ void RecvSelectedRows(const CommContext &rpc_ctx,
 
 template <typename T>
 void RecvLodTensor(const CommContext &rpc_ctx, const framework::Scope &scope) {
-  platform::RecordEvent record_event("ParameterRecv::RecvLodTensor",
-                                     platform::EventRole::kInnerOp);
+  platform::RecordEvent record_event_grpc("ParameterRecv::RecvLodTensor",
+                                          platform::EventRole::kInnerOp);
   platform::DeviceContextPool &pool = platform::DeviceContextPool::Instance();
   auto cpu_place = platform::CPUPlace();
   auto &cpu_ctx = *pool.Get(cpu_place);

paddle/fluid/operators/distributed/parameter_send.cc

@@ -98,8 +98,6 @@ template <typename T>
 void ParameterSend<T>::operator()(const CommContext &rpc_ctx,
                                   const framework::Scope &scope, bool sync,
                                   int multi_parts) {
-  platform::RecordEvent record_event("ParameterSend::operator",
-                                     platform::EventRole::kInnerOp);
   if (rpc_ctx.var_name == STEP_COUNTER) {
     SendByNotifyRPC(rpc_ctx, scope);
     return;
@@ -117,8 +115,8 @@ void ParameterSend<T>::operator()(const CommContext &rpc_ctx,
   auto *send_var = scope.FindVar(rpc_ctx.var_name);
 
   if (send_var->IsType<framework::LoDTensor>()) {
-    platform::RecordEvent record_event("ParameterSend::LoDTensor",
-                                       platform::EventRole::kInnerOp);
+    platform::RecordEvent record_event_grpc("ParameterSend::LoDTensor",
+                                            platform::EventRole::kInnerOp);
     size_t out_num = rpc_ctx.splited_varnames.size();
     if (out_num > 1) {
       auto &send_tensor = send_var->Get<framework::LoDTensor>();
@@ -167,8 +165,8 @@ void ParameterSend<T>::operator()(const CommContext &rpc_ctx,
       }
     }
   } else if (send_var->IsType<framework::SelectedRows>()) {
-    platform::RecordEvent record_event("ParameterSend::SelectedRows",
-                                       platform::EventRole::kInnerOp);
+    platform::RecordEvent record_event_grpc("ParameterSend::SelectedRows",
+                                            platform::EventRole::kInnerOp);
     auto &send_slr = send_var->Get<framework::SelectedRows>();
 
     auto &send_rows = send_slr.rows();

paddle/fluid/platform/device_tracer.cc

@@ -694,8 +694,16 @@ Event *CurAnnotation() {
   return annotation_stack.back();
 }
 std::string CurAnnotationName() {
-  if (annotation_stack.empty()) return "Unknown";
-  return annotation_stack.back()->name();
+  std::string curr_annotation_name = "Unknown";
+  try {
+    if (annotation_stack.empty()) {
+      return curr_annotation_name;
+    }
+    return annotation_stack.back()->name();
+  } catch (...) {
+    VLOG(1) << "CurAnnotationName Failed";
+  }
+  return curr_annotation_name;
 }
 
 void SetCurBlock(int block_id) { block_id_stack.push_back(block_id); }