Add profiler in tracer (#17076)

* add profiler in tracer.cc

* add profiler in layer.cc
test=develop

* add profiler in Layer.cc
test=develop

paddle/fluid/imperative/CMakeLists.txt

 if(WITH_PYTHON)
-cc_library(layer SRCS layer.cc DEPS proto_desc operator device_context blas pybind)
-cc_library(tracer SRCS tracer.cc DEPS proto_desc device_context pybind)
+cc_library(layer SRCS layer.cc DEPS proto_desc operator device_context blas pybind profiler)
+cc_library(tracer SRCS tracer.cc DEPS proto_desc device_context pybind profiler)
 cc_library(engine SRCS engine.cc)
 cc_library(imperative_profiler SRCS profiler.cc)
 cc_library(nccl_context SRCS nccl_context.cc DEPS device_context)

paddle/fluid/imperative/layer.cc

@@ -27,6 +27,7 @@
 #include "paddle/fluid/framework/tensor_util.h"
 #include "paddle/fluid/operators/math/blas.h"
 #include "paddle/fluid/platform/device_context.h"
+#include "paddle/fluid/platform/profiler.h"
 #include "paddle/fluid/string/printf.h"
 
 namespace paddle {
@@ -256,7 +257,6 @@ std::map<std::string, std::vector<VarBase*>> OpBase::ApplyGrad(
     const detail::BackwardStrategy& bck_stratedy) {
   PADDLE_ENFORCE(!grad_op_descs_.empty() || backward_id_ > 0,
                  "%s has no backward implementation", Type());
-
   VLOG(3) << "apply op grad: " << Type();
   std::vector<VarBasePtrMap> tmp_grad_outputs;
   if (backward_id_ > 0) {
@@ -272,8 +272,8 @@ std::map<std::string, std::vector<VarBase*>> OpBase::ApplyGrad(
     tmp_grad_outputs.resize(grad_op_count);
     for (size_t k = 0; k < grad_op_count; ++k) {
       framework::OpDesc* grad_op_desc = grad_op_descs_[k];
+      platform::RecordEvent record_event(grad_op_desc->Type());
       auto& grad_output_variable_map = grad_output_vars_[k];
-
       VLOG(3) << "apply grad op " << grad_op_desc->Type();
 
       // Allocate tmp grad output variable
@@ -345,6 +345,7 @@ std::map<std::string, std::vector<VarBase*>> OpBase::ApplyGrad(
     }
   }
 
+  platform::RecordEvent record_event("merge_grads");
   // Add tmp grad outputs to original grad vars
   for (size_t k = 0; k < grad_output_vars_.size(); ++k) {
     for (const auto& it : grad_output_vars_[k]) {
@@ -424,7 +425,7 @@ void OpBase::RegisterBackwardHooks(const py::object& callable) {
 
 void VarBase::RunBackward(const detail::BackwardStrategy& bck_stratedy) {
   if (!pre_op_) return;
-
+  platform::RecordEvent record_event("Imperative Backward");
   VLOG(3) << "start backward";
   auto grads_t = grads_->var_->GetMutable<framework::LoDTensor>();
   operators::math::set_constant(

paddle/fluid/imperative/tracer.cc

@@ -24,6 +24,7 @@
 #include "paddle/fluid/operators/math/math_function.h"
 #include "paddle/fluid/platform/device_context.h"
 #include "paddle/fluid/platform/enforce.h"
+#include "paddle/fluid/platform/profiler.h"
 
 namespace paddle {
 namespace imperative {
@@ -141,6 +142,7 @@ std::set<std::string> Tracer::Trace(OpBase* op, const VarBasePtrMap& inputs,
                                     framework::AttributeMap attrs_map,
                                     const platform::Place expected_place,
                                     const bool stop_gradient) {
+  platform::RecordEvent record_event(op->type_);
   framework::VariableValueMap invars_map;
   framework::VariableValueMap outvars_map;