add the python callstack for debug support test=develop

paddle/fluid/framework/op_proto_maker.cc

@@ -82,6 +82,10 @@ void OpProtoAndCheckerMaker::operator()(proto::OpProto* proto,
   AddAttr<std::string>(OpNamescopeAttrName(), "Operator name with namesope.")
       .SetDefault("");
 
+  AddAttr<std::vector<std::string>>(OpCreationCallstackAttrName(),
+                                    "Callstack for Op Creatation.")
+      .SetDefault({});
+
   Validate();
 }
 

paddle/fluid/framework/op_proto_maker.h

@@ -47,6 +47,7 @@ class OpProtoAndCheckerMaker {
   static const char *OpRoleAttrName() { return "op_role"; }
   static const char *OpRoleVarAttrName() { return "op_role_var"; }
   static const char *OpNamescopeAttrName() { return "op_namescope"; }
+  static const char *OpCreationCallstackAttrName() { return "op_callstack"; }
 
   void operator()(proto::OpProto *proto, OpAttrChecker *attr_checker);
 

paddle/fluid/framework/operator.cc

@@ -19,10 +19,12 @@ limitations under the License. */
 #include "paddle/fluid/framework/data_transform.h"
 #include "paddle/fluid/framework/executor.h"
 #include "paddle/fluid/framework/lod_tensor.h"
+#include "paddle/fluid/framework/op_proto_maker.h"
 #include "paddle/fluid/framework/operator.h"
 #include "paddle/fluid/framework/shape_inference.h"
 #include "paddle/fluid/framework/transfer_scope_cache.h"
 #include "paddle/fluid/framework/var_type.h"
+#include "paddle/fluid/platform/debug_support.h"
 #include "paddle/fluid/platform/profiler.h"
 
 DECLARE_bool(benchmark);
@@ -155,7 +157,18 @@ RuntimeContext::RuntimeContext(const VariableNameMap& innames,
   }
 }
 
+void OperatorBase::PreHook() {
+  auto attrName = OpProtoAndCheckerMaker::OpCreationCallstackAttrName();
+  if (HasAttr(attrName)) {
+    auto& callstack = Attr<std::vector<std::string>>(attrName);
+    platform::PythonDebugSupport::GetInstance()->SetInformation(callstack);
+  }
+}
+
 void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
+  VLOG(4) << "Call the prehook ... ";
+  PreHook();
+
   VLOG(4) << place << " " << DebugStringEx(&scope);
   if (platform::is_gpu_place(place)) {
 #ifndef PADDLE_WITH_CUDA
@@ -177,6 +190,13 @@ void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
     RunImpl(scope, place);
   }
   VLOG(3) << place << " " << DebugStringEx(&scope);
+
+  VLOG(4) << "Call the posthook ... ";
+  PostHook();
+}
+
+void OperatorBase::PostHook() {
+  // do nothing here
 }
 
 bool OperatorBase::HasInputs(const std::string& name) const {

paddle/fluid/framework/operator.h

@@ -160,6 +160,10 @@ class OperatorBase {
                                  const platform::Place& place,
                                  const RuntimeContext& ctx) const {}
 
+  // Add the hooks
+  virtual void PreHook();
+  virtual void PostHook();
+
  protected:
   std::string type_;
   // NOTE: in case of OpGrad, inputs_ contains:

paddle/fluid/platform/CMakeLists.txt

@@ -20,10 +20,12 @@ add_custom_command(TARGET profiler_py_proto POST_BUILD
         WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
 endif(NOT WIN32)
 
+cc_library(debug_support SRCS debug_support.cc)
+
 if(WITH_GPU)
-  nv_library(enforce SRCS enforce.cc)
+  nv_library(enforce SRCS enforce.cc DEPS debug_support)
 else()
-  cc_library(enforce SRCS enforce.cc)
+  cc_library(enforce SRCS enforce.cc DEPS debug_support)
 endif()
 cc_test(enforce_test SRCS enforce_test.cc DEPS stringpiece enforce)
 

paddle/fluid/platform/debug_support.cc

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#include <sstream>
+
+#include "paddle/fluid/platform/debug_support.h"
+
+namespace paddle {
+namespace platform {
+
+template <>
+std::string PythonDebugSupport::Format() const {
+  std::ostringstream sout;
+  sout << "\nPython Callstacks: \n";
+  for (auto& line : info) {
+    sout << line;
+  }
+  return sout.str();
+}
+
+}  // namespace platform
+}  // namespace paddle

paddle/fluid/platform/debug_support.h

+/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License. */
+
+#pragma once
+
+#include <exception>
+#include <iostream>
+#include <map>
+#include <mutex>
+#include <string>
+#include <thread>
+#include <vector>
+
+namespace paddle {
+namespace platform {
+
+template <typename T>
+class DebugSupport {
+ public:
+  // Returns the singleton of DebugSupport.
+  static DebugSupport* GetInstance() {
+    static std::unique_ptr<DebugSupport> debugSupport_(nullptr);
+    static std::once_flag init_flag_;
+
+    std::call_once(init_flag_,
+                   [&]() { debugSupport_.reset(new DebugSupport<T>()); });
+    return debugSupport_.get();
+  }
+
+  T GetInformation() const { return info; }
+
+  void SetInformation(const T& v) { info = v; }
+
+  std::string Format() const;
+
+ private:
+  T info;
+};
+
+using PythonDebugSupport = DebugSupport<std::vector<std::string>>;
+
+template <>
+std::string PythonDebugSupport::Format() const;
+
+}  // namespace platform
+}  // namespace paddle

paddle/fluid/platform/enforce.h

@@ -33,6 +33,7 @@ limitations under the License. */
 #include <string>
 
 #include "glog/logging.h"
+#include "paddle/fluid/platform/debug_support.h"
 #include "paddle/fluid/platform/macros.h"
 #include "paddle/fluid/platform/port.h"
 #include "paddle/fluid/string/printf.h"
@@ -68,6 +69,7 @@ struct EnforceNotMet : public std::exception {
       std::rethrow_exception(e);
     } catch (std::exception& e) {
       Init(e.what(), f, l);
+      err_str_ += platform::PythonDebugSupport::GetInstance()->Format();
     }
   }
 

paddle/fluid/pybind/const_value.cc

@@ -49,6 +49,9 @@ void BindConstValue(pybind11::module* m) {
   op_proto_and_checker_maker.def(
       "kOpNameScopeAttrName",
       framework::OpProtoAndCheckerMaker::OpNamescopeAttrName);
+  op_proto_and_checker_maker.def(
+      "kOpCreationCallstackAttrName",
+      framework::OpProtoAndCheckerMaker::OpCreationCallstackAttrName);
 }
 
 }  // namespace pybind

python/paddle/fluid/framework.py

@@ -19,6 +19,7 @@ from collections import defaultdict
 import contextlib
 import os
 import re
+import traceback
 import six
 
 import numpy as np
@@ -626,6 +627,10 @@ class Operator(object):
         if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0:
             del op_attrs[role_var_name]
 
+        callstack_var_name = op_maker.kOpCreationCallstackAttrName()
+        op_attrs[callstack_var_name] = list(
+            reversed(traceback.format_stack()))[1:]
+
         if len(self.desc.type()) != 0:
             return
         if type is None: