From a6f5ceee74739189623f6b0edfbb7afac55eb04b Mon Sep 17 00:00:00 2001 From: peizhilin Date: Tue, 8 Jan 2019 10:24:47 +0800 Subject: [PATCH] add the python callstack for debug support test=develop --- paddle/fluid/framework/op_proto_maker.cc | 4 ++ paddle/fluid/framework/op_proto_maker.h | 1 + paddle/fluid/framework/operator.cc | 20 +++++++++ paddle/fluid/framework/operator.h | 4 ++ paddle/fluid/platform/CMakeLists.txt | 6 ++- paddle/fluid/platform/debug_support.cc | 33 ++++++++++++++ paddle/fluid/platform/debug_support.h | 57 ++++++++++++++++++++++++ paddle/fluid/platform/enforce.h | 2 + paddle/fluid/pybind/const_value.cc | 3 ++ python/paddle/fluid/framework.py | 5 +++ 10 files changed, 133 insertions(+), 2 deletions(-) create mode 100644 paddle/fluid/platform/debug_support.cc create mode 100644 paddle/fluid/platform/debug_support.h diff --git a/paddle/fluid/framework/op_proto_maker.cc b/paddle/fluid/framework/op_proto_maker.cc index ca31303f77..2311614c33 100644 --- a/paddle/fluid/framework/op_proto_maker.cc +++ b/paddle/fluid/framework/op_proto_maker.cc @@ -82,6 +82,10 @@ void OpProtoAndCheckerMaker::operator()(proto::OpProto* proto, AddAttr(OpNamescopeAttrName(), "Operator name with namesope.") .SetDefault(""); + AddAttr>(OpCreationCallstackAttrName(), + "Callstack for Op Creatation.") + .SetDefault({}); + Validate(); } diff --git a/paddle/fluid/framework/op_proto_maker.h b/paddle/fluid/framework/op_proto_maker.h index 4c59c73d87..0a0f8f4655 100644 --- a/paddle/fluid/framework/op_proto_maker.h +++ b/paddle/fluid/framework/op_proto_maker.h @@ -47,6 +47,7 @@ class OpProtoAndCheckerMaker { static const char *OpRoleAttrName() { return "op_role"; } static const char *OpRoleVarAttrName() { return "op_role_var"; } static const char *OpNamescopeAttrName() { return "op_namescope"; } + static const char *OpCreationCallstackAttrName() { return "op_callstack"; } void operator()(proto::OpProto *proto, OpAttrChecker *attr_checker); diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index f10da22aec..4066907fff 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -19,10 +19,12 @@ limitations under the License. */ #include "paddle/fluid/framework/data_transform.h" #include "paddle/fluid/framework/executor.h" #include "paddle/fluid/framework/lod_tensor.h" +#include "paddle/fluid/framework/op_proto_maker.h" #include "paddle/fluid/framework/operator.h" #include "paddle/fluid/framework/shape_inference.h" #include "paddle/fluid/framework/transfer_scope_cache.h" #include "paddle/fluid/framework/var_type.h" +#include "paddle/fluid/platform/debug_support.h" #include "paddle/fluid/platform/profiler.h" DECLARE_bool(benchmark); @@ -155,7 +157,18 @@ RuntimeContext::RuntimeContext(const VariableNameMap& innames, } } +void OperatorBase::PreHook() { + auto attrName = OpProtoAndCheckerMaker::OpCreationCallstackAttrName(); + if (HasAttr(attrName)) { + auto& callstack = Attr>(attrName); + platform::PythonDebugSupport::GetInstance()->SetInformation(callstack); + } +} + void OperatorBase::Run(const Scope& scope, const platform::Place& place) { + VLOG(4) << "Call the prehook ... "; + PreHook(); + VLOG(4) << place << " " << DebugStringEx(&scope); if (platform::is_gpu_place(place)) { #ifndef PADDLE_WITH_CUDA @@ -177,6 +190,13 @@ void OperatorBase::Run(const Scope& scope, const platform::Place& place) { RunImpl(scope, place); } VLOG(3) << place << " " << DebugStringEx(&scope); + + VLOG(4) << "Call the posthook ... "; + PostHook(); +} + +void OperatorBase::PostHook() { + // do nothing here } bool OperatorBase::HasInputs(const std::string& name) const { diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h index 4d29564aee..4e96ca9f5f 100644 --- a/paddle/fluid/framework/operator.h +++ b/paddle/fluid/framework/operator.h @@ -160,6 +160,10 @@ class OperatorBase { const platform::Place& place, const RuntimeContext& ctx) const {} + // Add the hooks + virtual void PreHook(); + virtual void PostHook(); + protected: std::string type_; // NOTE: in case of OpGrad, inputs_ contains: diff --git a/paddle/fluid/platform/CMakeLists.txt b/paddle/fluid/platform/CMakeLists.txt index 1f51b5bab3..5889a72fc2 100644 --- a/paddle/fluid/platform/CMakeLists.txt +++ b/paddle/fluid/platform/CMakeLists.txt @@ -20,10 +20,12 @@ add_custom_command(TARGET profiler_py_proto POST_BUILD WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif(NOT WIN32) +cc_library(debug_support SRCS debug_support.cc) + if(WITH_GPU) - nv_library(enforce SRCS enforce.cc) + nv_library(enforce SRCS enforce.cc DEPS debug_support) else() - cc_library(enforce SRCS enforce.cc) + cc_library(enforce SRCS enforce.cc DEPS debug_support) endif() cc_test(enforce_test SRCS enforce_test.cc DEPS stringpiece enforce) diff --git a/paddle/fluid/platform/debug_support.cc b/paddle/fluid/platform/debug_support.cc new file mode 100644 index 0000000000..77ff721020 --- /dev/null +++ b/paddle/fluid/platform/debug_support.cc @@ -0,0 +1,33 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#include + +#include "paddle/fluid/platform/debug_support.h" + +namespace paddle { +namespace platform { + +template <> +std::string PythonDebugSupport::Format() const { + std::ostringstream sout; + sout << "\nPython Callstacks: \n"; + for (auto& line : info) { + sout << line; + } + return sout.str(); +} + +} // namespace platform +} // namespace paddle diff --git a/paddle/fluid/platform/debug_support.h b/paddle/fluid/platform/debug_support.h new file mode 100644 index 0000000000..9d1d54cef7 --- /dev/null +++ b/paddle/fluid/platform/debug_support.h @@ -0,0 +1,57 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace paddle { +namespace platform { + +template +class DebugSupport { + public: + // Returns the singleton of DebugSupport. + static DebugSupport* GetInstance() { + static std::unique_ptr debugSupport_(nullptr); + static std::once_flag init_flag_; + + std::call_once(init_flag_, + [&]() { debugSupport_.reset(new DebugSupport()); }); + return debugSupport_.get(); + } + + T GetInformation() const { return info; } + + void SetInformation(const T& v) { info = v; } + + std::string Format() const; + + private: + T info; +}; + +using PythonDebugSupport = DebugSupport>; + +template <> +std::string PythonDebugSupport::Format() const; + +} // namespace platform +} // namespace paddle diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h index 0668053950..71c8cc1e31 100644 --- a/paddle/fluid/platform/enforce.h +++ b/paddle/fluid/platform/enforce.h @@ -33,6 +33,7 @@ limitations under the License. */ #include #include "glog/logging.h" +#include "paddle/fluid/platform/debug_support.h" #include "paddle/fluid/platform/macros.h" #include "paddle/fluid/platform/port.h" #include "paddle/fluid/string/printf.h" @@ -68,6 +69,7 @@ struct EnforceNotMet : public std::exception { std::rethrow_exception(e); } catch (std::exception& e) { Init(e.what(), f, l); + err_str_ += platform::PythonDebugSupport::GetInstance()->Format(); } } diff --git a/paddle/fluid/pybind/const_value.cc b/paddle/fluid/pybind/const_value.cc index 06d8b65fb1..f8ded9f94e 100644 --- a/paddle/fluid/pybind/const_value.cc +++ b/paddle/fluid/pybind/const_value.cc @@ -49,6 +49,9 @@ void BindConstValue(pybind11::module* m) { op_proto_and_checker_maker.def( "kOpNameScopeAttrName", framework::OpProtoAndCheckerMaker::OpNamescopeAttrName); + op_proto_and_checker_maker.def( + "kOpCreationCallstackAttrName", + framework::OpProtoAndCheckerMaker::OpCreationCallstackAttrName); } } // namespace pybind diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index 70767c962f..f54016d504 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -19,6 +19,7 @@ from collections import defaultdict import contextlib import os import re +import traceback import six import numpy as np @@ -626,6 +627,10 @@ class Operator(object): if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0: del op_attrs[role_var_name] + callstack_var_name = op_maker.kOpCreationCallstackAttrName() + op_attrs[callstack_var_name] = list( + reversed(traceback.format_stack()))[1:] + if len(self.desc.type()) != 0: return if type is None: -- GitLab