diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index 9bc5cb6a7e00deb0cbbe1995463c3bd623873bfa..cee6ec5ebd6dfe053ffed1a1cc811931e530fa30 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -16,6 +16,11 @@ limitations under the License. */ #include #include +#include +#include +#include +#include "gflags/gflags.h" +#include "glog/logging.h" #include "paddle/fluid/framework/data_transform.h" #include "paddle/fluid/framework/executor.h" #include "paddle/fluid/framework/lod_tensor.h" @@ -32,12 +37,6 @@ DEFINE_bool(check_nan_inf, false, "Checking whether operator produce NAN/INF or not. It will be " "extremely slow so please use this flag wisely."); -DEFINE_bool( - enable_debug, false, - "The enable_debug indicate whether to give more detail information when, " - "use the paddlepaddle. However it may deduce the performance since it has" - "to record the information during runtime."); - namespace paddle { namespace framework { @@ -163,50 +162,56 @@ RuntimeContext::RuntimeContext(const VariableNameMap& innames, } } -void OperatorBase::PreHook(const Scope& scope, const platform::Place& place) { - auto attrName = OpProtoAndCheckerMaker::OpCreationCallstackAttrName(); - if (HasAttr(attrName)) { - auto& callstack = Attr>(attrName); - platform::PythonDebugSupport::GetInstance()->SetInformation(callstack); - } -} - void OperatorBase::Run(const Scope& scope, const platform::Place& place) { - if (FLAGS_enable_debug) { - VLOG(4) << "Call the prehook ... "; - PreHook(scope, place); - } - - VLOG(4) << place << " " << DebugStringEx(&scope); - if (platform::is_gpu_place(place)) { + try { + VLOG(4) << place << " " << DebugStringEx(&scope); + if (platform::is_gpu_place(place)) { #ifndef PADDLE_WITH_CUDA - PADDLE_THROW("Cannot run operator on place %s", place); + PADDLE_THROW("Cannot run operator on place %s", place); #else - auto dev_id = boost::get(place).device; - platform::SetDeviceId(dev_id); + auto dev_id = boost::get(place).device; + platform::SetDeviceId(dev_id); #endif - } + } - // The profile has a process-wide mutex, results in serious performance issue - // in concurrency scenerio. Here use an `if` to fix this issue. - // Please not remove the `if`, ask @Superjomn if there are any concern. - if (platform::IsProfileEnabled()) { - platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); - platform::RecordEvent record_event(Type(), pool.Get(place)); - RunImpl(scope, place); - } else { - RunImpl(scope, place); - } - VLOG(3) << place << " " << DebugStringEx(&scope); + // The profile has a process-wide mutex, results in serious performance + // issue + // in concurrency scenerio. Here use an `if` to fix this issue. + // Please not remove the `if`, ask @Superjomn if there are any concern. + if (platform::IsProfileEnabled()) { + platform::DeviceContextPool& pool = + platform::DeviceContextPool::Instance(); + platform::RecordEvent record_event(Type(), pool.Get(place)); + RunImpl(scope, place); + } else { + RunImpl(scope, place); + } - if (FLAGS_enable_debug) { - VLOG(4) << "Call the posthook ... "; - PostHook(scope, place); - } -} + VLOG(3) << place << " " << DebugStringEx(&scope); + } catch (platform::EnforceNotMet exception) { + if (Attrs().count("sub_block") != 0) { + throw exception; + } -void OperatorBase::PostHook(const Scope& scope, const platform::Place& place) { - // do nothing here + auto& callstack = Attr>( + OpProtoAndCheckerMaker::OpCreationCallstackAttrName()); + + if (callstack.empty()) { + throw exception; + } + std::ostringstream sout; + sout << "Invoke operator " << Type() << " error.\n"; + sout << "Python Callstacks: \n"; + for (auto& line : callstack) { + sout << line; + } + sout << "C++ Callstacks: \n"; + sout << exception.err_str_; + exception.err_str_ = sout.str(); + throw exception; + } catch (...) { + std::rethrow_exception(std::current_exception()); + } } bool OperatorBase::HasInputs(const std::string& name) const { diff --git a/paddle/fluid/framework/operator.h b/paddle/fluid/framework/operator.h index bdaaec71347272612082a2d60bc6423bd916fefe..041187665af6ad0d75a7c55fe6ed451fe6f45b73 100644 --- a/paddle/fluid/framework/operator.h +++ b/paddle/fluid/framework/operator.h @@ -160,10 +160,6 @@ class OperatorBase { const platform::Place& place, const RuntimeContext& ctx) const {} - // Add the hooks - virtual void PreHook(const Scope& scope, const platform::Place& place); - virtual void PostHook(const Scope& scope, const platform::Place& place); - protected: std::string type_; // NOTE: in case of OpGrad, inputs_ contains: diff --git a/paddle/fluid/platform/CMakeLists.txt b/paddle/fluid/platform/CMakeLists.txt index 5889a72fc2bfd29675be20a4f05743272e81ea8a..1f51b5bab3068cc89bffa85de28a9438359659f3 100644 --- a/paddle/fluid/platform/CMakeLists.txt +++ b/paddle/fluid/platform/CMakeLists.txt @@ -20,12 +20,10 @@ add_custom_command(TARGET profiler_py_proto POST_BUILD WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) endif(NOT WIN32) -cc_library(debug_support SRCS debug_support.cc) - if(WITH_GPU) - nv_library(enforce SRCS enforce.cc DEPS debug_support) + nv_library(enforce SRCS enforce.cc) else() - cc_library(enforce SRCS enforce.cc DEPS debug_support) + cc_library(enforce SRCS enforce.cc) endif() cc_test(enforce_test SRCS enforce_test.cc DEPS stringpiece enforce) diff --git a/paddle/fluid/platform/debug_support.cc b/paddle/fluid/platform/debug_support.cc deleted file mode 100644 index 98dcbc263758720b09b057d131ab1cf1caff4073..0000000000000000000000000000000000000000 --- a/paddle/fluid/platform/debug_support.cc +++ /dev/null @@ -1,44 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#include - -#include "paddle/fluid/platform/debug_support.h" - -namespace paddle { -namespace platform { - -template <> -std::string PythonDebugSupport::Format() const { - std::ostringstream sout; - sout << "\nPython Callstacks: \n"; - if (!info.empty()) { - for (auto &line : info) { - sout << line; - } - } else { -#ifdef _WIN32 - sout << "please set FLAGS_enable_debug=True to get more details regard to " - "this failure.\n"; -#else // _WIN32 - sout << "please export FLAGS_enable_debug=True to get more details regard " - "to " - "this failure.\n"; -#endif // _WIN32 - } - return sout.str(); -} - -} // namespace platform -} // namespace paddle diff --git a/paddle/fluid/platform/debug_support.h b/paddle/fluid/platform/debug_support.h deleted file mode 100644 index 2c8ee6ed1f9a460eba98f24634a4e4c42c72894b..0000000000000000000000000000000000000000 --- a/paddle/fluid/platform/debug_support.h +++ /dev/null @@ -1,57 +0,0 @@ -/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. */ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include - -namespace paddle { -namespace platform { - -template -class DebugSupport { - public: - // Returns the singleton of DebugSupport. - static DebugSupport* GetInstance() { - static thread_local std::unique_ptr debugSupport_(nullptr); - static thread_local std::once_flag init_flag_; - - std::call_once(init_flag_, - [&]() { debugSupport_.reset(new DebugSupport()); }); - return debugSupport_.get(); - } - - T GetInformation() const { return info; } - - void SetInformation(const T& v) { info = v; } - - std::string Format() const; - - private: - T info; -}; - -using PythonDebugSupport = DebugSupport>; - -template <> -std::string PythonDebugSupport::Format() const; - -} // namespace platform -} // namespace paddle diff --git a/paddle/fluid/platform/enforce.h b/paddle/fluid/platform/enforce.h index b4edd51568db58346af48e459036537d43fa7099..15413785bab3c0fd77244141e8f1840ca0cc1356 100644 --- a/paddle/fluid/platform/enforce.h +++ b/paddle/fluid/platform/enforce.h @@ -33,7 +33,6 @@ limitations under the License. */ #include #include "glog/logging.h" -#include "paddle/fluid/platform/debug_support.h" #include "paddle/fluid/platform/macros.h" #include "paddle/fluid/platform/port.h" #include "paddle/fluid/string/printf.h" @@ -69,7 +68,6 @@ struct EnforceNotMet : public std::exception { std::rethrow_exception(e); } catch (std::exception& e) { Init(e.what(), f, l); - err_str_ += platform::PythonDebugSupport::GetInstance()->Format(); } } diff --git a/python/paddle/fluid/framework.py b/python/paddle/fluid/framework.py index b83b8570ca2190d74b953abcad67b67821cad6ae..e9a9265931f432ebf7b2351cd6c74ece8b0f863a 100644 --- a/python/paddle/fluid/framework.py +++ b/python/paddle/fluid/framework.py @@ -621,21 +621,16 @@ class Operator(object): if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0: del op_attrs[role_var_name] - if 'FLAGS_enable_debug' in os.environ and os.environ[ - 'FLAGS_enable_debug']: - callstack_var_name = op_maker.kOpCreationCallstackAttrName() - op_attrs[callstack_var_name] = list( - reversed(traceback.format_stack()))[1:] - op_attrs[callstack_var_name].insert( - 0, - 'Invoke operator ' + ('' - if type is None else type) + ' error.\n') - if len(self.desc.type()) != 0: return if type is None: raise ValueError( "`type` to initilized an Operator can not be None.") + else: + callstack_var_name = op_maker.kOpCreationCallstackAttrName() + op_attrs[callstack_var_name] = list( + reversed(traceback.format_stack()))[1:] + self.desc.set_type(type) proto = OpProtoHolder.instance().get_op_proto(type)