From ac813bbaf4c5ed6a94f1776eb9a369edd3138335 Mon Sep 17 00:00:00 2001 From: Zeng Jinle <32832641+sneaxiy@users.noreply.github.com> Date: Thu, 24 Oct 2019 16:08:26 +0800 Subject: [PATCH] Add more error debug message to Operator::Run (#20793) * add more err msg, test=develop * add more unittests, test=develop --- paddle/fluid/framework/CMakeLists.txt | 1 + paddle/fluid/framework/op_call_stack.cc | 15 ++-- paddle/fluid/framework/operator.cc | 7 ++ .../framework/operator_exception_test.cc | 85 +++++++++++++++++++ 4 files changed, 103 insertions(+), 5 deletions(-) create mode 100644 paddle/fluid/framework/operator_exception_test.cc diff --git a/paddle/fluid/framework/CMakeLists.txt b/paddle/fluid/framework/CMakeLists.txt index 49be7f0e23..e0e5e18890 100644 --- a/paddle/fluid/framework/CMakeLists.txt +++ b/paddle/fluid/framework/CMakeLists.txt @@ -127,6 +127,7 @@ cc_library(operator SRCS operator.cc DEPS op_info device_context tensor scope gl shape_inference data_transform lod_tensor profiler transfer_scope_cache op_kernel_type op_call_stack) cc_test(operator_test SRCS operator_test.cc DEPS operator op_registry device_context) +cc_test(operator_exception_test SRCS operator_exception_test.cc DEPS operator op_registry device_context) cc_library(version SRCS version.cc) cc_test(version_test SRCS version_test.cc DEPS version) diff --git a/paddle/fluid/framework/op_call_stack.cc b/paddle/fluid/framework/op_call_stack.cc index 91da320261..db259bf038 100644 --- a/paddle/fluid/framework/op_call_stack.cc +++ b/paddle/fluid/framework/op_call_stack.cc @@ -26,17 +26,22 @@ void InsertCallStackInfo(const std::string &type, const AttributeMap &attrs, if (attrs.count("sub_block") != 0) { return; } - auto &callstack = boost::get>( - attrs.at(OpProtoAndCheckerMaker::OpCreationCallstackAttrName())); + + const std::vector *callstack = nullptr; + auto iter = attrs.find(OpProtoAndCheckerMaker::OpCreationCallstackAttrName()); + if (iter != attrs.end()) { + callstack = &boost::get>(iter->second); + if (callstack->empty()) callstack = nullptr; + } std::ostringstream sout; std::ostringstream sout_py_trace; // Step 1. Construct python call stack string - if (!callstack.empty()) { + if (callstack) { sout_py_trace << "\n------------------------------------------\n"; sout_py_trace << "Python Call Stacks (More useful to users):"; sout_py_trace << "\n------------------------------------------\n"; - for (auto &line : callstack) { + for (auto &line : *callstack) { sout_py_trace << line; } } @@ -55,7 +60,7 @@ void InsertCallStackInfo(const std::string &type, const AttributeMap &attrs, sout << "C++ Call Stacks (More useful to developers):"; sout << "\n--------------------------------------------\n"; sout << exception->err_str_; - if (!callstack.empty()) { + if (callstack) { sout << " [operator < " << type << " > error]"; } exception->err_str_ = sout.str(); diff --git a/paddle/fluid/framework/operator.cc b/paddle/fluid/framework/operator.cc index a608d1bd3a..d0a12fcfd1 100644 --- a/paddle/fluid/framework/operator.cc +++ b/paddle/fluid/framework/operator.cc @@ -183,7 +183,14 @@ void OperatorBase::Run(const Scope& scope, const platform::Place& place) { } catch (platform::EnforceNotMet& exception) { framework::InsertCallStackInfo(Type(), Attrs(), &exception); throw std::move(exception); + } catch (platform::EOFException&) { + std::rethrow_exception(std::current_exception()); + } catch (std::exception& ex) { + LOG(WARNING) << Type() << " raises an exception " + << platform::demangle(typeid(ex).name()) << ", " << ex.what(); + std::rethrow_exception(std::current_exception()); } catch (...) { + LOG(WARNING) << Type() << " raises an unknown exception"; std::rethrow_exception(std::current_exception()); } } diff --git a/paddle/fluid/framework/operator_exception_test.cc b/paddle/fluid/framework/operator_exception_test.cc new file mode 100644 index 0000000000..7b513996fb --- /dev/null +++ b/paddle/fluid/framework/operator_exception_test.cc @@ -0,0 +1,85 @@ +// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/operator.h" +#include +#include +#include +#include +#include "gtest/gtest.h" +#include "paddle/fluid/framework/scope.h" +#include "paddle/fluid/platform/place.h" + +namespace paddle { +namespace framework { + +class ExceptionThrownOperator : public OperatorBase { + public: + using OperatorBase::OperatorBase; + + template + void SetException(T &&obj) { + exception_ = std::make_exception_ptr(std::forward(obj)); + } + + protected: + void RunImpl(const Scope &, const platform::Place &) const override { + if (exception_) { + std::rethrow_exception(exception_); + } + } + + private: + std::exception_ptr exception_{nullptr}; +}; + +class StubException : public std::exception { + public: + const char *what() const noexcept override { return ""; } +}; + +template +bool ExceptionTestMain(T &&obj, bool set_exception) { + ExceptionThrownOperator op("", {}, {}, {}); + if (set_exception) { + op.SetException(std::forward(obj)); + } + Scope scope; + try { + op.Run(scope, platform::CPUPlace()); + return false; + } catch (T &) { + return true; + } +} + +TEST(test_operator_exception, test_operator_exception) { + platform::EnforceNotMet enforce_not_met("", __FILE__, __LINE__); + ASSERT_TRUE(ExceptionTestMain(enforce_not_met, true)); + ASSERT_FALSE(ExceptionTestMain(enforce_not_met, false)); + + platform::EOFException eof("", __FILE__, __LINE__); + ASSERT_TRUE(ExceptionTestMain(eof, true)); + ASSERT_FALSE(ExceptionTestMain(eof, false)); + + StubException stub; + ASSERT_TRUE(ExceptionTestMain(stub, true)); + ASSERT_FALSE(ExceptionTestMain(stub, false)); + + ASSERT_TRUE(ExceptionTestMain(1, true)); + ASSERT_FALSE(ExceptionTestMain(1, false)); +} + +} // namespace framework +} // namespace paddle -- GitLab