提交 eea75a1d 编写于 作者: P peizhilin

fix issue when type is invalid

test=develop
上级 9adb158e
...@@ -16,6 +16,11 @@ limitations under the License. */ ...@@ -16,6 +16,11 @@ limitations under the License. */
#include <glog/logging.h> #include <glog/logging.h>
#include <algorithm> #include <algorithm>
#include <sstream>
#include <string>
#include <vector>
#include "gflags/gflags.h"
#include "glog/logging.h"
#include "paddle/fluid/framework/data_transform.h" #include "paddle/fluid/framework/data_transform.h"
#include "paddle/fluid/framework/executor.h" #include "paddle/fluid/framework/executor.h"
#include "paddle/fluid/framework/lod_tensor.h" #include "paddle/fluid/framework/lod_tensor.h"
...@@ -32,12 +37,6 @@ DEFINE_bool(check_nan_inf, false, ...@@ -32,12 +37,6 @@ DEFINE_bool(check_nan_inf, false,
"Checking whether operator produce NAN/INF or not. It will be " "Checking whether operator produce NAN/INF or not. It will be "
"extremely slow so please use this flag wisely."); "extremely slow so please use this flag wisely.");
DEFINE_bool(
enable_debug, false,
"The enable_debug indicate whether to give more detail information when, "
"use the paddlepaddle. However it may deduce the performance since it has"
"to record the information during runtime.");
namespace paddle { namespace paddle {
namespace framework { namespace framework {
...@@ -163,20 +162,8 @@ RuntimeContext::RuntimeContext(const VariableNameMap& innames, ...@@ -163,20 +162,8 @@ RuntimeContext::RuntimeContext(const VariableNameMap& innames,
} }
} }
void OperatorBase::PreHook(const Scope& scope, const platform::Place& place) {
auto attrName = OpProtoAndCheckerMaker::OpCreationCallstackAttrName();
if (HasAttr(attrName)) {
auto& callstack = Attr<std::vector<std::string>>(attrName);
platform::PythonDebugSupport::GetInstance()->SetInformation(callstack);
}
}
void OperatorBase::Run(const Scope& scope, const platform::Place& place) { void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
if (FLAGS_enable_debug) { try {
VLOG(4) << "Call the prehook ... ";
PreHook(scope, place);
}
VLOG(4) << place << " " << DebugStringEx(&scope); VLOG(4) << place << " " << DebugStringEx(&scope);
if (platform::is_gpu_place(place)) { if (platform::is_gpu_place(place)) {
#ifndef PADDLE_WITH_CUDA #ifndef PADDLE_WITH_CUDA
...@@ -187,26 +174,44 @@ void OperatorBase::Run(const Scope& scope, const platform::Place& place) { ...@@ -187,26 +174,44 @@ void OperatorBase::Run(const Scope& scope, const platform::Place& place) {
#endif #endif
} }
// The profile has a process-wide mutex, results in serious performance issue // The profile has a process-wide mutex, results in serious performance
// issue
// in concurrency scenerio. Here use an `if` to fix this issue. // in concurrency scenerio. Here use an `if` to fix this issue.
// Please not remove the `if`, ask @Superjomn if there are any concern. // Please not remove the `if`, ask @Superjomn if there are any concern.
if (platform::IsProfileEnabled()) { if (platform::IsProfileEnabled()) {
platform::DeviceContextPool& pool = platform::DeviceContextPool::Instance(); platform::DeviceContextPool& pool =
platform::DeviceContextPool::Instance();
platform::RecordEvent record_event(Type(), pool.Get(place)); platform::RecordEvent record_event(Type(), pool.Get(place));
RunImpl(scope, place); RunImpl(scope, place);
} else { } else {
RunImpl(scope, place); RunImpl(scope, place);
} }
VLOG(3) << place << " " << DebugStringEx(&scope);
if (FLAGS_enable_debug) { VLOG(3) << place << " " << DebugStringEx(&scope);
VLOG(4) << "Call the posthook ... "; } catch (platform::EnforceNotMet exception) {
PostHook(scope, place); if (Attrs().count("sub_block") != 0) {
throw exception;
} }
}
void OperatorBase::PostHook(const Scope& scope, const platform::Place& place) { auto& callstack = Attr<std::vector<std::string>>(
// do nothing here OpProtoAndCheckerMaker::OpCreationCallstackAttrName());
if (callstack.empty()) {
throw exception;
}
std::ostringstream sout;
sout << "Invoke operator " << Type() << " error.\n";
sout << "Python Callstacks: \n";
for (auto& line : callstack) {
sout << line;
}
sout << "C++ Callstacks: \n";
sout << exception.err_str_;
exception.err_str_ = sout.str();
throw exception;
} catch (...) {
std::rethrow_exception(std::current_exception());
}
} }
bool OperatorBase::HasInputs(const std::string& name) const { bool OperatorBase::HasInputs(const std::string& name) const {
......
...@@ -160,10 +160,6 @@ class OperatorBase { ...@@ -160,10 +160,6 @@ class OperatorBase {
const platform::Place& place, const platform::Place& place,
const RuntimeContext& ctx) const {} const RuntimeContext& ctx) const {}
// Add the hooks
virtual void PreHook(const Scope& scope, const platform::Place& place);
virtual void PostHook(const Scope& scope, const platform::Place& place);
protected: protected:
std::string type_; std::string type_;
// NOTE: in case of OpGrad, inputs_ contains: // NOTE: in case of OpGrad, inputs_ contains:
......
...@@ -20,12 +20,10 @@ add_custom_command(TARGET profiler_py_proto POST_BUILD ...@@ -20,12 +20,10 @@ add_custom_command(TARGET profiler_py_proto POST_BUILD
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}) WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR})
endif(NOT WIN32) endif(NOT WIN32)
cc_library(debug_support SRCS debug_support.cc)
if(WITH_GPU) if(WITH_GPU)
nv_library(enforce SRCS enforce.cc DEPS debug_support) nv_library(enforce SRCS enforce.cc)
else() else()
cc_library(enforce SRCS enforce.cc DEPS debug_support) cc_library(enforce SRCS enforce.cc)
endif() endif()
cc_test(enforce_test SRCS enforce_test.cc DEPS stringpiece enforce) cc_test(enforce_test SRCS enforce_test.cc DEPS stringpiece enforce)
......
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include <sstream>
#include "paddle/fluid/platform/debug_support.h"
namespace paddle {
namespace platform {
template <>
std::string PythonDebugSupport::Format() const {
std::ostringstream sout;
sout << "\nPython Callstacks: \n";
if (!info.empty()) {
for (auto &line : info) {
sout << line;
}
} else {
#ifdef _WIN32
sout << "please set FLAGS_enable_debug=True to get more details regard to "
"this failure.\n";
#else // _WIN32
sout << "please export FLAGS_enable_debug=True to get more details regard "
"to "
"this failure.\n";
#endif // _WIN32
}
return sout.str();
}
} // namespace platform
} // namespace paddle
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <exception>
#include <iostream>
#include <map>
#include <mutex>
#include <string>
#include <thread>
#include <vector>
namespace paddle {
namespace platform {
template <typename T>
class DebugSupport {
public:
// Returns the singleton of DebugSupport.
static DebugSupport* GetInstance() {
static thread_local std::unique_ptr<DebugSupport> debugSupport_(nullptr);
static thread_local std::once_flag init_flag_;
std::call_once(init_flag_,
[&]() { debugSupport_.reset(new DebugSupport<T>()); });
return debugSupport_.get();
}
T GetInformation() const { return info; }
void SetInformation(const T& v) { info = v; }
std::string Format() const;
private:
T info;
};
using PythonDebugSupport = DebugSupport<std::vector<std::string>>;
template <>
std::string PythonDebugSupport::Format() const;
} // namespace platform
} // namespace paddle
...@@ -33,7 +33,6 @@ limitations under the License. */ ...@@ -33,7 +33,6 @@ limitations under the License. */
#include <string> #include <string>
#include "glog/logging.h" #include "glog/logging.h"
#include "paddle/fluid/platform/debug_support.h"
#include "paddle/fluid/platform/macros.h" #include "paddle/fluid/platform/macros.h"
#include "paddle/fluid/platform/port.h" #include "paddle/fluid/platform/port.h"
#include "paddle/fluid/string/printf.h" #include "paddle/fluid/string/printf.h"
...@@ -69,7 +68,6 @@ struct EnforceNotMet : public std::exception { ...@@ -69,7 +68,6 @@ struct EnforceNotMet : public std::exception {
std::rethrow_exception(e); std::rethrow_exception(e);
} catch (std::exception& e) { } catch (std::exception& e) {
Init(e.what(), f, l); Init(e.what(), f, l);
err_str_ += platform::PythonDebugSupport::GetInstance()->Format();
} }
} }
......
...@@ -621,21 +621,16 @@ class Operator(object): ...@@ -621,21 +621,16 @@ class Operator(object):
if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0: if role_var_name in op_attrs and len(op_attrs[role_var_name]) == 0:
del op_attrs[role_var_name] del op_attrs[role_var_name]
if 'FLAGS_enable_debug' in os.environ and os.environ[
'FLAGS_enable_debug']:
callstack_var_name = op_maker.kOpCreationCallstackAttrName()
op_attrs[callstack_var_name] = list(
reversed(traceback.format_stack()))[1:]
op_attrs[callstack_var_name].insert(
0,
'Invoke operator ' + (''
if type is None else type) + ' error.\n')
if len(self.desc.type()) != 0: if len(self.desc.type()) != 0:
return return
if type is None: if type is None:
raise ValueError( raise ValueError(
"`type` to initilized an Operator can not be None.") "`type` to initilized an Operator can not be None.")
else:
callstack_var_name = op_maker.kOpCreationCallstackAttrName()
op_attrs[callstack_var_name] = list(
reversed(traceback.format_stack()))[1:]
self.desc.set_type(type) self.desc.set_type(type)
proto = OpProtoHolder.instance().get_op_proto(type) proto = OpProtoHolder.instance().get_op_proto(type)
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册