未验证 提交 ca359fec 编写于 作者: C Chen Weihang 提交者: GitHub

[CompileOpt] Refine enforce code and remove boost/variant include (#41093)

* refine enforce code

* refine enforce code

* fix compile failed

* fix infrt failed
上级 68babef1
......@@ -21,8 +21,8 @@ limitations under the License. */
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "paddle/fluid/imperative/type_defs.h"
#include "paddle/fluid/platform/variant.h"
#include "paddle/phi/core/compat/type_defs.h"
#include "paddle/utils/small_vector.h"
namespace paddle {
......@@ -39,6 +39,12 @@ class InferNoNeedBufferVarsFN;
using VariableNameMap = std::map<std::string, std::vector<std::string>>;
using VariableValueMap = std::map<std::string, std::vector<Variable*>>;
using Attribute = boost::variant<
boost::blank, int, float, std::string, std::vector<int>, std::vector<float>,
std::vector<std::string>, bool, std::vector<bool>, BlockDesc*, int64_t,
std::vector<BlockDesc*>, std::vector<int64_t>, std::vector<double>>;
using AttributeMap = std::unordered_map<std::string, Attribute>;
#ifdef PADDLE_WITH_ASCEND_CL
using NPUAttribute =
boost::variant<boost::blank, int, float, std::string, std::vector<int>,
......
......@@ -16,8 +16,8 @@
#include <glog/logging.h>
#include <memory>
#include <unordered_set>
#include "paddle/fluid/framework/type_defs.h"
#include "paddle/phi/common/layout.h"
#include "paddle/phi/core/compat/type_defs.h"
namespace paddle {
namespace imperative {
......
......@@ -23,10 +23,10 @@
#include "paddle/fluid/imperative/execution_context.h"
#include "paddle/fluid/imperative/layer.h"
#include "paddle/fluid/imperative/tracer.h"
#include "paddle/fluid/imperative/type_defs.h"
#include "paddle/fluid/imperative/var_helper.h"
#include "paddle/fluid/memory/memcpy.h"
#include "paddle/fluid/platform/device_context.h"
#include "paddle/phi/core/compat/type_defs.h"
namespace paddle {
namespace imperative {
......
......@@ -13,4 +13,58 @@ See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include "paddle/phi/core/compat/type_defs.h"
#include <map>
#include <memory>
#include <string>
#include <vector>
namespace egr {
class EagerVariable;
}
namespace paddle {
namespace imperative {
class VariableWrapper;
class SavedVariableWrapperList;
class VarBase;
class OpBase;
class GradOpNode;
class Tracer;
using WeakNameVarBaseMap =
std::map<std::string, std::vector<std::weak_ptr<VarBase>>>;
namespace details {
template <typename T>
struct NameVarMapTrait {};
template <>
struct NameVarMapTrait<VarBase> {
using Type = std::map<std::string, std::vector<std::shared_ptr<VarBase>>>;
};
template <>
struct NameVarMapTrait<VariableWrapper> {
using Type = std::map<std::string, SavedVariableWrapperList>;
};
template <>
struct NameVarMapTrait<egr::EagerVariable> {
using Type =
std::map<std::string, std::vector<std::shared_ptr<egr::EagerVariable>>>;
};
} // namespace details
template <typename T>
using NameVarMap = typename details::NameVarMapTrait<T>::Type;
using NameVarBaseMap = NameVarMap<VarBase>;
using NameVariableWrapperMap = NameVarMap<VariableWrapper>;
using NameTensorMap = NameVarMap<egr::EagerVariable>;
using VariableWrapperList = std::vector<std::shared_ptr<VariableWrapper>>;
} // namespace imperative
} // namespace paddle
......@@ -33,8 +33,8 @@ namespace allocation {
// Exception when `Alloc`/`AllocShared` failed
struct BadAlloc : public std::exception {
inline explicit BadAlloc(std::string err_msg, const char* file, int line)
: err_str_(platform::GetTraceBackString(std::move(err_msg), file, line)) {
}
: err_str_(platform::GetCompleteTraceBackString(std::move(err_msg), file,
line)) {}
const char* what() const noexcept override { return err_str_.c_str(); }
......
......@@ -59,7 +59,6 @@ limitations under the License. */
#include "paddle/fluid/pybind/tensor_py.h"
#include "paddle/fluid/pybind/uva_utils.h"
#include "paddle/phi/core/compat/arg_map_context.h"
#include "paddle/phi/core/compat/type_defs.h"
#include "paddle/phi/core/type_defs.h"
namespace paddle {
......
......@@ -14,6 +14,8 @@
#pragma once
#include <unordered_map>
#include "paddle/phi/core/dense_tensor.h"
namespace infrt {
......
......@@ -14,6 +14,7 @@
#include "paddle/phi/backends/device_base.h"
#include "gflags/gflags.h"
#include "glog/logging.h"
#include "paddle/phi/core/enforce.h"
DECLARE_double(fraction_of_gpu_memory_to_use);
......
......@@ -15,6 +15,8 @@
#pragma once
#ifdef PADDLE_WITH_CUSTOM_DEVICE
#include <unordered_map>
#include "paddle/phi/backends/device_base.h"
#include "paddle/phi/backends/device_ext.h"
#include "paddle/phi/backends/event.h"
......
......@@ -14,6 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/backends/gpu/gpu_context.h"
#include <algorithm>
#include <array>
#include <functional>
......@@ -21,8 +22,9 @@ limitations under the License. */
#include <memory>
#include <mutex>
#include "paddle/phi/api/ext/exception.h"
#include "glog/logging.h"
#include "paddle/phi/api/ext/exception.h"
#include "paddle/phi/backends/gpu/gpu_decls.h"
#include "paddle/phi/backends/gpu/gpu_info.h"
#include "paddle/phi/backends/gpu/gpu_resources.h"
......
......@@ -6,7 +6,7 @@ set(phi_enforce_deps errors flags)
if(WITH_GPU)
set(phi_enforce_deps ${phi_enforce_deps} external_error_proto)
endif()
cc_library(phi_enforce INTERFACE SRCS enforce.cc DEPS ${phi_enforce_deps})
cc_library(phi_enforce SRCS enforce.cc DEPS ${phi_enforce_deps})
cc_library(kernel_factory SRCS kernel_factory.cc DEPS phi_enforce fluid_convert_utils)
cc_library(kernel_context SRCS kernel_context.cc DEPS phi_enforce phi_context)
......
......@@ -17,6 +17,8 @@ limitations under the License. */
#include <string>
#include <unordered_set>
#include "glog/logging.h"
#include "paddle/phi/core/compat/arg_map_context.h"
#include "paddle/phi/core/enforce.h"
#include "paddle/phi/core/infermeta_utils.h"
......
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include <boost/variant.hpp>
namespace egr {
class EagerVariable;
}
namespace paddle {
namespace framework {
// The order should be as same as framework.proto
// NOTE(xiongkun): we extract from framework/typedef.h to ensure we can transfer
// enforce.h
class BlockDesc;
using Attribute = boost::variant<boost::blank,
int,
float,
std::string,
std::vector<int>,
std::vector<float>,
std::vector<std::string>,
bool,
std::vector<bool>,
BlockDesc*,
int64_t,
std::vector<BlockDesc*>,
std::vector<int64_t>,
std::vector<double>>;
using AttributeMap = std::unordered_map<std::string, Attribute>;
} // namespace framework
namespace imperative {
class VariableWrapper;
class SavedVariableWrapperList;
class VarBase;
class OpBase;
class GradOpNode;
class Tracer;
using WeakNameVarBaseMap =
std::map<std::string, std::vector<std::weak_ptr<VarBase>>>;
namespace details {
template <typename T>
struct NameVarMapTrait {};
template <>
struct NameVarMapTrait<VarBase> {
using Type = std::map<std::string, std::vector<std::shared_ptr<VarBase>>>;
};
template <>
struct NameVarMapTrait<VariableWrapper> {
using Type = std::map<std::string, SavedVariableWrapperList>;
};
template <>
struct NameVarMapTrait<egr::EagerVariable> {
using Type =
std::map<std::string, std::vector<std::shared_ptr<egr::EagerVariable>>>;
};
} // namespace details
template <typename T>
using NameVarMap = typename details::NameVarMapTrait<T>::Type;
using NameVarBaseMap = NameVarMap<VarBase>;
using NameVariableWrapperMap = NameVarMap<VariableWrapper>;
using NameTensorMap = NameVarMap<egr::EagerVariable>;
using VariableWrapperList = std::vector<std::shared_ptr<VariableWrapper>>;
} // namespace imperative
} // namespace paddle
......@@ -14,6 +14,8 @@
#include "paddle/phi/core/custom_kernel.h"
#include "glog/logging.h"
namespace phi {
void CustomKernelMap::RegisterCustomKernel(const std::string& name,
......
......@@ -13,3 +13,162 @@ See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/phi/core/enforce.h"
#include <map>
#include <memory>
#include <unordered_map>
#include <vector>
#include <boost/variant.hpp>
// <boost/variant.hpp> is not suitable to be placed in the header file,
// it will introduce a large number of unnecessary includes, and these type
// declarations that depend on boost are also not suitable for the phi header
// file. Do some repeated forward declarations here to avoid
// <boost/variant.hpp> spreading to a large number of phi kernel files
namespace egr {
class EagerVariable;
}
namespace paddle {
namespace framework {
class BlockDesc;
using Attribute = boost::variant<boost::blank,
int,
float,
std::string,
std::vector<int>,
std::vector<float>,
std::vector<std::string>,
bool,
std::vector<bool>,
BlockDesc*,
int64_t,
std::vector<BlockDesc*>,
std::vector<int64_t>,
std::vector<double>>;
using AttributeMap = std::unordered_map<std::string, Attribute>;
} // namespace framework
namespace imperative {
class VariableWrapper;
class SavedVariableWrapperList;
class VarBase;
namespace details {
template <typename T>
struct NameVarMapTrait {};
template <>
struct NameVarMapTrait<VarBase> {
using Type = std::map<std::string, std::vector<std::shared_ptr<VarBase>>>;
};
template <>
struct NameVarMapTrait<VariableWrapper> {
using Type = std::map<std::string, SavedVariableWrapperList>;
};
template <>
struct NameVarMapTrait<egr::EagerVariable> {
using Type =
std::map<std::string, std::vector<std::shared_ptr<egr::EagerVariable>>>;
};
} // namespace details
template <typename T>
using NameVarMap = typename details::NameVarMapTrait<T>::Type;
using NameVarBaseMap = NameVarMap<VarBase>;
using NameVariableWrapperMap = NameVarMap<VariableWrapper>;
using NameTensorMap = NameVarMap<egr::EagerVariable>;
} // namespace imperative
} // namespace paddle
namespace phi {
namespace enforce {
template <typename T>
static std::string ReplaceComplexTypeStr(std::string str,
const std::string& type_name) {
auto demangle_type_str = demangle(typeid(T).name());
size_t start_pos = 0;
while ((start_pos = str.find(demangle_type_str, start_pos)) !=
std::string::npos) {
str.replace(start_pos, demangle_type_str.length(), type_name);
start_pos += type_name.length();
}
return str;
}
#define __REPLACE_COMPLEX_TYPE_STR__(__TYPENAME, __STR) \
do { \
__STR = \
phi::enforce::ReplaceComplexTypeStr<__TYPENAME>(__STR, #__TYPENAME); \
} while (0)
static std::string SimplifyDemangleStr(std::string str) {
// the older is important, you have to put complex types in front
__REPLACE_COMPLEX_TYPE_STR__(paddle::framework::AttributeMap, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::framework::Attribute, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameVariableWrapperMap, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameVarBaseMap, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameTensorMap, str);
__REPLACE_COMPLEX_TYPE_STR__(std::string, str);
return str;
}
std::string GetCurrentTraceBackString(bool for_signal) {
std::ostringstream sout;
if (!for_signal) {
sout << "\n\n--------------------------------------\n";
sout << "C++ Traceback (most recent call last):";
sout << "\n--------------------------------------\n";
}
#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL)
static constexpr int TRACE_STACK_LIMIT = 100;
void* call_stack[TRACE_STACK_LIMIT];
auto size = backtrace(call_stack, TRACE_STACK_LIMIT);
auto symbols = backtrace_symbols(call_stack, size);
Dl_info info;
int idx = 0;
// `for_signal` used to remove the stack trace introduced by
// obtaining the error stack trace when the signal error occurred,
// that is not related to the signal error self, remove it to
// avoid misleading users and developers
int end_idx = for_signal ? 2 : 0;
for (int i = size - 1; i >= end_idx; --i) {
if (dladdr(call_stack[i], &info) && info.dli_sname) {
auto demangled = demangle(info.dli_sname);
std::string path(info.dli_fname);
// C++ traceback info are from core.so
if (path.substr(path.length() - 3).compare(".so") == 0) {
sout << paddle::string::Sprintf(
"%-3d %s\n", idx++, SimplifyDemangleStr(demangled));
}
}
}
free(symbols);
#else
sout << "Not support stack backtrace yet.\n";
#endif
return sout.str();
}
std::string SimplifyErrorTypeFormat(const std::string& str) {
std::ostringstream sout;
size_t type_end_pos = str.find(":", 0);
if (type_end_pos == std::string::npos) {
sout << str;
} else {
// Remove "Error:", add "()""
sout << "(" << str.substr(0, type_end_pos - 5) << ")"
<< str.substr(type_end_pos + 1);
}
return sout.str();
}
} // namespace enforce
} // namespace phi
......@@ -28,38 +28,32 @@ limitations under the License. */
#include <windows.h> // GetModuleFileName, Sleep
#endif
#include <fstream>
#include <iomanip>
#include <memory>
#include <sstream>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <utility>
#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL)
#include <execinfo.h>
#endif
#define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h
#include "gflags/gflags.h"
#include "glog/logging.h"
#define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h
#include "paddle/phi/core/errors.h"
#include "paddle/utils/string/printf.h"
#include "paddle/utils/string/to_string.h"
// Note: these headers for simplify demangle type string
#include "paddle/phi/core/compat/type_defs.h"
DECLARE_int32(call_stack_level);
namespace phi {
class ErrorSummary;
} // namespace phi
DECLARE_int32(call_stack_level);
namespace phi {
namespace enforce {
/** HELPER MACROS AND FUNCTIONS **/
/** HELPER MACROS AND FUNCTIONS **/
#ifndef PADDLE_MAY_THROW
#define PADDLE_MAY_THROW noexcept(false)
#endif
......@@ -180,76 +174,11 @@ struct BinaryCompareMessageConverter<false> {
};
} // namespace details
template <typename T>
inline std::string ReplaceComplexTypeStr(std::string str,
const std::string& type_name) {
auto demangle_type_str = demangle(typeid(T).name());
size_t start_pos = 0;
while ((start_pos = str.find(demangle_type_str, start_pos)) !=
std::string::npos) {
str.replace(start_pos, demangle_type_str.length(), type_name);
start_pos += type_name.length();
}
return str;
}
#define __REPLACE_COMPLEX_TYPE_STR__(__TYPENAME, __STR) \
do { \
__STR = \
phi::enforce::ReplaceComplexTypeStr<__TYPENAME>(__STR, #__TYPENAME); \
} while (0)
inline std::string SimplifyDemangleStr(std::string str) {
// the older is important, you have to put complex types in front
__REPLACE_COMPLEX_TYPE_STR__(paddle::framework::AttributeMap, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::framework::Attribute, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameVariableWrapperMap, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameVarBaseMap, str);
__REPLACE_COMPLEX_TYPE_STR__(std::string, str);
return str;
}
inline std::string GetCurrentTraceBackString(bool for_signal = false) {
std::ostringstream sout;
if (!for_signal) {
sout << "\n\n--------------------------------------\n";
sout << "C++ Traceback (most recent call last):";
sout << "\n--------------------------------------\n";
}
#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL)
static constexpr int TRACE_STACK_LIMIT = 100;
void* call_stack[TRACE_STACK_LIMIT];
auto size = backtrace(call_stack, TRACE_STACK_LIMIT);
auto symbols = backtrace_symbols(call_stack, size);
Dl_info info;
int idx = 0;
// `for_signal` used to remove the stack trace introduced by
// obtaining the error stack trace when the signal error occurred,
// that is not related to the signal error self, remove it to
// avoid misleading users and developers
int end_idx = for_signal ? 2 : 0;
for (int i = size - 1; i >= end_idx; --i) {
if (dladdr(call_stack[i], &info) && info.dli_sname) {
auto demangled = demangle(info.dli_sname);
std::string path(info.dli_fname);
// C++ traceback info are from core.so
if (path.substr(path.length() - 3).compare(".so") == 0) {
sout << paddle::string::Sprintf(
"%-3d %s\n", idx++, SimplifyDemangleStr(demangled));
}
}
}
free(symbols);
#else
sout << "Not support stack backtrace yet.\n";
#endif
return sout.str();
}
std::string GetCurrentTraceBackString(bool for_signal = false);
std::string SimplifyErrorTypeFormat(const std::string& str);
template <typename StrType>
inline std::string GetErrorSumaryString(StrType&& what,
static std::string GetErrorSumaryString(StrType&& what,
const char* file,
int line) {
std::ostringstream sout;
......@@ -264,7 +193,20 @@ inline std::string GetErrorSumaryString(StrType&& what,
}
template <typename StrType>
inline std::string GetTraceBackString(StrType&& what,
std::string GetCompleteTraceBackString(StrType&& what,
const char* file,
int line) {
std::ostringstream sout;
sout << "\n----------------------\nError Message "
"Summary:\n----------------------\n";
sout << paddle::string::Sprintf(
"%s (at %s:%d)", std::forward<StrType>(what), file, line)
<< std::endl;
return GetCurrentTraceBackString() + sout.str();
}
template <typename StrType>
static std::string GetTraceBackString(StrType&& what,
const char* file,
int line) {
if (FLAGS_call_stack_level > 1) {
......@@ -275,19 +217,6 @@ inline std::string GetTraceBackString(StrType&& what,
}
}
inline std::string SimplifyErrorTypeFormat(const std::string& str) {
std::ostringstream sout;
size_t type_end_pos = str.find(":", 0);
if (type_end_pos == std::string::npos) {
sout << str;
} else {
// Remove "Error:", add "()""
sout << "(" << str.substr(0, type_end_pos - 5) << ")"
<< str.substr(type_end_pos + 1);
}
return sout.str();
}
inline bool is_error(bool stat) { return !stat; }
// Note: This Macro can only be used within enforce.h
......@@ -348,6 +277,8 @@ struct EnforceNotMet : public std::exception {
}
}
~EnforceNotMet() override = default;
private:
// Used to determine the final type of exception thrown
phi::ErrorCode code_ = phi::ErrorCode::LEGACY;
......
......@@ -14,7 +14,8 @@
#include "paddle/phi/core/kernel_factory.h"
// See Note [ Why still include the fluid headers? ]
#include "glog/logging.h"
#include "paddle/phi/core/enforce.h"
namespace phi {
......
......@@ -15,6 +15,7 @@ limitations under the License. */
#pragma once
#include <cassert>
#include <map>
#include <mutex>
#include <string>
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册