Skip to content

P
Paddle

机器未来 / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
未验证 提交 6ca49164 编写于 作者: X xiongkun 提交者： GitHub
浏览文件
操作

[PTen] Migrate string tinyformat errors and part of enforce into pten (#39051) 

* transfer: string tinyformat errors and part of enforce into pten

* remove comment

* fix by code review

* assert is not compile in -DNDEBUG

* add string as dependences of paddle_inference
上级 d43655ba
隐藏空白更改
内联 并排
Showing with 1664 addition and 1320 deletion
cmake/inference_lib.cmake
浏览文件 @ 6ca49164
...@@ -189,6 +189,7 @@ copy(inference_lib_dist ...@@ -189,6 +189,7 @@ copy(inference_lib_dist
copy_part_of_thrid_party(inference_lib_dist ${PADDLE_INFERENCE_INSTALL_DIR}) copy_part_of_thrid_party(inference_lib_dist ${PADDLE_INFERENCE_INSTALL_DIR})
set(src_dir "${PADDLE_SOURCE_DIR}/paddle/fluid") set(src_dir "${PADDLE_SOURCE_DIR}/paddle/fluid")
if(WIN32) if(WIN32)
if(WITH_STATIC_LIB) if(WITH_STATIC_LIB)
set(paddle_inference_lib $<TARGET_FILE_DIR:paddle_inference>/libpaddle_inference.lib set(paddle_inference_lib $<TARGET_FILE_DIR:paddle_inference>/libpaddle_inference.lib
...@@ -304,7 +305,7 @@ copy(fluid_lib_dist ...@@ -304,7 +305,7 @@ copy(fluid_lib_dist
) )
set(module "platform") set(module "platform")
set(platform_lib_deps profiler_proto error_codes_proto) set(platform_lib_deps profiler_proto errors)
if(WITH_GPU) if(WITH_GPU)
set(platform_lib_deps ${platform_lib_deps} external_error_proto) set(platform_lib_deps ${platform_lib_deps} external_error_proto)
endif(WITH_GPU) endif(WITH_GPU)
...@@ -317,7 +318,7 @@ copy(fluid_lib_dist ...@@ -317,7 +318,7 @@ copy(fluid_lib_dist
set(module "string") set(module "string")
copy(fluid_lib_dist copy(fluid_lib_dist
SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/tinyformat/*.h SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/${module}/*.h ${PADDLE_SOURCE_DIR}/paddle/utils/${module}/tinyformat/*.h
DSTS ${dst_dir}/${module} ${dst_dir}/${module}/tinyformat DSTS ${dst_dir}/${module} ${dst_dir}/${module}/tinyformat
) )
......
paddle/CMakeLists.txt
浏览文件 @ 6ca49164
add_subdirectory(utils)
add_subdirectory(scripts) add_subdirectory(scripts)
add_subdirectory(testing) add_subdirectory(testing)
set(PYTHON_TESTS_DIR ${PADDLE_BINARY_DIR}/python/paddle/fluid/tests CACHE INTERNAL "python tests directory") set(PYTHON_TESTS_DIR ${PADDLE_BINARY_DIR}/python/paddle/fluid/tests CACHE INTERNAL "python tests directory")
......
paddle/fluid/CMakeLists.txt
浏览文件 @ 6ca49164
...@@ -4,7 +4,6 @@ add_subdirectory(distributed) ...@@ -4,7 +4,6 @@ add_subdirectory(distributed)
add_subdirectory(framework) add_subdirectory(framework)
add_subdirectory(imperative) add_subdirectory(imperative)
add_subdirectory(operators) add_subdirectory(operators)
add_subdirectory(string)
add_subdirectory(pybind) add_subdirectory(pybind)
add_subdirectory(eager) add_subdirectory(eager)
# NOTE: please add subdirectory inference at last. # NOTE: please add subdirectory inference at last.
......
paddle/fluid/framework/op_call_stack.h
浏览文件 @ 6ca49164
...@@ -19,12 +19,6 @@ limitations under the License. */ ...@@ -19,12 +19,6 @@ limitations under the License. */
#include "paddle/fluid/framework/type_defs.h" #include "paddle/fluid/framework/type_defs.h"
#include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/enforce.h"
namespace paddle {
namespace platform {
struct EnforceNotMet;
} // namespace platform
} // namespace paddle
namespace paddle { namespace paddle {
namespace framework { namespace framework {
......
paddle/fluid/framework/op_proto_maker_test.cc
浏览文件 @ 6ca49164
...@@ -18,12 +18,6 @@ limitations under the License. */ ...@@ -18,12 +18,6 @@ limitations under the License. */
#include "gtest/gtest-test-part.h" #include "gtest/gtest-test-part.h"
#include "gtest/gtest.h" #include "gtest/gtest.h"
namespace paddle {
namespace platform {
struct EnforceNotMet;
} // namespace platform
} // namespace paddle
class TestAttrProtoMaker : public paddle::framework::OpProtoAndCheckerMaker { class TestAttrProtoMaker : public paddle::framework::OpProtoAndCheckerMaker {
public: public:
void Make() { void Make() {
......
paddle/fluid/framework/type_defs.h
浏览文件 @ 6ca49164
...@@ -21,8 +21,8 @@ limitations under the License. */ ...@@ -21,8 +21,8 @@ limitations under the License. */
#include <unordered_map> #include <unordered_map>
#include <unordered_set> #include <unordered_set>
#include <vector> #include <vector>
#include "paddle/fluid/imperative/type_defs.h"
#include "paddle/fluid/platform/variant.h" #include "paddle/fluid/platform/variant.h"
#include "paddle/pten/core/type_defs.h"
#include "paddle/utils/small_vector.h" #include "paddle/utils/small_vector.h"
namespace paddle { namespace paddle {
...@@ -39,14 +39,6 @@ class InferNoNeedBufferVarsFN; ...@@ -39,14 +39,6 @@ class InferNoNeedBufferVarsFN;
using VariableNameMap = std::map<std::string, std::vector<std::string>>; using VariableNameMap = std::map<std::string, std::vector<std::string>>;
using VariableValueMap = std::map<std::string, std::vector<Variable*>>; using VariableValueMap = std::map<std::string, std::vector<Variable*>>;
// The order should be as same as framework.proto
using Attribute = boost::variant<
boost::blank, int, float, std::string, std::vector<int>, std::vector<float>,
std::vector<std::string>, bool, std::vector<bool>, BlockDesc*, int64_t,
std::vector<BlockDesc*>, std::vector<int64_t>, std::vector<double>>;
using AttributeMap = std::unordered_map<std::string, Attribute>;
#ifdef PADDLE_WITH_ASCEND_CL #ifdef PADDLE_WITH_ASCEND_CL
using NPUAttribute = using NPUAttribute =
boost::variant<boost::blank, int, float, std::string, std::vector<int>, boost::variant<boost::blank, int, float, std::string, std::vector<int>,
......
paddle/fluid/imperative/type_defs.h
浏览文件 @ 6ca49164
...@@ -13,47 +13,4 @@ See the License for the specific language governing permissions and ...@@ -13,47 +13,4 @@ See the License for the specific language governing permissions and
limitations under the License. */ limitations under the License. */
#pragma once #pragma once
#include "paddle/pten/core/type_defs.h"
#include <map>
#include <memory>
#include <string>
#include <vector>
namespace paddle {
namespace imperative {
class VariableWrapper;
class SavedVariableWrapperList;
class VarBase;
class OpBase;
class GradOpNode;
class Tracer;
using WeakNameVarBaseMap =
std::map<std::string, std::vector<std::weak_ptr<VarBase>>>;
namespace details {
template <typename T>
struct NameVarMapTrait {};
template <>
struct NameVarMapTrait<VarBase> {
using Type = std::map<std::string, std::vector<std::shared_ptr<VarBase>>>;
};
template <>
struct NameVarMapTrait<VariableWrapper> {
using Type = std::map<std::string, SavedVariableWrapperList>;
};
} // namespace details
template <typename T>
using NameVarMap = typename details::NameVarMapTrait<T>::Type;
using NameVarBaseMap = NameVarMap<VarBase>;
using NameVariableWrapperMap = NameVarMap<VariableWrapper>;
using VariableWrapperList = std::vector<std::shared_ptr<VariableWrapper>>;
} // namespace imperative
} // namespace paddle
paddle/fluid/inference/CMakeLists.txt
浏览文件 @ 6ca49164
...@@ -36,6 +36,7 @@ endif() ...@@ -36,6 +36,7 @@ endif()
# fluid_modules exclude API-interface of inference/api and inference/capi_exp # fluid_modules exclude API-interface of inference/api and inference/capi_exp
get_property(fluid_modules GLOBAL PROPERTY FLUID_MODULES) get_property(fluid_modules GLOBAL PROPERTY FLUID_MODULES)
get_property(pten_modules GLOBAL PROPERTY PTEN_MODULES) get_property(pten_modules GLOBAL PROPERTY PTEN_MODULES)
set(utils_modules stringpiece pretty_log string_helper)
add_subdirectory(api) add_subdirectory(api)
...@@ -46,9 +47,9 @@ set(STATIC_INFERENCE_API paddle_inference_api analysis_predictor ...@@ -46,9 +47,9 @@ set(STATIC_INFERENCE_API paddle_inference_api analysis_predictor
analysis_config paddle_pass_builder activation_functions ${mkldnn_quantizer_cfg}) analysis_config paddle_pass_builder activation_functions ${mkldnn_quantizer_cfg})
#TODO(wilber, T8T9): Do we still need to support windows gpu static library? #TODO(wilber, T8T9): Do we still need to support windows gpu static library?
if(WIN32 AND WITH_GPU) if(WIN32 AND WITH_GPU)
cc_library(paddle_inference DEPS ${fluid_modules} ${pten_modules} ${STATIC_INFERENCE_API}) cc_library(paddle_inference DEPS ${fluid_modules} ${pten_modules} ${STATIC_INFERENCE_API} ${utils_modules})
else() else()
create_static_lib(paddle_inference ${fluid_modules} ${pten_modules} ${STATIC_INFERENCE_API}) create_static_lib(paddle_inference ${fluid_modules} ${pten_modules} ${STATIC_INFERENCE_API} ${utils_modules})
if(WITH_IPU) if(WITH_IPU)
target_link_libraries(paddle_inference -Wl,--allow-multiple-definition popart_canonicalization_utils) target_link_libraries(paddle_inference -Wl,--allow-multiple-definition popart_canonicalization_utils)
endif() endif()
......
paddle/fluid/platform/CMakeLists.txt
浏览文件 @ 6ca49164
proto_library(profiler_proto SRCS profiler.proto DEPS framework_proto simple_threadpool) proto_library(profiler_proto SRCS profiler.proto DEPS framework_proto simple_threadpool)
proto_library(error_codes_proto SRCS error_codes.proto)
if(WITH_GPU) if(WITH_GPU)
proto_library(external_error_proto SRCS external_error.proto) proto_library(external_error_proto SRCS external_error.proto)
endif(WITH_GPU) endif(WITH_GPU)
if (WITH_PYTHON) if (WITH_PYTHON)
py_proto_compile(profiler_py_proto SRCS profiler.proto) py_proto_compile(profiler_py_proto SRCS profiler.proto)
add_custom_target(profiler_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py) add_custom_target(profiler_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
...@@ -28,10 +26,9 @@ endif() ...@@ -28,10 +26,9 @@ endif()
cc_library(flags SRCS flags.cc DEPS gflags boost) cc_library(flags SRCS flags.cc DEPS gflags boost)
cc_library(denormal SRCS denormal.cc DEPS) cc_library(denormal SRCS denormal.cc DEPS)
cc_library(errors SRCS errors.cc DEPS error_codes_proto)
cc_test(errors_test SRCS errors_test.cc DEPS errors enforce) cc_test(errors_test SRCS errors_test.cc DEPS errors enforce)
set(enforce_deps flags errors boost flags) set(enforce_deps flags errors boost flags pten_enforce)
if(WITH_GPU) if(WITH_GPU)
set(enforce_deps ${enforce_deps} external_error_proto) set(enforce_deps ${enforce_deps} external_error_proto)
endif() endif()
......
paddle/fluid/platform/enforce.h
浏览文件 @ 6ca49164
...@@ -95,17 +95,16 @@ limitations under the License. */ ...@@ -95,17 +95,16 @@ limitations under the License. */
// Note: these headers for simplify demangle type string // Note: these headers for simplify demangle type string
#include "paddle/fluid/framework/type_defs.h" #include "paddle/fluid/framework/type_defs.h"
#include "paddle/fluid/imperative/type_defs.h" #include "paddle/fluid/imperative/type_defs.h"
#include "paddle/pten/core/enforce.h"
// Note: this header for simplify HIP and CUDA type string // Note: this header for simplify HIP and CUDA type string
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
#include "paddle/fluid/platform/device/gpu/gpu_types.h" #include "paddle/fluid/platform/device/gpu/gpu_types.h"
#endif #endif
#include "paddle/fluid/platform/flags.h" #include "paddle/fluid/platform/flags.h"
namespace paddle { namespace pten {
namespace platform {
class ErrorSummary; class ErrorSummary;
} // namespace platform } // namespace pten
} // namespace paddle
#if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP) #if defined(PADDLE_WITH_CUDA) || defined(PADDLE_WITH_HIP)
DECLARE_int64(gpu_allocator_retry_time); DECLARE_int64(gpu_allocator_retry_time);
...@@ -114,6 +113,7 @@ DECLARE_int32(call_stack_level); ...@@ -114,6 +113,7 @@ DECLARE_int32(call_stack_level);
namespace paddle { namespace paddle {
namespace platform { namespace platform {
using namespace ::pten::enforce; // NOLINT
/** HELPER MACROS AND FUNCTIONS **/ /** HELPER MACROS AND FUNCTIONS **/
...@@ -121,478 +121,6 @@ namespace platform { ...@@ -121,478 +121,6 @@ namespace platform {
#define PADDLE_MAY_THROW noexcept(false) #define PADDLE_MAY_THROW noexcept(false)
#endif #endif
// Because most enforce conditions would evaluate to true, we can use
// __builtin_expect to instruct the C++ compiler to generate code that
// always forces branch prediction of true.
// This generates faster binary code. __builtin_expect is since C++11.
// For more details, please check https://stackoverflow.com/a/43870188/724872.
#if !defined(_WIN32)
#define UNLIKELY(condition) __builtin_expect(static_cast<bool>(condition), 0)
#else
// there is no equivalent intrinsics in msvc.
#define UNLIKELY(condition) (condition)
#endif
#if !defined(_WIN32)
#define LIKELY(condition) __builtin_expect(static_cast<bool>(condition), 1)
#else
// there is no equivalent intrinsics in msvc.
#define LIKELY(condition) (condition)
#endif
#if defined _WIN32 && defined PADDLE_ON_INFERENCE && defined PADDLE_NO_PYTHON
#define HANDLE_THE_ERROR try {
#define END_HANDLE_THE_ERROR \
} \
catch (const std::exception& e) { \
std::cout << e.what() << std::endl; \
throw; \
}
#else
#define HANDLE_THE_ERROR
#define END_HANDLE_THE_ERROR
#endif
#ifdef __GNUC__
inline std::string demangle(std::string name) {
int status = -4; // some arbitrary value to eliminate the compiler warning
std::unique_ptr<char, void (*)(void*)> res{
abi::__cxa_demangle(name.c_str(), NULL, NULL, &status), std::free};
return (status == 0) ? res.get() : name;
}
#else
inline std::string demangle(std::string name) { return name; }
#endif
namespace details {
template <typename T>
inline constexpr bool IsArithmetic() {
return std::is_arithmetic<T>::value;
}
template <typename T1, typename T2, bool kIsArithmetic /* = true */>
struct TypeConverterImpl {
using Type1 = typename std::common_type<T1, T2>::type;
using Type2 = Type1;
};
template <typename T1, typename T2>
struct TypeConverterImpl<T1, T2, false> {
using Type1 = T1;
using Type2 = T2;
};
template <typename T1, typename T2>
struct TypeConverter {
static constexpr bool kIsArithmetic =
IsArithmetic<T1>() && IsArithmetic<T2>();
using Type1 = typename TypeConverterImpl<T1, T2, kIsArithmetic>::Type1;
using Type2 = typename TypeConverterImpl<T1, T2, kIsArithmetic>::Type2;
};
template <typename T1, typename T2>
using CommonType1 = typename std::add_lvalue_reference<
typename std::add_const<typename TypeConverter<T1, T2>::Type1>::type>::type;
template <typename T1, typename T2>
using CommonType2 = typename std::add_lvalue_reference<
typename std::add_const<typename TypeConverter<T1, T2>::Type2>::type>::type;
// Here, we use SFINAE to check whether T can be converted to std::string
template <typename T>
struct CanToString {
private:
using YesType = uint8_t;
using NoType = uint16_t;
template <typename U>
static YesType Check(decltype(std::cout << std::declval<U>())) {
return 0;
}
template <typename U>
static NoType Check(...) {
return 0;
}
public:
static constexpr bool kValue =
std::is_same<YesType, decltype(Check<T>(std::cout))>::value;
};
template <bool kCanToString /* = true */>
struct BinaryCompareMessageConverter {
template <typename T>
static std::string Convert(const char* expression, const T& value) {
return expression + std::string(":") + string::to_string(value);
}
};
template <>
struct BinaryCompareMessageConverter<false> {
template <typename T>
static const char* Convert(const char* expression, const T& value) {
return expression;
}
};
} // namespace details
template <typename T>
inline std::string ReplaceComplexTypeStr(std::string str,
const std::string& type_name) {
auto demangle_type_str = demangle(typeid(T).name());
size_t start_pos = 0;
while ((start_pos = str.find(demangle_type_str, start_pos)) !=
std::string::npos) {
str.replace(start_pos, demangle_type_str.length(), type_name);
start_pos += type_name.length();
}
return str;
}
#define __REPLACE_COMPLEX_TYPE_STR__(__TYPENAME, __STR) \
do { \
__STR = paddle::platform::ReplaceComplexTypeStr<__TYPENAME>(__STR, \
#__TYPENAME); \
} while (0)
inline std::string SimplifyDemangleStr(std::string str) {
// the older is important, you have to put complex types in front
__REPLACE_COMPLEX_TYPE_STR__(paddle::framework::AttributeMap, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::framework::Attribute, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameVariableWrapperMap, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameVarBaseMap, str);
__REPLACE_COMPLEX_TYPE_STR__(std::string, str);
return str;
}
inline std::string GetCurrentTraceBackString(bool for_signal = false) {
std::ostringstream sout;
if (!for_signal) {
sout << "\n\n--------------------------------------\n";
sout << "C++ Traceback (most recent call last):";
sout << "\n--------------------------------------\n";
}
#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL)
static constexpr int TRACE_STACK_LIMIT = 100;
void* call_stack[TRACE_STACK_LIMIT];
auto size = backtrace(call_stack, TRACE_STACK_LIMIT);
auto symbols = backtrace_symbols(call_stack, size);
Dl_info info;
int idx = 0;
// `for_signal` used to remove the stack trace introduced by
// obtaining the error stack trace when the signal error occurred,
// that is not related to the signal error self, remove it to
// avoid misleading users and developers
int end_idx = for_signal ? 2 : 0;
for (int i = size - 1; i >= end_idx; --i) {
if (dladdr(call_stack[i], &info) && info.dli_sname) {
auto demangled = demangle(info.dli_sname);
std::string path(info.dli_fname);
// C++ traceback info are from core.so
if (path.substr(path.length() - 3).compare(".so") == 0) {
sout << string::Sprintf("%-3d %s\n", idx++,
SimplifyDemangleStr(demangled));
}
}
}
free(symbols);
#else
sout << "Not support stack backtrace yet.\n";
#endif
return sout.str();
}
template <typename StrType>
inline std::string GetErrorSumaryString(StrType&& what, const char* file,
int line) {
std::ostringstream sout;
if (FLAGS_call_stack_level > 1) {
sout << "\n----------------------\nError Message "
"Summary:\n----------------------\n";
}
sout << string::Sprintf("%s (at %s:%d)", std::forward<StrType>(what), file,
line)
<< std::endl;
return sout.str();
}
template <typename StrType>
inline std::string GetTraceBackString(StrType&& what, const char* file,
int line) {
if (FLAGS_call_stack_level > 1) {
// FLAGS_call_stack_level>1 means showing c++ call stack
return GetCurrentTraceBackString() + GetErrorSumaryString(what, file, line);
} else {
return GetErrorSumaryString(what, file, line);
}
}
inline std::string SimplifyErrorTypeFormat(const std::string& str) {
std::ostringstream sout;
size_t type_end_pos = str.find(":", 0);
if (type_end_pos == std::string::npos) {
sout << str;
} else {
// Remove "Error:", add "()""
sout << "(" << str.substr(0, type_end_pos - 5) << ")"
<< str.substr(type_end_pos + 1);
}
return sout.str();
}
inline bool is_error(bool stat) { return !stat; }
// Note: This Macro can only be used within enforce.h
#define __THROW_ERROR_INTERNAL__(__ERROR_SUMMARY) \
do { \
HANDLE_THE_ERROR \
throw ::paddle::platform::EnforceNotMet(__ERROR_SUMMARY, __FILE__, \
__LINE__); \
END_HANDLE_THE_ERROR \
} while (0)
/** ENFORCE EXCEPTION AND MACROS **/
struct EnforceNotMet : public std::exception {
public:
EnforceNotMet(std::exception_ptr e, const char* file, int line) {
try {
std::rethrow_exception(e);
} catch (platform::EnforceNotMet& e) {
code_ = e.code();
err_str_ = GetTraceBackString(e.what(), file, line);
simple_err_str_ = SimplifyErrorTypeFormat(err_str_);
} catch (std::exception& e) {
err_str_ = GetTraceBackString(e.what(), file, line);
simple_err_str_ = SimplifyErrorTypeFormat(err_str_);
}
}
EnforceNotMet(const std::string& str, const char* file, int line)
: err_str_(GetTraceBackString(str, file, line)) {
simple_err_str_ = SimplifyErrorTypeFormat(err_str_);
}
EnforceNotMet(const ErrorSummary& error, const char* file, int line)
: code_(error.code()),
err_str_(GetTraceBackString(error.to_string(), file, line)) {
simple_err_str_ = SimplifyErrorTypeFormat(err_str_);
}
const char* what() const noexcept override {
if (FLAGS_call_stack_level > 1) {
return err_str_.c_str();
} else {
return simple_err_str_.c_str();
}
}
error::Code code() const { return code_; }
const std::string& error_str() const { return err_str_; }
const std::string& simple_error_str() const { return simple_err_str_; }
void set_error_str(std::string str) {
if (FLAGS_call_stack_level > 1) {
err_str_ = str;
} else {
simple_err_str_ = str;
}
}
private:
// Used to determine the final type of exception thrown
error::Code code_ = error::LEGACY;
// Complete error message
// e.g. InvalidArgumentError: ***
std::string err_str_;
// Simple errror message used when no C++ stack and python compile stack
// e.g. (InvalidArgument) ***
std::string simple_err_str_;
};
#define PADDLE_THROW(...) \
do { \
HANDLE_THE_ERROR \
throw ::paddle::platform::EnforceNotMet( \
::paddle::platform::ErrorSummary(__VA_ARGS__), __FILE__, __LINE__); \
END_HANDLE_THE_ERROR \
} while (0)
#if defined(__CUDA_ARCH__)
// For cuda, the assertions can affect performance and it is therefore
// recommended to disable them in production code
// https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#assertion
#define PADDLE_ENFORCE(_IS_NOT_ERROR, __FORMAT, ...) \
do { \
if (!(_IS_NOT_ERROR)) { \
printf("Error: %s:%d Assertion `%s` failed. " __FORMAT "\n", __FILE__, \
__LINE__, #_IS_NOT_ERROR, ##__VA_ARGS__); \
asm("trap;"); \
} \
} while (0)
#elif defined(__HIPCC__)
#define PADDLE_ENFORCE(_IS_NOT_ERROR, __FORMAT, ...) \
do { \
if (!(_IS_NOT_ERROR)) { \
printf("Error: %s:%d Assertion `%s` failed. " __FORMAT "\n", __FILE__, \
__LINE__, #_IS_NOT_ERROR, ##__VA_ARGS__); \
abort(); \
} \
} while (0)
#else
#define PADDLE_ENFORCE(COND, ...) \
do { \
auto __cond__ = (COND); \
if (UNLIKELY(::paddle::platform::is_error(__cond__))) { \
__THROW_ERROR_INTERNAL__(::paddle::platform::ErrorSummary(__VA_ARGS__)); \
} \
} while (0)
#endif
/*
* Some enforce helpers here, usage:
* int a = 1;
* int b = 2;
* PADDLE_ENFORCE_EQ(a, b);
*
* will raise an expression described as follows:
* "Expected input a == b, but received a(1) != b(2)."
* with detailed stack information.
*
* extra messages is also supported, for example:
* PADDLE_ENFORCE(a, b, "some simple enforce failed between %d numbers", 2)
*/
#define PADDLE_ENFORCE_NOT_NULL(__VAL, ...) \
do { \
if (UNLIKELY(nullptr == (__VAL))) { \
auto __summary__ = ::paddle::platform::ErrorSummary(__VA_ARGS__); \
auto __message__ = ::paddle::string::Sprintf( \
"%s\n [Hint: " #__VAL " should not be null.]", \
__summary__.error_message()); \
__THROW_ERROR_INTERNAL__( \
::paddle::platform::ErrorSummary(__summary__.code(), __message__)); \
} \
} while (0)
#define __PADDLE_BINARY_COMPARE(__VAL1, __VAL2, __CMP, __INV_CMP, ...) \
do { \
auto __val1 = (__VAL1); \
auto __val2 = (__VAL2); \
using __TYPE1__ = decltype(__val1); \
using __TYPE2__ = decltype(__val2); \
using __COMMON_TYPE1__ = \
::paddle::platform::details::CommonType1<__TYPE1__, __TYPE2__>; \
using __COMMON_TYPE2__ = \
::paddle::platform::details::CommonType2<__TYPE1__, __TYPE2__>; \
bool __is_not_error = (static_cast<__COMMON_TYPE1__>(__val1))__CMP( \
static_cast<__COMMON_TYPE2__>(__val2)); \
if (UNLIKELY(!__is_not_error)) { \
auto __summary__ = ::paddle::platform::ErrorSummary(__VA_ARGS__); \
constexpr bool __kCanToString__ = \
::paddle::platform::details::CanToString<__TYPE1__>::kValue && \
::paddle::platform::details::CanToString<__TYPE2__>::kValue; \
auto __message__ = ::paddle::string::Sprintf( \
"%s\n [Hint: Expected %s " #__CMP \
" %s, but received %s " #__INV_CMP " %s.]", \
__summary__.error_message(), #__VAL1, #__VAL2, \
::paddle::platform::details::BinaryCompareMessageConverter< \
__kCanToString__>::Convert(#__VAL1, __val1), \
::paddle::platform::details::BinaryCompareMessageConverter< \
__kCanToString__>::Convert(#__VAL2, __val2)); \
__THROW_ERROR_INTERNAL__( \
::paddle::platform::ErrorSummary(__summary__.code(), __message__)); \
} \
} while (0)
#define PADDLE_ENFORCE_EQ(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, ==, !=, __VA_ARGS__)
#define PADDLE_ENFORCE_NE(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, !=, ==, __VA_ARGS__)
#define PADDLE_ENFORCE_GT(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, >, <=, __VA_ARGS__)
#define PADDLE_ENFORCE_GE(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, >=, <, __VA_ARGS__)
#define PADDLE_ENFORCE_LT(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, <, >=, __VA_ARGS__)
#define PADDLE_ENFORCE_LE(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, <=, >, __VA_ARGS__)
/** EXTENDED TOOL FUNCTIONS WITH CHECKING **/
/*
* Summary: This macro is used to get Variable or internal type
* data (such as LoDTensor or SelectedRows) of the Input and
* Output in op, generally used when call scope.FindVar(Input/
* Output("Name")) or ctx.Input<LoDTensor>().
* Firstly this macro check whether the obtained pointer is null,
* and then return data if it is not null.
*
* Note: This macro is only suitable for specific scenarios and
* does not intended to be widely used. If it cannot meet the
* requirements, please use other PADDLE_ENFORCE** check macro.
*
* Parameters:
*     __PTR: pointer
* __ROLE: (string), Input or Output
* __NAME: (string), Input or Output name
* __OP_TYPE: (string), the op type
*  
* Return: The data pointed to by the pointer.
*
* Examples:
* GET_DATA_SAFELY(ctx.Input<LoDTensor>("X"), "Input", "X", "Mul");
*/
#define GET_DATA_SAFELY(__PTR, __ROLE, __NAME, __OP_TYPE) \
(([&]() -> std::add_lvalue_reference<decltype(*(__PTR))>::type { \
auto* __ptr = (__PTR); \
if (UNLIKELY(nullptr == __ptr)) { \
auto __summary__ = paddle::platform::errors::NotFound( \
"Unable to get %s data of %s %s in operator %s. " \
"Possible reasons are:\n" \
" 1. The %s is not the %s of operator %s;\n" \
" 2. The %s has no corresponding variable passed in;\n" \
" 3. The %s corresponding variable is not initialized.", \
paddle::platform::demangle( \
typeid(std::add_lvalue_reference<decltype(*__ptr)>::type) \
.name()), \
__ROLE, __NAME, __OP_TYPE, __NAME, __ROLE, __OP_TYPE, __NAME, \
__NAME); \
auto __message__ = ::paddle::string::Sprintf( \
"%s\n [Hint: pointer " #__PTR " should not be null.]", \
__summary__.error_message()); \
__THROW_ERROR_INTERNAL__( \
::paddle::platform::ErrorSummary(__summary__.code(), __message__)); \
} \
return *__ptr; \
})())
/*
* Summary: This macro is used to check whether op has specified
* Input or Output Variables. Because op's Input and Output
* checking are written similarly, so abstract this macro.
*
* Parameters:
*     __EXPR: (bool), the bool expression
* __ROLE: (string), Input or Output
* __NAME: (string), Input or Output name
* __OP_TYPE: (string), the op type
*
* Examples:
* OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "Mul");
*/
#define OP_INOUT_CHECK(__EXPR, __ROLE, __NAME, __OP_TYPE) \
do { \
PADDLE_ENFORCE_EQ(__EXPR, true, paddle::platform::errors::NotFound( \
"No %s(%s) found for %s operator.", \
__ROLE, __NAME, __OP_TYPE)); \
} while (0)
/* /*
* Summary: This BOOST_GET(_**) series macros are used to call boost::get * Summary: This BOOST_GET(_**) series macros are used to call boost::get
* safely. boost::get is not a completely safe api, although it will not * safely. boost::get is not a completely safe api, although it will not
...@@ -616,6 +144,8 @@ struct EnforceNotMet : public std::exception { ...@@ -616,6 +144,8 @@ struct EnforceNotMet : public std::exception {
*/ */
namespace details { namespace details {
using namespace pten::enforce::details; // NOLINT
#define DEFINE_SAFE_BOOST_GET(__InputType, __OutputType, __OutputTypePtr, \ #define DEFINE_SAFE_BOOST_GET(__InputType, __OutputType, __OutputTypePtr, \
__FuncName) \ __FuncName) \
template <typename OutputType, typename InputType> \ template <typename OutputType, typename InputType> \
...@@ -627,13 +157,12 @@ namespace details { ...@@ -627,13 +157,12 @@ namespace details {
return boost::get<OutputType>(input); \ return boost::get<OutputType>(input); \
} catch (boost::bad_get&) { \ } catch (boost::bad_get&) { \
HANDLE_THE_ERROR \ HANDLE_THE_ERROR \
throw ::paddle::platform::EnforceNotMet( \ throw ::pten::enforce::EnforceNotMet( \
::paddle::platform::errors::InvalidArgument( \ pten::errors::InvalidArgument( \
"boost::get failed, cannot get value " \ "boost::get failed, cannot get value " \
"(%s) by type %s, its type is %s.", \ "(%s) by type %s, its type is %s.", \
expression, \ expression, pten::enforce::demangle(typeid(OutputType).name()), \
paddle::platform::demangle(typeid(OutputType).name()), \ pten::enforce::demangle(input.type().name())), \
paddle::platform::demangle(input.type().name())), \
file, line); \ file, line); \
END_HANDLE_THE_ERROR \ END_HANDLE_THE_ERROR \
} \ } \
...@@ -647,44 +176,43 @@ DEFINE_SAFE_BOOST_GET(InputType&&, OutputType, OutputType*, ...@@ -647,44 +176,43 @@ DEFINE_SAFE_BOOST_GET(InputType&&, OutputType, OutputType*,
} // namespace details } // namespace details
#define BOOST_GET(__TYPE, __VALUE) \ #define BOOST_GET(__TYPE, __VALUE) \
::paddle::platform::details::SafeBoostGet<__TYPE>(__VALUE, #__VALUE, \ paddle::platform::details::SafeBoostGet<__TYPE>(__VALUE, #__VALUE, __FILE__, \
__FILE__, __LINE__) __LINE__)
#define BOOST_GET_CONST(__TYPE, __VALUE) \ #define BOOST_GET_CONST(__TYPE, __VALUE) \
::paddle::platform::details::SafeBoostGetConst<__TYPE>(__VALUE, #__VALUE, \ paddle::platform::details::SafeBoostGetConst<__TYPE>(__VALUE, #__VALUE, \
__FILE__, __LINE__)
#define BOOST_GET_MUTABLE(__TYPE, __VALUE) \
paddle::platform::details::SafeBoostGetMutable<__TYPE>(__VALUE, #__VALUE, \
__FILE__, __LINE__) __FILE__, __LINE__)
#define BOOST_GET_MUTABLE(__TYPE, __VALUE) \
::paddle::platform::details::SafeBoostGetMutable<__TYPE>(__VALUE, #__VALUE, \
__FILE__, __LINE__)
/** OTHER EXCEPTION AND ENFORCE **/ /** OTHER EXCEPTION AND ENFORCE **/
struct EOFException : public std::exception { struct EOFException : public std::exception {
std::string err_str_; std::string err_str_;
EOFException(const char* err_msg, const char* file, int line) { EOFException(const char* err_msg, const char* file, int line) {
err_str_ = string::Sprintf("%s at [%s:%d]", err_msg, file, line); err_str_ = paddle::string::Sprintf("%s at [%s:%d]", err_msg, file, line);
} }
const char* what() const noexcept override { return err_str_.c_str(); } const char* what() const noexcept override { return err_str_.c_str(); }
}; };
#define PADDLE_THROW_EOF() \ #define PADDLE_THROW_EOF() \
do { \
HANDLE_THE_ERROR \
throw ::paddle::platform::EOFException("There is no next data.", __FILE__, \
__LINE__); \
END_HANDLE_THE_ERROR \
} while (0)
#define PADDLE_THROW_BAD_ALLOC(...) \
do { \ do { \
HANDLE_THE_ERROR \ HANDLE_THE_ERROR \
throw ::paddle::memory::allocation::BadAlloc( \ throw paddle::platform::EOFException("There is no next data.", __FILE__, \
::paddle::platform::ErrorSummary(__VA_ARGS__).to_string(), __FILE__, \ __LINE__); \
__LINE__); \
END_HANDLE_THE_ERROR \ END_HANDLE_THE_ERROR \
} while (0) } while (0)
#define PADDLE_THROW_BAD_ALLOC(...) \
do { \
HANDLE_THE_ERROR \
throw ::paddle::memory::allocation::BadAlloc( \
pten::ErrorSummary(__VA_ARGS__).to_string(), __FILE__, __LINE__); \
END_HANDLE_THE_ERROR \
} while (0)
/**************************************************************************/ /**************************************************************************/
/**************************** NVIDIA ERROR ********************************/ /**************************** NVIDIA ERROR ********************************/
#ifdef PADDLE_WITH_CUDA #ifdef PADDLE_WITH_CUDA
...@@ -970,7 +498,7 @@ inline std::string build_nvidia_error_msg(ncclResult_t nccl_result) { ...@@ -970,7 +498,7 @@ inline std::string build_nvidia_error_msg(ncclResult_t nccl_result) {
::paddle::platform::details::ExternalApiType< \ ::paddle::platform::details::ExternalApiType< \
__CUDA_STATUS_TYPE__>::kSuccess; \ __CUDA_STATUS_TYPE__>::kSuccess; \
if (UNLIKELY(__cond__ != __success_type__)) { \ if (UNLIKELY(__cond__ != __success_type__)) { \
auto __summary__ = ::paddle::platform::errors::External( \ auto __summary__ = pten::errors::External( \
::paddle::platform::build_nvidia_error_msg(__cond__)); \ ::paddle::platform::build_nvidia_error_msg(__cond__)); \
__THROW_ERROR_INTERNAL__(__summary__); \ __THROW_ERROR_INTERNAL__(__summary__); \
} \ } \
...@@ -1016,7 +544,7 @@ inline void retry_sleep(unsigned milliseconds) { ...@@ -1016,7 +544,7 @@ inline void retry_sleep(unsigned milliseconds) {
++retry_count; \ ++retry_count; \
} \ } \
if (UNLIKELY(__cond__ != __success_type__)) { \ if (UNLIKELY(__cond__ != __success_type__)) { \
auto __summary__ = ::paddle::platform::errors::External( \ auto __summary__ = pten::errors::External( \
::paddle::platform::build_nvidia_error_msg(__cond__)); \ ::paddle::platform::build_nvidia_error_msg(__cond__)); \
__THROW_ERROR_INTERNAL__(__summary__); \ __THROW_ERROR_INTERNAL__(__summary__); \
} \ } \
...@@ -1176,7 +704,7 @@ DEFINE_EXTERNAL_API_TYPE(ncclResult_t, ncclSuccess); ...@@ -1176,7 +704,7 @@ DEFINE_EXTERNAL_API_TYPE(ncclResult_t, ncclSuccess);
::paddle::platform::details::ExternalApiType< \ ::paddle::platform::details::ExternalApiType< \
__CUDA_STATUS_TYPE__>::kSuccess; \ __CUDA_STATUS_TYPE__>::kSuccess; \
if (UNLIKELY(__cond__ != __success_type__)) { \ if (UNLIKELY(__cond__ != __success_type__)) { \
auto __summary__ = ::paddle::platform::errors::External( \ auto __summary__ = pten::errors::External( \
::paddle::platform::build_rocm_error_msg(__cond__)); \ ::paddle::platform::build_rocm_error_msg(__cond__)); \
__THROW_ERROR_INTERNAL__(__summary__); \ __THROW_ERROR_INTERNAL__(__summary__); \
} \ } \
...@@ -1204,7 +732,7 @@ inline void retry_sleep(unsigned millisecond) { ...@@ -1204,7 +732,7 @@ inline void retry_sleep(unsigned millisecond) {
++retry_count; \ ++retry_count; \
} \ } \
if (UNLIKELY(__cond__ != __success_type__)) { \ if (UNLIKELY(__cond__ != __success_type__)) { \
auto __summary__ = ::paddle::platform::errors::External( \ auto __summary__ = pten::errors::External( \
::paddle::platform::build_rocm_error_msg(__cond__)); \ ::paddle::platform::build_rocm_error_msg(__cond__)); \
__THROW_ERROR_INTERNAL__(__summary__); \ __THROW_ERROR_INTERNAL__(__summary__); \
} \ } \
......
paddle/fluid/platform/errors.h
浏览文件 @ 6ca49164
...@@ -13,76 +13,10 @@ See the License for the specific language governing permissions and ...@@ -13,76 +13,10 @@ See the License for the specific language governing permissions and
limitations under the License. */ limitations under the License. */
#pragma once #pragma once
#include "paddle/pten/core/errors.h"
#include <memory>
#include <stdexcept>
#include <string>
#include <tuple>
#include <type_traits>
#include "paddle/fluid/platform/error_codes.pb.h"
#include "paddle/fluid/string/printf.h"
namespace paddle { namespace paddle {
namespace platform { namespace platform {
namespace errors = ::pten::errors;
typedef ::paddle::platform::error::Code Code; using error = ::pten::ErrorCode;
}
class ErrorSummary { }
public:
// Note(chenweihang): Final deprecated constructor
// This constructor is used to be compatible with
// current existing untyped PADDLE_ENFORCE_*
// PADDLE_ENFORCE
// Note(chenweihang): Windows openblas need this
// constructor for compiling PADDLE_ENFORCE in *.cu,
// this is a bug cause we can't remove this
// constructor now.
template <typename... Args>
explicit ErrorSummary(Args... args) {
code_ = paddle::platform::error::LEGACY;
msg_ = paddle::string::Sprintf(args...);
}
// Note(chenweihang): Only recommended constructor
// No longer supports PADDLE_ENFORCE without type or without error message
explicit ErrorSummary(Code code, std::string msg) : code_(code), msg_(msg) {}
Code code() const { return code_; }
const std::string& error_message() const { return msg_; }
std::string to_string() const;
private:
Code code_;
std::string msg_;
};
namespace errors {
#define REGISTER_ERROR(FUNC, CONST, ...) \
template <typename... Args> \
::paddle::platform::ErrorSummary FUNC(Args... args) { \
return ::paddle::platform::ErrorSummary( \
::paddle::platform::error::CONST, ::paddle::string::Sprintf(args...)); \
}
REGISTER_ERROR(InvalidArgument, INVALID_ARGUMENT)
REGISTER_ERROR(NotFound, NOT_FOUND)
REGISTER_ERROR(OutOfRange, OUT_OF_RANGE)
REGISTER_ERROR(AlreadyExists, ALREADY_EXISTS)
REGISTER_ERROR(ResourceExhausted, RESOURCE_EXHAUSTED)
REGISTER_ERROR(PreconditionNotMet, PRECONDITION_NOT_MET)
REGISTER_ERROR(PermissionDenied, PERMISSION_DENIED)
REGISTER_ERROR(ExecutionTimeout, EXECUTION_TIMEOUT)
REGISTER_ERROR(Unimplemented, UNIMPLEMENTED)
REGISTER_ERROR(Unavailable, UNAVAILABLE)
REGISTER_ERROR(Fatal, FATAL)
REGISTER_ERROR(External, EXTERNAL)
#undef REGISTER_ERROR
} // namespace errors
} // namespace platform
} // namespace paddle
paddle/fluid/string/piece.h
浏览文件 @ 6ca49164
...@@ -17,89 +17,4 @@ ...@@ -17,89 +17,4 @@
#include <ostream> #include <ostream>
#include <string> #include <string>
namespace paddle { #include "paddle/utils/string/piece.h"
namespace string {
// Piece points into a std::string object but doesn't own the
// string. It is for efficient access to strings. Like Go's string
// type. Not that Piece doesn't mutate the underlying string,
// so it is thread-safe given that the underlying string doesn't
// change. Because Piece contains a little data members, and
// its syntax is simple as it doesn't own/manage the string, it is
// cheap to construct Pieces and pass them around.
class Piece {
public:
static const size_t npos = static_cast<size_t>(-1);
// We provide non-explicit singleton constructors so users can
// pass in a "const char*" or a "string" wherever a "Piece"
// is expected. These constructors ensure that if data_ is NULL,
// size_ is 0.
Piece();
Piece(const char* d, size_t n);
Piece(const char* d); // NOLINT: accept C string into Piece.
Piece(const std::string& s); // NOLINT: accept C++ string into Piece.
const char* data() const { return data_; }
size_t len() const { return size_; }
char operator[](size_t n) const;
// Piece doesn't own the string, so both iterator and const
// iterator are const char* indeed.
typedef const char* const_iterator;
typedef const char* iterator;
iterator begin() const { return data_; }
iterator end() const { return data_ + size_; }
// Return a string that contains the copy of the referenced data.
std::string ToString() const { return std::string(data_, size_); }
private:
const char* data_;
size_t size_;
// Intentionally copyable
};
int Compare(Piece a, Piece b);
bool operator==(Piece x, Piece y);
bool operator!=(Piece x, Piece y);
bool operator<(Piece x, Piece y);
bool operator>(Piece x, Piece y);
bool operator<=(Piece x, Piece y);
bool operator>=(Piece x, Piece y);
bool HasPrefix(Piece s, Piece prefix);
bool HasSuffix(Piece s, Piece suffix);
Piece SkipPrefix(Piece s, size_t n);
Piece SkipSuffix(Piece s, size_t n);
// Skip the prefix (or suffix) if it matches with the string.
Piece TrimPrefix(Piece s, Piece prefix);
Piece TrimSuffix(Piece s, Piece suffix);
// Returns if s contains sub. Any s except for empty s contains an
// empty sub.
bool Contains(Piece s, Piece sub);
// Return the first occurrence of sub in s, or npos. If both s and
// sub is empty, it returns npos; otherwise, if only sub is empty, it
// returns 0.
size_t Index(Piece s, Piece sub);
// Return the first occurrence of c in s[pos:end], or npos.
size_t Find(Piece s, char c, size_t pos);
// Search range is [0..pos] inclusive. If pos == npos, search everything.
size_t RFind(Piece s, char c, size_t pos);
Piece SubStr(Piece s, size_t pos, size_t n);
// allow Piece to be logged
std::ostream& operator<<(std::ostream& o, Piece piece);
} // namespace string
} // namespace paddle
paddle/fluid/string/pretty_log.h
浏览文件 @ 6ca49164
...@@ -19,70 +19,4 @@ ...@@ -19,70 +19,4 @@
#include <utility> #include <utility>
#include "gflags/gflags.h" #include "gflags/gflags.h"
#include "paddle/fluid/string/printf.h" #include "paddle/utils/string/pretty_log.h"
DECLARE_bool(color);
namespace paddle {
namespace string {
inline std::string black() { return FLAGS_color ? "\e[30m" : ""; }
inline std::string red() { return FLAGS_color ? "\e[31m" : ""; }
inline std::string b_red() { return FLAGS_color ? "\e[41m" : ""; }
inline std::string green() { return FLAGS_color ? "\e[32m" : ""; }
inline std::string yellow() { return FLAGS_color ? "\e[33m" : ""; }
inline std::string blue() { return FLAGS_color ? "\e[34m" : ""; }
inline std::string purple() { return FLAGS_color ? "\e[35m" : ""; }
inline std::string cyan() { return FLAGS_color ? "\e[36m" : ""; }
inline std::string light_gray() { return FLAGS_color ? "\e[37m" : ""; }
inline std::string white() { return FLAGS_color ? "\e[37m" : ""; }
inline std::string light_red() { return FLAGS_color ? "\e[91m" : ""; }
inline std::string dim() { return FLAGS_color ? "\e[2m" : ""; }
inline std::string bold() { return FLAGS_color ? "\e[1m" : ""; }
inline std::string underline() { return FLAGS_color ? "\e[4m" : ""; }
inline std::string blink() { return FLAGS_color ? "\e[5m" : ""; }
inline std::string reset() { return FLAGS_color ? "\e[0m" : ""; }
using TextBlock = std::pair<std::string, std::string>;
struct Style {
static std::string info() { return black(); }
static std::string warn() { return b_red(); }
static std::string suc() { return green(); }
static std::string H1() { return bold() + purple(); }
static std::string H2() { return green(); }
static std::string H3() { return green(); }
static std::string detail() { return light_gray(); }
};
template <typename... Args>
static void PrettyLogEndl(const std::string &style, const char *fmt,
const Args &... args) {
std::cerr << style << Sprintf(fmt, args...) << reset() << std::endl;
}
template <typename... Args>
static void PrettyLog(const std::string &style, const char *fmt,
const Args &... args) {
std::cerr << style << Sprintf(fmt, args...) << reset();
}
template <typename... Args>
static void PrettyLogInfo(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::info(), fmt, args...);
}
template <typename... Args>
static void PrettyLogDetail(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::detail(), fmt, args...);
}
template <typename... Args>
static void PrettyLogH1(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::H1(), fmt, args...);
}
template <typename... Args>
static void PrettyLogH2(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::H2(), fmt, args...);
}
} // namespace string
} // namespace paddle
paddle/fluid/string/printf.h
浏览文件 @ 6ca49164
...@@ -12,113 +12,5 @@ ...@@ -12,113 +12,5 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
// Compared with std::stringstream, there are primary purpose of
// string::Printf:
//
// 1. Type-safe printing, with why and how explained in
// http://www.drdobbs.com/stringprintf-a-typesafe-printf-family-fo/184401999.
// Implementation includes
//
// https://github.com/c42f/tinyformat
// boost::format
// std::stringstream
//
// std::stringstream is not convenient enough in many cases. For example:
//
// std::cout << std::setprecision(2) << std::fixed << 1.23456 << "\n";
//
// boost::format is the most convenient one. We can have
//
// std::cout << format("%2% %1%") % 36 % 77;
//
// or
//
// format fmter("%2% %1%");
// fmter % 36; fmter % 77;
// std::cout << fmter.c_str();
//
// But the overloading of % might be overkilling and it would be
// more efficient if it can write to std::cout directly.
//
// tinyformat has an interface compatible with the C-printf style,
// and it can writes to a stream or returns a std::string:
//
// std::cout << tfm::printf(
// "%s, %s %d, %.2d:%.2d\n",
// weekday, month, day, hour, min);
//
// or
//
// tfm::format(std::cout,
// "%s, %s %d, %.2d:%.2d\n",
// weekday, month, day, hour, min);
//
// 2. High-performance -- most printed strings are not too long and
// doens't need dynamic memory allocation. Many StringPrintf
// implementations doesn't enforce type-safe, but are
// high-performance, including
//
// https://developers.google.com/optimization/reference/base/stringprintf/
// https://github.com/adobe/chromium/blob/master/base/stringprintf.h
// https://github.com/google/protobuf/blob/master/src/google/protobuf/stubs/stringprintf.h
//
// According to
// https://github.com/c42f/tinyformat#compile-time-and-code-bloat,
// boost::format runs too slow and results in large executable binary
// files. So here we port tinyformat.
#pragma once #pragma once
#include "paddle/utils/string/printf.h"
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "tinyformat/tinyformat.h" // https://github.com/c42f/tinyformat
namespace paddle {
namespace string {
template <typename... Args>
void Fprintf(std::ostream& out, const char* fmt, const Args&... args) {
tinyformat::vformat(out, fmt, tinyformat::makeFormatList(args...));
}
inline std::string Sprintf() { return ""; }
template <typename... Args>
std::string Sprintf(const Args&... args) {
std::ostringstream oss;
Fprintf(oss, "%s", args...);
return oss.str();
}
template <typename... Args>
std::string Sprintf(const char* fmt, const Args&... args) {
std::ostringstream oss;
Fprintf(oss, fmt, args...);
return oss.str();
}
template <typename... Args>
void Printf(const char* fmt, const Args&... args) {
Fprintf(std::cout, fmt, args...);
}
inline std::string HumanReadableSize(double f_size) {
size_t i = 0;
double orig = f_size;
const std::vector<std::string> units(
{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"});
while (f_size >= 1024) {
f_size /= 1024;
i++;
}
if (i >= units.size()) {
return Sprintf("%fB", orig);
}
return Sprintf("%f%s", f_size, units[i]);
}
} // namespace string
} // namespace paddle
paddle/fluid/string/split.h
浏览文件 @ 6ca49164
...@@ -17,21 +17,4 @@ limitations under the License. */ ...@@ -17,21 +17,4 @@ limitations under the License. */
#include <string> #include <string>
#include <vector> #include <vector>
namespace paddle { #include "paddle/utils/string/split.h"
namespace string {
static inline std::vector<std::string> Split(std::string const& original,
char separator) {
std::vector<std::string> results;
std::string token;
std::istringstream is(original);
while (std::getline(is, token, separator)) {
if (!token.empty()) {
results.push_back(token);
}
}
return results;
}
} // namespace string
} // namespace paddle
paddle/fluid/string/string_helper.h
浏览文件 @ 6ca49164
...@@ -14,219 +14,4 @@ ...@@ -14,219 +14,4 @@
#pragma once #pragma once
#include <ctype.h> #include "paddle/utils/string/string_helper.h"
#include <stdio.h>
#include <cstring>
#include <string>
#include <utility>
#include <vector>
#include "glog/logging.h"
namespace paddle {
namespace string {
inline size_t count_spaces(const char* s) {
size_t count = 0;
while (*s != 0 && isspace(*s++)) {
count++;
}
return count;
}
inline size_t count_nonspaces(const char* s) {
size_t count = 0;
while (*s != 0 && !isspace(*s++)) {
count++;
}
return count;
}
template <class... ARGS>
void format_string_append(std::string& str, const char* fmt, // NOLINT
ARGS&&... args) {
int len = snprintf(NULL, 0, fmt, args...);
CHECK_GE(len, 0);
size_t oldlen = str.length();
str.resize(oldlen + len + 1);
CHECK(snprintf(&str[oldlen], (size_t)len + 1, fmt, args...) == // NOLINT
len);
str.resize(oldlen + len);
}
template <class... ARGS>
void format_string_append(std::string& str, const std::string& fmt, // NOLINT
ARGS&&... args) {
format_string_append(str, fmt.c_str(), args...);
}
template <class... ARGS>
std::string format_string(const char* fmt, ARGS&&... args) {
std::string str;
format_string_append(str, fmt, args...);
return str;
}
template <class... ARGS>
std::string format_string(const std::string& fmt, ARGS&&... args) {
return format_string(fmt.c_str(), args...);
}
// remove leading and tailing spaces
std::string trim_spaces(const std::string& str);
// erase all spaces in str
std::string erase_spaces(const std::string& str);
inline int str_to_float(const char* str, float* v) {
const char* head = str;
char* cursor = NULL;
int index = 0;
while (*(head += count_spaces(head)) != 0) {
v[index++] = std::strtof(head, &cursor);
if (head == cursor) {
break;
}
head = cursor;
}
return index;
}
// checks whether the test string is a suffix of the input string.
bool ends_with(std::string const& input, std::string const& test);
// split string by delim
template <class T = std::string>
std::vector<T> split_string(const std::string& str, const std::string& delim) {
size_t pre_pos = 0;
size_t pos = 0;
std::string tmp_str;
std::vector<T> res_list;
res_list.clear();
if (str.empty()) {
return res_list;
}
while ((pos = str.find(delim, pre_pos)) != std::string::npos) {
tmp_str.assign(str, pre_pos, pos - pre_pos);
res_list.push_back(tmp_str);
pre_pos = pos + 1;
}
tmp_str.assign(str, pre_pos, str.length() - pre_pos);
if (!tmp_str.empty()) {
res_list.push_back(tmp_str);
}
return res_list;
}
// split string by spaces. Leading and tailing spaces are ignored. Consecutive
// spaces are treated as one delim.
template <class T = std::string>
std::vector<T> split_string(const std::string& str) {
std::vector<T> list;
const char* p;
int pre_pos = 0;
int pos = 0;
std::string tmp_str;
if (str.empty()) {
return list;
}
for (p = str.c_str(); *p != 0;) {
if (!isspace(*p)) {
pos = pre_pos;
p++;
while (*p != 0 && !isspace(*p)) {
pos++;
p++;
}
tmp_str.assign(str, pre_pos, pos - pre_pos + 1);
list.push_back(tmp_str);
pre_pos = pos + 1;
} else {
pre_pos++;
p++;
}
}
return list;
}
template <class Container>
std::string join_strings(const Container& strs, char delim) {
std::string str;
size_t i = 0;
for (auto& elem : strs) {
if (i > 0) {
str += delim;
}
std::stringstream ss;
ss << elem;
str += ss.str();
++i;
}
return str;
}
template <class Container>
std::string join_strings(const Container& strs, const std::string& delim) {
std::string str;
size_t i = 0;
for (auto& elem : strs) {
if (i > 0) {
str += delim;
}
std::stringstream ss;
ss << elem;
str += ss.str();
++i;
}
return str;
}
template <class Container, class DelimT, class ConvertFunc>
std::string join_strings(const Container& strs, DelimT&& delim,
ConvertFunc&& func) {
std::stringstream ss;
size_t i = 0;
for (const auto& elem : strs) {
if (i > 0) {
ss << delim;
}
ss << func(elem);
++i;
}
return ss.str();
}
// A helper class for reading lines from file. A line buffer is maintained. It
// doesn't need to know the maximum possible length of a line.
class LineFileReader {
public:
LineFileReader() {}
LineFileReader(LineFileReader&&) = delete;
LineFileReader(const LineFileReader&) = delete;
~LineFileReader() { ::free(_buffer); }
char* getline(FILE* f) { return this->getdelim(f, '\n'); }
char* getdelim(FILE* f, char delim);
char* get() { return _buffer; }
size_t length() { return _length; }
private:
char* _buffer = NULL;
size_t _buf_size = 0;
size_t _length = 0;
};
} // end namespace string
} // end namespace paddle
paddle/fluid/string/to_string.h
浏览文件 @ 6ca49164
...@@ -13,48 +13,4 @@ See the License for the specific language governing permissions and ...@@ -13,48 +13,4 @@ See the License for the specific language governing permissions and
limitations under the License. */ limitations under the License. */
#pragma once #pragma once
#include <sstream> #include "paddle/utils/string/to_string.h"
#include <string>
#include <type_traits>
#include <typeindex>
#include <vector>
namespace paddle {
namespace string {
inline std::ostream& operator<<(std::ostream& s, const std::type_index& t) {
s << t.name();
return s;
}
template <typename T,
typename std::enable_if<!std::is_enum<T>::value, int>::type = 0>
inline std::string to_string(T v) {
std::ostringstream sout;
sout << v;
return sout.str();
}
template <typename T,
typename std::enable_if<std::is_enum<T>::value, int>::type = 0>
inline std::string to_string(T v) {
return std::to_string(static_cast<int>(v));
}
template <>
inline std::string to_string(std::type_index t) {
return t.name();
}
// Faster std::string/const char* type
template <>
inline std::string to_string(std::string v) {
return v;
}
template <>
inline std::string to_string(const char* v) {
return std::string(v);
}
} // namespace string
} // namespace paddle
paddle/pten/api/lib/CMakeLists.txt
浏览文件 @ 6ca49164
...@@ -3,11 +3,11 @@ add_subdirectory(utils) ...@@ -3,11 +3,11 @@ add_subdirectory(utils)
cc_library(ext_compat_utils SRCS ext_compat_utils.cc DEPS place) cc_library(ext_compat_utils SRCS ext_compat_utils.cc DEPS place)
if (WITH_GPU) if (WITH_GPU)
nv_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils enforce) nv_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils pten_enforce)
elseif (WITH_ROCM) elseif (WITH_ROCM)
hip_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils enforce) hip_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils pten_enforce)
else() else()
cc_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils enforce) cc_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils pten_enforce)
endif() endif()
cc_library(kernel_dispatch SRCS kernel_dispatch.cc DEPS pten_tensor pten_context kernel_factory) cc_library(kernel_dispatch SRCS kernel_dispatch.cc DEPS pten_tensor pten_context kernel_factory)
......
paddle/pten/api/lib/op_meta_info.cc
浏览文件 @ 6ca49164
...@@ -19,7 +19,7 @@ limitations under the License. */ ...@@ -19,7 +19,7 @@ limitations under the License. */
#include <vector> #include <vector>
#include "paddle/fluid/framework/custom_operator.h" #include "paddle/fluid/framework/custom_operator.h"
#include "paddle/fluid/platform/enforce.h" #include "paddle/pten/core/enforce.h"
namespace paddle { namespace paddle {
...@@ -74,7 +74,7 @@ OpMetaInfoBuilder::OpMetaInfoBuilder(std::string&& name, size_t index) { ...@@ -74,7 +74,7 @@ OpMetaInfoBuilder::OpMetaInfoBuilder(std::string&& name, size_t index) {
PADDLE_ENFORCE_EQ( PADDLE_ENFORCE_EQ(
info_vector.size(), info_vector.size(),
index_, index_,
platform::errors::PreconditionNotMet( pten::errors::PreconditionNotMet(
"The operator %s's meta info register failed. " "The operator %s's meta info register failed. "
"Please make sure you call marcos as order `PD_BUILD_OP`, " "Please make sure you call marcos as order `PD_BUILD_OP`, "
"`PD_BUILD_GRAD_OP`, `PD_BUILD_DOUBLE_GRAD_OP`.", "`PD_BUILD_GRAD_OP`, `PD_BUILD_DOUBLE_GRAD_OP`.",
...@@ -88,7 +88,7 @@ OpMetaInfoBuilder::OpMetaInfoBuilder(std::string&& name, size_t index) { ...@@ -88,7 +88,7 @@ OpMetaInfoBuilder::OpMetaInfoBuilder(std::string&& name, size_t index) {
case 2: case 2:
name_ = name_ + "_grad_grad"; name_ = name_ + "_grad_grad";
default: default:
PADDLE_THROW(platform::errors::InvalidArgument( PADDLE_THROW(pten::errors::InvalidArgument(
"Not support index `%d` when construct OpMetaInfoBuilder, " "Not support index `%d` when construct OpMetaInfoBuilder, "
"now only support `0, 1, 2`.", "now only support `0, 1, 2`.",
index_)); index_));
...@@ -130,7 +130,7 @@ OpMetaInfoBuilder& OpMetaInfoBuilder::SetInferDtypeFn(InferDtypeFunc func) { ...@@ -130,7 +130,7 @@ OpMetaInfoBuilder& OpMetaInfoBuilder::SetInferDtypeFn(InferDtypeFunc func) {
PADDLE_ENFORCE_EQ( PADDLE_ENFORCE_EQ(
index_, index_,
0UL, 0UL,
platform::errors::Unimplemented( pten::errors::Unimplemented(
"Currently, the InferDtypeFn setting of Grad Op is not supported, " "Currently, the InferDtypeFn setting of Grad Op is not supported, "
"And backward Tensor `X@GRAD` will use the dtype of forward Tensor " "And backward Tensor `X@GRAD` will use the dtype of forward Tensor "
"`X` by default.")); "`X` by default."));
......
paddle/pten/api/lib/tensor.cc
浏览文件 @ 6ca49164
...@@ -48,12 +48,12 @@ limitations under the License. */ ...@@ -48,12 +48,12 @@ limitations under the License. */
* or the corresponding components will be re-implemented. * or the corresponding components will be re-implemented.
*/ */
#include "paddle/fluid/memory/memory.h" #include "paddle/fluid/memory/memory.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/fluid/platform/place.h" #include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/stream/cuda_stream.h" #include "paddle/fluid/platform/stream/cuda_stream.h"
#include "paddle/pten/common/complex.h" #include "paddle/pten/common/complex.h"
#include "paddle/pten/common/float16.h" #include "paddle/pten/common/float16.h"
#include "paddle/pten/core/ddim.h" #include "paddle/pten/core/ddim.h"
#include "paddle/pten/core/enforce.h"
namespace paddle { namespace paddle {
namespace experimental { namespace experimental {
...@@ -68,7 +68,7 @@ Tensor cast(const Tensor &x, DataType out_dtype); ...@@ -68,7 +68,7 @@ Tensor cast(const Tensor &x, DataType out_dtype);
Tensor::Tensor(std::shared_ptr<pten::TensorBase> tensor_impl) Tensor::Tensor(std::shared_ptr<pten::TensorBase> tensor_impl)
: impl_(std::move(tensor_impl)) { : impl_(std::move(tensor_impl)) {
PADDLE_ENFORCE_NOT_NULL(impl_, PADDLE_ENFORCE_NOT_NULL(impl_,
platform::errors::InvalidArgument( pten::errors::InvalidArgument(
"TensorImpl with nullptr is not supported")); "TensorImpl with nullptr is not supported"));
} }
...@@ -113,7 +113,7 @@ void Tensor::reshape(const std::vector<int64_t> &shape) { ...@@ -113,7 +113,7 @@ void Tensor::reshape(const std::vector<int64_t> &shape) {
std::dynamic_pointer_cast<pten::DenseTensor>(impl_)->set_meta( std::dynamic_pointer_cast<pten::DenseTensor>(impl_)->set_meta(
pten::DenseTensorMeta(dtype(), framework::make_ddim(shape))); pten::DenseTensorMeta(dtype(), framework::make_ddim(shape)));
} else { } else {
PADDLE_THROW(platform::errors::Unimplemented( PADDLE_THROW(pten::errors::Unimplemented(
"Only support reshape operation on DenseTensor now.")); "Only support reshape operation on DenseTensor now."));
} }
} }
...@@ -177,8 +177,8 @@ T *Tensor::mutable_data(const PlaceType &place) { ...@@ -177,8 +177,8 @@ T *Tensor::mutable_data(const PlaceType &place) {
PADDLE_ENFORCE_EQ( PADDLE_ENFORCE_EQ(
platform::is_same_place(inner_place, impl_->place()), platform::is_same_place(inner_place, impl_->place()),
true, true,
platform::errors::Unimplemented("Modification of tensor place through " pten::errors::Unimplemented("Modification of tensor place through "
"mutable_data is not supported now")); "mutable_data is not supported now"));
} }
if (is_dense_tensor()) { if (is_dense_tensor()) {
return std::dynamic_pointer_cast<pten::DenseTensor>(impl_)->mutable_data<T>( return std::dynamic_pointer_cast<pten::DenseTensor>(impl_)->mutable_data<T>(
...@@ -236,7 +236,7 @@ Tensor::data<paddle::platform::bfloat16>() const; ...@@ -236,7 +236,7 @@ Tensor::data<paddle::platform::bfloat16>() const;
template <typename T> template <typename T>
T *Tensor::data() { T *Tensor::data() {
PADDLE_THROW(platform::errors::Unimplemented( PADDLE_THROW(pten::errors::Unimplemented(
"It is not currently supported to directly obtain the modifiable data " "It is not currently supported to directly obtain the modifiable data "
"address through the tensor::data<T>() method, please use the " "address through the tensor::data<T>() method, please use the "
"tensor::mutable_data<T>() method.")); "tensor::mutable_data<T>() method."));
...@@ -267,7 +267,7 @@ Tensor Tensor::slice(int64_t begin_idx, int64_t end_idx) const { ...@@ -267,7 +267,7 @@ Tensor Tensor::slice(int64_t begin_idx, int64_t end_idx) const {
begin_idx, begin_idx,
end_idx)))); end_idx))));
} else { } else {
PADDLE_THROW(platform::errors::Unimplemented( PADDLE_THROW(pten::errors::Unimplemented(
"Only support slice operation on DenseTensor now.")); "Only support slice operation on DenseTensor now."));
} }
} }
......
paddle/pten/core/CMakeLists.txt
浏览文件 @ 6ca49164
...@@ -9,12 +9,16 @@ else() ...@@ -9,12 +9,16 @@ else()
cc_library(convert_utils SRCS convert_utils.cc DEPS data_type place) cc_library(convert_utils SRCS convert_utils.cc DEPS data_type place)
endif() endif()
cc_library(kernel_factory SRCS kernel_factory.cc DEPS enforce convert_utils) cc_library(errors SRCS errors.cc)
cc_library(kernel_context SRCS kernel_context.cc DEPS enforce pten_context) set(pten_enforce_deps errors flags)
cc_library(pten_enforce INTERFACE SRCS enforce.cc DEPS ${pten_enforce_deps})
cc_library(tensor_base SRCS tensor_base.cc allocator.cc storage.cc DEPS enforce) cc_library(kernel_factory SRCS kernel_factory.cc DEPS pten_enforce convert_utils)
cc_library(tensor_meta SRCS tensor_meta.cc DEPS enforce mixed_vector) cc_library(kernel_context SRCS kernel_context.cc DEPS pten_enforce pten_context)
cc_library(lod_utils SRCS lod_utils.cc DEPS enforce mixed_vector)
cc_library(tensor_base SRCS tensor_base.cc allocator.cc storage.cc DEPS pten_enforce)
cc_library(tensor_meta SRCS tensor_meta.cc DEPS pten_enforce mixed_vector)
cc_library(lod_utils SRCS lod_utils.cc DEPS pten_enforce mixed_vector)
cc_library(dense_tensor SRCS dense_tensor.cc DEPS convert_utils tensor_meta tensor_base) cc_library(dense_tensor SRCS dense_tensor.cc DEPS convert_utils tensor_meta tensor_base)
cc_library(pten_device_context SRCS device_context.cc DEPS tensor_base ) cc_library(pten_device_context SRCS device_context.cc DEPS tensor_base )
...@@ -25,7 +29,6 @@ cc_test(unroll_array_ops_test SRCS unroll_array_ops_test.cc) ...@@ -25,7 +29,6 @@ cc_test(unroll_array_ops_test SRCS unroll_array_ops_test.cc)
cc_library(ddim SRCS ddim.cc DEPS eigen3 boost enforce) cc_library(ddim SRCS ddim.cc DEPS eigen3 boost enforce)
cc_library(selected_rows SRCS selected_rows.cc DEPS dense_tensor mixed_vector enforce ddim) cc_library(selected_rows SRCS selected_rows.cc DEPS dense_tensor mixed_vector enforce ddim)
# Will remove once we implemented MKLDNN_Tensor # Will remove once we implemented MKLDNN_Tensor
if(WITH_MKLDNN) if(WITH_MKLDNN)
add_dependencies(dense_tensor mkldnn) add_dependencies(dense_tensor mkldnn)
......
paddle/pten/core/compat/CMakeLists.txt
浏览文件 @ 6ca49164
cc_library(arg_map_context SRCS arg_map_context.cc DEPS enforce) cc_library(arg_map_context SRCS arg_map_context.cc DEPS pten_enforce)
paddle/pten/core/compat/arg_map_context.cc
浏览文件 @ 6ca49164
...@@ -15,9 +15,9 @@ limitations under the License. */ ...@@ -15,9 +15,9 @@ limitations under the License. */
#include "paddle/pten/core/compat/arg_map_context.h" #include "paddle/pten/core/compat/arg_map_context.h"
#include "paddle/fluid/string/string_helper.h" #include "paddle/fluid/string/string_helper.h"
#include "paddle/pten/core/enforce.h"
namespace pten { namespace pten {
std::ostream& operator<<(std::ostream& os, KernelSignature signature) { std::ostream& operator<<(std::ostream& os, KernelSignature signature) {
os << "Kernel Signature - name: " << signature.name << "; inputs: " os << "Kernel Signature - name: " << signature.name << "; inputs: "
<< paddle::string::join_strings(std::get<0>(signature.args), ", ") << paddle::string::join_strings(std::get<0>(signature.args), ", ")
......
paddle/pten/core/enforce.cc 0 → 100644
浏览文件 @ 6ca49164
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/pten/core/enforce.h"
paddle/pten/core/enforce.h 0 → 100644
浏览文件 @ 6ca49164
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#ifdef __GNUC__
#include <cxxabi.h> // for __cxa_demangle
#endif // __GNUC__
#if !defined(_WIN32)
#include <dlfcn.h> // dladdr
#include <unistd.h> // sleep, usleep
#else // _WIN32
#ifndef NOMINMAX
#define NOMINMAX // msvc max/min macro conflict with std::min/max
#endif
#include <windows.h> // GetModuleFileName, Sleep
#endif
#include <fstream>
#include <iomanip>
#include <memory>
#include <sstream>
#include <stdexcept>
#include <string>
#include <type_traits>
#include <utility>
#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL)
#include <execinfo.h>
#endif
#define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h
#include "gflags/gflags.h"
#include "glog/logging.h"
#include "paddle/pten/core/errors.h"
#include "paddle/utils/string/printf.h"
#include "paddle/utils/string/to_string.h"
// Note: these headers for simplify demangle type string
#include "paddle/pten/core/type_defs.h"
namespace pten {
class ErrorSummary;
} // namespace pten
DECLARE_int32(call_stack_level);
namespace pten {
namespace enforce {
/** HELPER MACROS AND FUNCTIONS **/
#ifndef PADDLE_MAY_THROW
#define PADDLE_MAY_THROW noexcept(false)
#endif
// Because most enforce conditions would evaluate to true, we can use
// __builtin_expect to instruct the C++ compiler to generate code that
// always forces branch prediction of true.
// This generates faster binary code. __builtin_expect is since C++11.
// For more details, please check https://stackoverflow.com/a/43870188/724872.
#if !defined(_WIN32)
#define UNLIKELY(condition) __builtin_expect(static_cast<bool>(condition), 0)
#else
// there is no equivalent intrinsics in msvc.
#define UNLIKELY(condition) (condition)
#endif
#if !defined(_WIN32)
#define LIKELY(condition) __builtin_expect(static_cast<bool>(condition), 1)
#else
// there is no equivalent intrinsics in msvc.
#define LIKELY(condition) (condition)
#endif
#if defined _WIN32 && defined PADDLE_ON_INFERENCE && defined PADDLE_NO_PYTHON
#define HANDLE_THE_ERROR try {
#define END_HANDLE_THE_ERROR \
} \
catch (const std::exception& e) { \
std::cout << e.what() << std::endl; \
throw; \
}
#else
#define HANDLE_THE_ERROR
#define END_HANDLE_THE_ERROR
#endif
#ifdef __GNUC__
inline std::string demangle(std::string name) {
int status = -4; // some arbitrary value to eliminate the compiler warning
std::unique_ptr<char, void (*)(void*)> res{
abi::__cxa_demangle(name.c_str(), NULL, NULL, &status), std::free};
return (status == 0) ? res.get() : name;
}
#else
inline std::string demangle(std::string name) { return name; }
#endif
namespace details {
template <typename T>
inline constexpr bool IsArithmetic() {
return std::is_arithmetic<T>::value;
}
template <typename T1, typename T2, bool kIsArithmetic /* = true */>
struct TypeConverterImpl {
using Type1 = typename std::common_type<T1, T2>::type;
using Type2 = Type1;
};
template <typename T1, typename T2>
struct TypeConverterImpl<T1, T2, false> {
using Type1 = T1;
using Type2 = T2;
};
template <typename T1, typename T2>
struct TypeConverter {
static constexpr bool kIsArithmetic =
IsArithmetic<T1>() && IsArithmetic<T2>();
using Type1 = typename TypeConverterImpl<T1, T2, kIsArithmetic>::Type1;
using Type2 = typename TypeConverterImpl<T1, T2, kIsArithmetic>::Type2;
};
template <typename T1, typename T2>
using CommonType1 = typename std::add_lvalue_reference<
typename std::add_const<typename TypeConverter<T1, T2>::Type1>::type>::type;
template <typename T1, typename T2>
using CommonType2 = typename std::add_lvalue_reference<
typename std::add_const<typename TypeConverter<T1, T2>::Type2>::type>::type;
// Here, we use SFINAE to check whether T can be converted to std::string
template <typename T>
struct CanToString {
private:
using YesType = uint8_t;
using NoType = uint16_t;
template <typename U>
static YesType Check(decltype(std::cout << std::declval<U>())) {
return 0;
}
template <typename U>
static NoType Check(...) {
return 0;
}
public:
static constexpr bool kValue =
std::is_same<YesType, decltype(Check<T>(std::cout))>::value;
};
template <bool kCanToString /* = true */>
struct BinaryCompareMessageConverter {
template <typename T>
static std::string Convert(const char* expression, const T& value) {
return expression + std::string(":") + paddle::string::to_string(value);
}
};
template <>
struct BinaryCompareMessageConverter<false> {
template <typename T>
static const char* Convert(const char* expression, const T& value) {
return expression;
}
};
} // namespace details
template <typename T>
inline std::string ReplaceComplexTypeStr(std::string str,
const std::string& type_name) {
auto demangle_type_str = demangle(typeid(T).name());
size_t start_pos = 0;
while ((start_pos = str.find(demangle_type_str, start_pos)) !=
std::string::npos) {
str.replace(start_pos, demangle_type_str.length(), type_name);
start_pos += type_name.length();
}
return str;
}
#define __REPLACE_COMPLEX_TYPE_STR__(__TYPENAME, __STR) \
do { \
__STR = \
pten::enforce::ReplaceComplexTypeStr<__TYPENAME>(__STR, #__TYPENAME); \
} while (0)
inline std::string SimplifyDemangleStr(std::string str) {
// the older is important, you have to put complex types in front
__REPLACE_COMPLEX_TYPE_STR__(paddle::framework::AttributeMap, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::framework::Attribute, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameVariableWrapperMap, str);
__REPLACE_COMPLEX_TYPE_STR__(paddle::imperative::NameVarBaseMap, str);
__REPLACE_COMPLEX_TYPE_STR__(std::string, str);
return str;
}
inline std::string GetCurrentTraceBackString(bool for_signal = false) {
std::ostringstream sout;
if (!for_signal) {
sout << "\n\n--------------------------------------\n";
sout << "C++ Traceback (most recent call last):";
sout << "\n--------------------------------------\n";
}
#if !defined(_WIN32) && !defined(PADDLE_WITH_MUSL)
static constexpr int TRACE_STACK_LIMIT = 100;
void* call_stack[TRACE_STACK_LIMIT];
auto size = backtrace(call_stack, TRACE_STACK_LIMIT);
auto symbols = backtrace_symbols(call_stack, size);
Dl_info info;
int idx = 0;
// `for_signal` used to remove the stack trace introduced by
// obtaining the error stack trace when the signal error occurred,
// that is not related to the signal error self, remove it to
// avoid misleading users and developers
int end_idx = for_signal ? 2 : 0;
for (int i = size - 1; i >= end_idx; --i) {
if (dladdr(call_stack[i], &info) && info.dli_sname) {
auto demangled = demangle(info.dli_sname);
std::string path(info.dli_fname);
// C++ traceback info are from core.so
if (path.substr(path.length() - 3).compare(".so") == 0) {
sout << paddle::string::Sprintf(
"%-3d %s\n", idx++, SimplifyDemangleStr(demangled));
}
}
}
free(symbols);
#else
sout << "Not support stack backtrace yet.\n";
#endif
return sout.str();
}
template <typename StrType>
inline std::string GetErrorSumaryString(StrType&& what,
const char* file,
int line) {
std::ostringstream sout;
if (FLAGS_call_stack_level > 1) {
sout << "\n----------------------\nError Message "
"Summary:\n----------------------\n";
}
sout << paddle::string::Sprintf(
"%s (at %s:%d)", std::forward<StrType>(what), file, line)
<< std::endl;
return sout.str();
}
template <typename StrType>
inline std::string GetTraceBackString(StrType&& what,
const char* file,
int line) {
if (FLAGS_call_stack_level > 1) {
// FLAGS_call_stack_level>1 means showing c++ call stack
return GetCurrentTraceBackString() + GetErrorSumaryString(what, file, line);
} else {
return GetErrorSumaryString(what, file, line);
}
}
inline std::string SimplifyErrorTypeFormat(const std::string& str) {
std::ostringstream sout;
size_t type_end_pos = str.find(":", 0);
if (type_end_pos == std::string::npos) {
sout << str;
} else {
// Remove "Error:", add "()""
sout << "(" << str.substr(0, type_end_pos - 5) << ")"
<< str.substr(type_end_pos + 1);
}
return sout.str();
}
inline bool is_error(bool stat) { return !stat; }
// Note: This Macro can only be used within enforce.h
#define __THROW_ERROR_INTERNAL__(__ERROR_SUMMARY) \
do { \
HANDLE_THE_ERROR \
throw ::pten::enforce::EnforceNotMet(__ERROR_SUMMARY, __FILE__, __LINE__); \
END_HANDLE_THE_ERROR \
} while (0)
/** ENFORCE EXCEPTION AND MACROS **/
struct EnforceNotMet : public std::exception {
public:
EnforceNotMet(std::exception_ptr e, const char* file, int line) {
try {
std::rethrow_exception(e);
} catch (EnforceNotMet& e) {
code_ = e.code();
err_str_ = GetTraceBackString(e.what(), file, line);
simple_err_str_ = SimplifyErrorTypeFormat(err_str_);
} catch (std::exception& e) {
err_str_ = GetTraceBackString(e.what(), file, line);
simple_err_str_ = SimplifyErrorTypeFormat(err_str_);
}
}
EnforceNotMet(const std::string& str, const char* file, int line)
: err_str_(GetTraceBackString(str, file, line)) {
simple_err_str_ = SimplifyErrorTypeFormat(err_str_);
}
EnforceNotMet(const pten::ErrorSummary& error, const char* file, int line)
: code_(error.code()),
err_str_(GetTraceBackString(error.to_string(), file, line)) {
simple_err_str_ = SimplifyErrorTypeFormat(err_str_);
}
const char* what() const noexcept override {
if (FLAGS_call_stack_level > 1) {
return err_str_.c_str();
} else {
return simple_err_str_.c_str();
}
}
pten::ErrorCode code() const { return code_; }
const std::string& error_str() const { return err_str_; }
const std::string& simple_error_str() const { return simple_err_str_; }
void set_error_str(std::string str) {
if (FLAGS_call_stack_level > 1) {
err_str_ = str;
} else {
simple_err_str_ = str;
}
}
private:
// Used to determine the final type of exception thrown
pten::ErrorCode code_ = pten::ErrorCode::LEGACY;
// Complete error message
// e.g. InvalidArgumentError: ***
std::string err_str_;
// Simple errror message used when no C++ stack and python compile stack
// e.g. (InvalidArgument) ***
std::string simple_err_str_;
};
#define PADDLE_THROW(...) \
do { \
HANDLE_THE_ERROR \
throw ::pten::enforce::EnforceNotMet( \
::pten::ErrorSummary(__VA_ARGS__), __FILE__, __LINE__); \
END_HANDLE_THE_ERROR \
} while (0)
#if defined(__CUDA_ARCH__)
// For cuda, the assertions can affect performance and it is therefore
// recommended to disable them in production code
// https://docs.nvidia.com/cuda/cuda-c-programming-guide/index.html#assertion
#define PADDLE_ENFORCE(_IS_NOT_ERROR, __FORMAT, ...) \
do { \
if (!(_IS_NOT_ERROR)) { \
printf("Error: %s:%d Assertion `%s` failed. " __FORMAT "\n", \
__FILE__, \
__LINE__, \
#_IS_NOT_ERROR, \
##__VA_ARGS__); \
asm("trap;"); \
} \
} while (0)
#elif defined(__HIPCC__)
#define PADDLE_ENFORCE(_IS_NOT_ERROR, __FORMAT, ...) \
do { \
if (!(_IS_NOT_ERROR)) { \
printf("Error: %s:%d Assertion `%s` failed. " __FORMAT "\n", \
__FILE__, \
__LINE__, \
#_IS_NOT_ERROR, \
##__VA_ARGS__); \
abort(); \
} \
} while (0)
#else
#define PADDLE_ENFORCE(COND, ...) \
do { \
auto __cond__ = (COND); \
if (UNLIKELY(::pten::is_error(__cond__))) { \
__THROW_ERROR_INTERNAL__(pten::ErrorSummary(__VA_ARGS__)); \
} \
} while (0)
#endif
/*
* Some enforce helpers here, usage:
* int a = 1;
* int b = 2;
* PADDLE_ENFORCE_EQ(a, b);
*
* will raise an expression described as follows:
* "Expected input a == b, but received a(1) != b(2)."
* with detailed stack information.
*
* extra messages is also supported, for example:
* PADDLE_ENFORCE(a, b, "some simple enforce failed between %d numbers", 2)
*/
#define PADDLE_ENFORCE_NOT_NULL(__VAL, ...) \
do { \
if (UNLIKELY(nullptr == (__VAL))) { \
auto __summary__ = pten::ErrorSummary(__VA_ARGS__); \
auto __message__ = ::paddle::string::Sprintf( \
"%s\n [Hint: " #__VAL " should not be null.]", \
__summary__.error_message()); \
__THROW_ERROR_INTERNAL__( \
pten::ErrorSummary(__summary__.code(), __message__)); \
} \
} while (0)
#define __PADDLE_BINARY_COMPARE(__VAL1, __VAL2, __CMP, __INV_CMP, ...) \
do { \
auto __val1 = (__VAL1); \
auto __val2 = (__VAL2); \
using __TYPE1__ = decltype(__val1); \
using __TYPE2__ = decltype(__val2); \
using __COMMON_TYPE1__ = \
::pten::details::CommonType1<__TYPE1__, __TYPE2__>; \
using __COMMON_TYPE2__ = \
::pten::details::CommonType2<__TYPE1__, __TYPE2__>; \
bool __is_not_error = (static_cast<__COMMON_TYPE1__>(__val1))__CMP( \
static_cast<__COMMON_TYPE2__>(__val2)); \
if (UNLIKELY(!__is_not_error)) { \
auto __summary__ = pten::ErrorSummary(__VA_ARGS__); \
constexpr bool __kCanToString__ = \
::pten::details::CanToString<__TYPE1__>::kValue && \
::pten::details::CanToString<__TYPE2__>::kValue; \
auto __message__ = ::paddle::string::Sprintf( \
"%s\n [Hint: Expected %s " #__CMP \
" %s, but received %s " #__INV_CMP " %s.]", \
__summary__.error_message(), \
#__VAL1, \
#__VAL2, \
::pten::details::BinaryCompareMessageConverter< \
__kCanToString__>::Convert(#__VAL1, __val1), \
::pten::details::BinaryCompareMessageConverter< \
__kCanToString__>::Convert(#__VAL2, __val2)); \
__THROW_ERROR_INTERNAL__( \
pten::ErrorSummary(__summary__.code(), __message__)); \
} \
} while (0)
#define PADDLE_ENFORCE_EQ(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, ==, !=, __VA_ARGS__)
#define PADDLE_ENFORCE_NE(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, !=, ==, __VA_ARGS__)
#define PADDLE_ENFORCE_GT(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, >, <=, __VA_ARGS__)
#define PADDLE_ENFORCE_GE(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, >=, <, __VA_ARGS__)
#define PADDLE_ENFORCE_LT(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, <, >=, __VA_ARGS__)
#define PADDLE_ENFORCE_LE(__VAL0, __VAL1, ...) \
__PADDLE_BINARY_COMPARE(__VAL0, __VAL1, <=, >, __VA_ARGS__)
/** EXTENDED TOOL FUNCTIONS WITH CHECKING **/
/*
* Summary: This macro is used to get Variable or internal type
* data (such as LoDTensor or SelectedRows) of the Input and
* Output in op, generally used when call scope.FindVar(Input/
* Output("Name")) or ctx.Input<LoDTensor>().
* Firstly this macro check whether the obtained pointer is null,
* and then return data if it is not null.
*
* Note: This macro is only suitable for specific scenarios and
* does not intended to be widely used. If it cannot meet the
* requirements, please use other PADDLE_ENFORCE** check macro.
*
* Parameters:
*     __PTR: pointer
* __ROLE: (string), Input or Output
* __NAME: (string), Input or Output name
* __OP_TYPE: (string), the op type
*  
* Return: The data pointed to by the pointer.
*
* Examples:
* GET_DATA_SAFELY(ctx.Input<LoDTensor>("X"), "Input", "X", "Mul");
*/
#define GET_DATA_SAFELY(__PTR, __ROLE, __NAME, __OP_TYPE) \
(([&]() -> std::add_lvalue_reference<decltype(*(__PTR))>::type { \
auto* __ptr = (__PTR); \
if (UNLIKELY(nullptr == __ptr)) { \
auto __summary__ = pten::errors::NotFound( \
"Unable to get %s data of %s %s in operator %s. " \
"Possible reasons are:\n" \
" 1. The %s is not the %s of operator %s;\n" \
" 2. The %s has no corresponding variable passed in;\n" \
" 3. The %s corresponding variable is not initialized.", \
pten::demangle( \
typeid(std::add_lvalue_reference<decltype(*__ptr)>::type) \
.name()), \
__ROLE, \
__NAME, \
__OP_TYPE, \
__NAME, \
__ROLE, \
__OP_TYPE, \
__NAME, \
__NAME); \
auto __message__ = ::paddle::string::Sprintf( \
"%s\n [Hint: pointer " #__PTR " should not be null.]", \
__summary__.error_message()); \
__THROW_ERROR_INTERNAL__( \
pten::ErrorSummary(__summary__.code(), __message__)); \
} \
return *__ptr; \
})())
/*
* Summary: This macro is used to check whether op has specified
* Input or Output Variables. Because op's Input and Output
* checking are written similarly, so abstract this macro.
*
* Parameters:
*     __EXPR: (bool), the bool expression
* __ROLE: (string), Input or Output
* __NAME: (string), Input or Output name
* __OP_TYPE: (string), the op type
*
* Examples:
* OP_INOUT_CHECK(ctx->HasInput("X"), "Input", "X", "Mul");
*/
#define OP_INOUT_CHECK(__EXPR, __ROLE, __NAME, __OP_TYPE) \
do { \
PADDLE_ENFORCE_EQ( \
__EXPR, \
true, \
pten::errors::NotFound( \
"No %s(%s) found for %s operator.", __ROLE, __NAME, __OP_TYPE)); \
} while (0)
} // namespace enforce
using namespace enforce; // NOLINT
} // namespace pten
paddle/fluid/platform/errors.cc → paddle/pten/core/errors.cc
浏览文件 @ 6ca49164
...@@ -12,54 +12,50 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ...@@ -12,54 +12,50 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and See the License for the specific language governing permissions and
limitations under the License. */ limitations under the License. */
#include "paddle/fluid/platform/errors.h" #include "paddle/pten/core/errors.h"
#include <stdexcept> #include <stdexcept>
namespace paddle { namespace pten {
namespace platform { std::string error_name(ErrorCode code) {
typedef ::paddle::platform::error::Code Code;
std::string error_name(Code code) {
switch (code) { switch (code) {
case paddle::platform::error::LEGACY: case ErrorCode::LEGACY:
return "Error"; return "Error";
break; break;
case paddle::platform::error::INVALID_ARGUMENT: case ErrorCode::INVALID_ARGUMENT:
return "InvalidArgumentError"; return "InvalidArgumentError";
break; break;
case paddle::platform::error::NOT_FOUND: case ErrorCode::NOT_FOUND:
return "NotFoundError"; return "NotFoundError";
break; break;
case paddle::platform::error::OUT_OF_RANGE: case ErrorCode::OUT_OF_RANGE:
return "OutOfRangeError"; return "OutOfRangeError";
break; break;
case paddle::platform::error::ALREADY_EXISTS: case ErrorCode::ALREADY_EXISTS:
return "AlreadyExistsError"; return "AlreadyExistsError";
break; break;
case paddle::platform::error::RESOURCE_EXHAUSTED: case ErrorCode::RESOURCE_EXHAUSTED:
return "ResourceExhaustedError"; return "ResourceExhaustedError";
break; break;
case paddle::platform::error::PRECONDITION_NOT_MET: case ErrorCode::PRECONDITION_NOT_MET:
return "PreconditionNotMetError"; return "PreconditionNotMetError";
break; break;
case paddle::platform::error::PERMISSION_DENIED: case ErrorCode::PERMISSION_DENIED:
return "PermissionDeniedError"; return "PermissionDeniedError";
break; break;
case paddle::platform::error::EXECUTION_TIMEOUT: case ErrorCode::EXECUTION_TIMEOUT:
return "ExecutionTimeoutError"; return "ExecutionTimeoutError";
break; break;
case paddle::platform::error::UNIMPLEMENTED: case ErrorCode::UNIMPLEMENTED:
return "UnimplementedError"; return "UnimplementedError";
break; break;
case paddle::platform::error::UNAVAILABLE: case ErrorCode::UNAVAILABLE:
return "UnavailableError"; return "UnavailableError";
break; break;
case paddle::platform::error::FATAL: case ErrorCode::FATAL:
return "FatalError"; return "FatalError";
break; break;
case paddle::platform::error::EXTERNAL: case ErrorCode::EXTERNAL:
return "ExternalError"; return "ExternalError";
break; break;
default: default:
...@@ -74,6 +70,4 @@ std::string ErrorSummary::to_string() const { ...@@ -74,6 +70,4 @@ std::string ErrorSummary::to_string() const {
result += error_message(); result += error_message();
return result; return result;
} }
} // namespace pten
} // namespace platform
} // namespace paddle
paddle/fluid/platform/error_codes.proto → paddle/pten/core/errors.h
浏览文件 @ 6ca49164
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved. /* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License"); licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License. you may not use this file except in compliance with the License.
You may obtain a copy of the License at You may obtain a copy of the License at
...@@ -12,69 +12,135 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ...@@ -12,69 +12,135 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and See the License for the specific language governing permissions and
limitations under the License. */ limitations under the License. */
syntax = "proto2"; #pragma once
option optimize_for = LITE_RUNTIME;
package paddle.platform.error;
enum Code { #include <memory>
#include <stdexcept>
#include <string>
#include <tuple>
#include <type_traits>
#include "paddle/utils/string/printf.h"
namespace pten {
enum ErrorCode {
// Legacy error. // Legacy error.
// Error type string: "Error" // Error type string: "Error"
LEGACY = 0; LEGACY = 0,
// Client specified an invalid argument. // Client specified an invalid argument.
// Error type string: "InvalidArgumentError" // Error type string: "InvalidArgumentError"
INVALID_ARGUMENT = 1; INVALID_ARGUMENT = 1,
// Some requested entity (e.g., file or directory) was not found. // Some requested entity (e.g., file or directory) was not found.
// Error type string: "NotFoundError" // Error type string: "NotFoundError"
NOT_FOUND = 2; NOT_FOUND = 2,
// Operation tried to iterate past the valid input range. E.g., seeking or // Operation tried to iterate past the valid input range. E.g., seeking or
// reading past end of file. // reading past end of file.
// Error type string: "OutOfRangeError" // Error type string: "OutOfRangeError"
OUT_OF_RANGE = 3; OUT_OF_RANGE = 3,
// Some entity that we attempted to create (e.g., file or directory) // Some entity that we attempted to create (e.g., file or directory)
// already exists. // already exists.
// Error type string: "AlreadyExistsError" // Error type string: "AlreadyExistsError"
ALREADY_EXISTS = 4; ALREADY_EXISTS = 4,
// Some resource has been exhausted, perhaps a per-user quota, or // Some resource has been exhausted, perhaps a per-user quota, or
// perhaps the entire file system is out of space. // perhaps the entire file system is out of space.
// Error type string: "ResourceExhaustedError" // Error type string: "ResourceExhaustedError"
RESOURCE_EXHAUSTED = 5; RESOURCE_EXHAUSTED = 5,
// Operation was rejected because the system is not in a state // Operation was rejected because the system is not in a state
// required for the operation's execution. // required for the operation's execution.
// Error type string: "PreconditionNotMetError" // Error type string: "PreconditionNotMetError"
PRECONDITION_NOT_MET = 6; PRECONDITION_NOT_MET = 6,
// The caller does not have permission to execute the specified // The caller does not have permission to execute the specified
// operation. // operation.
// Error type string: "PermissionDeniedError" // Error type string: "PermissionDeniedError"
PERMISSION_DENIED = 7; PERMISSION_DENIED = 7,
// Deadline expired before operation could complete. // Deadline expired before operation could complete.
// Error type string: "ExecutionTimeout" // Error type string: "ExecutionTimeout"
EXECUTION_TIMEOUT = 8; EXECUTION_TIMEOUT = 8,
// Operation is not implemented or not supported/enabled in this service. // Operation is not implemented or not supported/enabled in this service.
// Error type string: "UnimpelmentedError" // Error type string: "UnimpelmentedError"
UNIMPLEMENTED = 9; UNIMPLEMENTED = 9,
// The service is currently unavailable. This is a most likely a // The service is currently unavailable. This is a most likely a
// transient condition and may be corrected by retrying with // transient condition and may be corrected by retrying with
// a backoff. // a backoff.
// Error type string: "UnavailableError" // Error type string: "UnavailableError"
UNAVAILABLE = 10; UNAVAILABLE = 10,
// Fatal errors. Means some invariant expected by the underlying // Fatal errors. Means some invariant expected by the underlying
// system has been broken. If you see one of these errors, // system has been broken. If you see one of these errors,
// something is very broken. // something is very broken.
// Error type string: "FatalError" // Error type string: "FatalError"
FATAL = 11; FATAL = 11,
// Third-party library error. // Third-party library error.
// Error type string: "ExternalError" // Error type string: "ExternalError"
EXTERNAL = 12; EXTERNAL = 12,
} };
class ErrorSummary {
public:
// Note(chenweihang): Final deprecated constructor
// This constructor is used to be compatible with
// current existing untyped PADDLE_ENFORCE_*
// PADDLE_ENFORCE
// Note(chenweihang): Windows openblas need this
// constructor for compiling PADDLE_ENFORCE in *.cu,
// this is a bug cause we can't remove this
// constructor now.
template <typename... Args>
explicit ErrorSummary(Args... args) {
code_ = pten::ErrorCode::LEGACY;
msg_ = paddle::string::Sprintf(args...);
}
// Note(chenweihang): Only recommended constructor
// No longer supports PADDLE_ENFORCE without type or without error message
explicit ErrorSummary(ErrorCode code, std::string msg)
: code_(code), msg_(msg) {}
ErrorCode code() const { return code_; }
const std::string& error_message() const { return msg_; }
std::string to_string() const;
private:
ErrorCode code_;
std::string msg_;
};
namespace errors {
#define REGISTER_ERROR(FUNC, CONST, ...) \
template <typename... Args> \
::pten::ErrorSummary FUNC(Args... args) { \
return ::pten::ErrorSummary(::pten::CONST, \
::paddle::string::Sprintf(args...)); \
}
REGISTER_ERROR(InvalidArgument, ErrorCode::INVALID_ARGUMENT)
REGISTER_ERROR(NotFound, ErrorCode::NOT_FOUND)
REGISTER_ERROR(OutOfRange, ErrorCode::OUT_OF_RANGE)
REGISTER_ERROR(AlreadyExists, ErrorCode::ALREADY_EXISTS)
REGISTER_ERROR(ResourceExhausted, ErrorCode::RESOURCE_EXHAUSTED)
REGISTER_ERROR(PreconditionNotMet, ErrorCode::PRECONDITION_NOT_MET)
REGISTER_ERROR(PermissionDenied, ErrorCode::PERMISSION_DENIED)
REGISTER_ERROR(ExecutionTimeout, ErrorCode::EXECUTION_TIMEOUT)
REGISTER_ERROR(Unimplemented, ErrorCode::UNIMPLEMENTED)
REGISTER_ERROR(Unavailable, ErrorCode::UNAVAILABLE)
REGISTER_ERROR(Fatal, ErrorCode::FATAL)
REGISTER_ERROR(External, ErrorCode::EXTERNAL)
#undef REGISTER_ERROR
} // namespace errors
} // namespace pten
paddle/pten/core/kernel_context.h
浏览文件 @ 6ca49164
...@@ -24,7 +24,7 @@ ...@@ -24,7 +24,7 @@
// See Note [ Why still include the fluid headers? ] // See Note [ Why still include the fluid headers? ]
#include "paddle/fluid/platform/device_context.h" #include "paddle/fluid/platform/device_context.h"
#include "paddle/fluid/platform/enforce.h" #include "paddle/pten/core/enforce.h"
namespace pten { namespace pten {
...@@ -123,7 +123,7 @@ class KernelContext { ...@@ -123,7 +123,7 @@ class KernelContext {
try { try {
return paddle::any_cast<AttrType>(attrs_.at(idx)); return paddle::any_cast<AttrType>(attrs_.at(idx));
} catch (paddle::bad_any_cast&) { } catch (paddle::bad_any_cast&) {
PADDLE_THROW(paddle::platform::errors::InvalidArgument( PADDLE_THROW(pten::errors::InvalidArgument(
"Attribute cast error in Op Kernel Context.")); "Attribute cast error in Op Kernel Context."));
} }
} }
......
paddle/pten/core/kernel_factory.cc
浏览文件 @ 6ca49164
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
#include "paddle/pten/core/kernel_factory.h" #include "paddle/pten/core/kernel_factory.h"
// See Note [ Why still include the fluid headers? ] // See Note [ Why still include the fluid headers? ]
#include "paddle/fluid/platform/enforce.h" #include "paddle/pten/core/enforce.h"
namespace pten { namespace pten {
...@@ -64,8 +64,8 @@ const Kernel& KernelFactory::SelectKernelOrThrowError( ...@@ -64,8 +64,8 @@ const Kernel& KernelFactory::SelectKernelOrThrowError(
auto iter = kernels_.find(kernel_name); auto iter = kernels_.find(kernel_name);
PADDLE_ENFORCE_NE(iter, PADDLE_ENFORCE_NE(iter,
kernels_.end(), kernels_.end(),
paddle::platform::errors::NotFound( pten::errors::NotFound("The kernel `%s` is not registered.",
"The kernel `%s` is not registered.", kernel_name)); kernel_name));
auto kernel_iter = iter->second.find(kernel_key); auto kernel_iter = iter->second.find(kernel_key);
// TODO(chenweihang): polish refind impl here // TODO(chenweihang): polish refind impl here
...@@ -78,7 +78,7 @@ const Kernel& KernelFactory::SelectKernelOrThrowError( ...@@ -78,7 +78,7 @@ const Kernel& KernelFactory::SelectKernelOrThrowError(
PADDLE_ENFORCE_NE( PADDLE_ENFORCE_NE(
kernel_iter, kernel_iter,
iter->second.end(), iter->second.end(),
paddle::platform::errors::NotFound( pten::errors::NotFound(
"The kernel with key %s of kernel `%s` is not registered.", "The kernel with key %s of kernel `%s` is not registered.",
kernel_key, kernel_key,
kernel_name)); kernel_name));
......
paddle/pten/core/kernel_factory.h
浏览文件 @ 6ca49164
...@@ -27,7 +27,7 @@ ...@@ -27,7 +27,7 @@
#include "paddle/pten/core/kernel_def.h" #include "paddle/pten/core/kernel_def.h"
// See Note [ Why still include the fluid headers? ] // See Note [ Why still include the fluid headers? ]
#include "paddle/fluid/platform/enforce.h" #include "paddle/pten/core/enforce.h"
#include "paddle/utils/flat_hash_map.h" #include "paddle/utils/flat_hash_map.h"
#include "paddle/utils/small_vector.h" #include "paddle/utils/small_vector.h"
......
paddle/pten/core/kernel_registry.h
浏览文件 @ 6ca49164
...@@ -26,7 +26,7 @@ ...@@ -26,7 +26,7 @@
#include "paddle/pten/core/kernel_utils.h" #include "paddle/pten/core/kernel_utils.h"
#include "paddle/pten/core/macros.h" #include "paddle/pten/core/macros.h"
#include "paddle/fluid/platform/enforce.h" #include "paddle/pten/core/enforce.h"
namespace pten { namespace pten {
......
paddle/pten/core/kernel_utils.h
浏览文件 @ 6ca49164
...@@ -22,7 +22,7 @@ ...@@ -22,7 +22,7 @@
#include "paddle/pten/core/kernel_def.h" #include "paddle/pten/core/kernel_def.h"
// See Note [ Why still include the fluid headers? ] // See Note [ Why still include the fluid headers? ]
#include "paddle/fluid/platform/enforce.h" #include "paddle/pten/core/enforce.h"
namespace pten { namespace pten {
......
paddle/pten/core/type_defs.h 0 → 100644
浏览文件 @ 6ca49164
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include <boost/variant.hpp>
namespace paddle {
namespace framework {
// The order should be as same as framework.proto
// NOTE(xiongkun): we extract from framework/typedef.h to ensure we can transfer
// enforce.h
class BlockDesc;
using Attribute = boost::variant<boost::blank,
int,
float,
std::string,
std::vector<int>,
std::vector<float>,
std::vector<std::string>,
bool,
std::vector<bool>,
BlockDesc*,
int64_t,
std::vector<BlockDesc*>,
std::vector<int64_t>,
std::vector<double>>;
using AttributeMap = std::unordered_map<std::string, Attribute>;
} // namespace framework
namespace imperative {
class VariableWrapper;
class SavedVariableWrapperList;
class VarBase;
class OpBase;
class GradOpNode;
class Tracer;
using WeakNameVarBaseMap =
std::map<std::string, std::vector<std::weak_ptr<VarBase>>>;
namespace details {
template <typename T>
struct NameVarMapTrait {};
template <>
struct NameVarMapTrait<VarBase> {
using Type = std::map<std::string, std::vector<std::shared_ptr<VarBase>>>;
};
template <>
struct NameVarMapTrait<VariableWrapper> {
using Type = std::map<std::string, SavedVariableWrapperList>;
};
} // namespace details
template <typename T>
using NameVarMap = typename details::NameVarMapTrait<T>::Type;
using NameVarBaseMap = NameVarMap<VarBase>;
using NameVariableWrapperMap = NameVarMap<VariableWrapper>;
using VariableWrapperList = std::vector<std::shared_ptr<VariableWrapper>>;
} // namespace imperative
} // namespace paddle
paddle/pten/core/utils/intrusive_ptr.h
浏览文件 @ 6ca49164
...@@ -16,7 +16,7 @@ limitations under the License. */ ...@@ -16,7 +16,7 @@ limitations under the License. */
#include <utility> #include <utility>
#include "glog/logging.h" #include "glog/logging.h"
#include "paddle/fluid/platform/enforce.h" #include "paddle/pten/core/enforce.h"
namespace pten { namespace pten {
...@@ -62,7 +62,7 @@ class intrusive_ptr { ...@@ -62,7 +62,7 @@ class intrusive_ptr {
T& operator*() const { T& operator*() const {
PADDLE_ENFORCE_NOT_NULL( PADDLE_ENFORCE_NOT_NULL(
px, px,
paddle::platform::errors::PreconditionNotMet( pten::errors::PreconditionNotMet(
"The pointer must be non-null before the dereference operation.")); "The pointer must be non-null before the dereference operation."));
return *px; return *px;
} }
...@@ -70,7 +70,7 @@ class intrusive_ptr { ...@@ -70,7 +70,7 @@ class intrusive_ptr {
T* operator->() const { T* operator->() const {
PADDLE_ENFORCE_NOT_NULL( PADDLE_ENFORCE_NOT_NULL(
px, px,
paddle::platform::errors::PreconditionNotMet( pten::errors::PreconditionNotMet(
"The pointer must be non-null before the dereference operation.")); "The pointer must be non-null before the dereference operation."));
return px; return px;
} }
......
paddle/pten/kernels/funcs/elementwise_functor.h
浏览文件 @ 6ca49164
...@@ -14,8 +14,8 @@ limitations under the License. */ ...@@ -14,8 +14,8 @@ limitations under the License. */
#pragma once #pragma once
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/common/float16.h" #include "paddle/pten/common/float16.h"
#include "paddle/pten/core/enforce.h"
#include "paddle/pten/core/hostdevice.h" #include "paddle/pten/core/hostdevice.h"
namespace pten { namespace pten {
......
paddle/pten/kernels/gpu/math_kernel.cu
浏览文件 @ 6ca49164
...@@ -27,10 +27,10 @@ limitations under the License. */ ...@@ -27,10 +27,10 @@ limitations under the License. */
namespace cub = hipcub; namespace cub = hipcub;
#endif #endif
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/common/complex.h" #include "paddle/pten/common/complex.h"
#include "paddle/pten/common/float16.h" #include "paddle/pten/common/float16.h"
#include "paddle/pten/core/convert_utils.h" #include "paddle/pten/core/convert_utils.h"
#include "paddle/pten/core/enforce.h"
#include "paddle/pten/core/kernel_registry.h" #include "paddle/pten/core/kernel_registry.h"
namespace pten { namespace pten {
......
paddle/pten/kernels/gpu/reduce.h
浏览文件 @ 6ca49164
...@@ -37,10 +37,10 @@ namespace cub = hipcub; ...@@ -37,10 +37,10 @@ namespace cub = hipcub;
#include "paddle/fluid/operators/kernel_primitives/kernel_primitives.h" #include "paddle/fluid/operators/kernel_primitives/kernel_primitives.h"
#include "paddle/fluid/platform/device/gpu/gpu_device_function.h" #include "paddle/fluid/platform/device/gpu/gpu_device_function.h"
#include "paddle/fluid/platform/device/gpu/gpu_info.h" #include "paddle/fluid/platform/device/gpu/gpu_info.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/fluid/platform/fast_divmod.h" #include "paddle/fluid/platform/fast_divmod.h"
#include "paddle/fluid/string/string_helper.h" #include "paddle/fluid/string/string_helper.h"
#include "paddle/pten/core/array.h" #include "paddle/pten/core/array.h"
#include "paddle/pten/core/enforce.h"
#include "paddle/pten/api/ext/dispatch.h" #include "paddle/pten/api/ext/dispatch.h"
#include "paddle/pten/backends/gpu/gpu_context.h" #include "paddle/pten/backends/gpu/gpu_context.h"
...@@ -94,7 +94,7 @@ static inline void CheckReduceRank(int reduce_rank, int rank) { ...@@ -94,7 +94,7 @@ static inline void CheckReduceRank(int reduce_rank, int rank) {
if (rank % 2 == 0) { if (rank % 2 == 0) {
PADDLE_ENFORCE_EQ(reduce_rank, PADDLE_ENFORCE_EQ(reduce_rank,
rank / 2, rank / 2,
paddle::platform::errors::InvalidArgument( pten::errors::InvalidArgument(
"ReduceOp: invalid reduce rank. When rank = %d, " "ReduceOp: invalid reduce rank. When rank = %d, "
"reduce_rank must be %d, but got %d.", "reduce_rank must be %d, but got %d.",
rank, rank,
...@@ -106,7 +106,7 @@ static inline void CheckReduceRank(int reduce_rank, int rank) { ...@@ -106,7 +106,7 @@ static inline void CheckReduceRank(int reduce_rank, int rank) {
PADDLE_ENFORCE_EQ( PADDLE_ENFORCE_EQ(
reduce_rank == lower_rank || reduce_rank == upper_rank, reduce_rank == lower_rank || reduce_rank == upper_rank,
true, true,
paddle::platform::errors::InvalidArgument( pten::errors::InvalidArgument(
"ReduceOp: invalid reduce rank. When rank = %d, reduce_rank " "ReduceOp: invalid reduce rank. When rank = %d, reduce_rank "
"must be %d or %d, but got %d.", "must be %d or %d, but got %d.",
rank, rank,
...@@ -122,7 +122,7 @@ static inline pten::framework::Array<T, ElementCount> VectorToArray( ...@@ -122,7 +122,7 @@ static inline pten::framework::Array<T, ElementCount> VectorToArray(
const VectorLikeType& vec) { const VectorLikeType& vec) {
PADDLE_ENFORCE_LE(vec.size(), PADDLE_ENFORCE_LE(vec.size(),
ElementCount, ElementCount,
paddle::platform::errors::InvalidArgument( pten::errors::InvalidArgument(
"Cub reduce Array: size not match. Received " "Cub reduce Array: size not match. Received "
"vec.size() %d > ElementCount %d.", "vec.size() %d > ElementCount %d.",
vec.size(), vec.size(),
...@@ -149,7 +149,7 @@ static inline std::vector<int> GetReduceDim(const std::vector<int64_t>& dims, ...@@ -149,7 +149,7 @@ static inline std::vector<int> GetReduceDim(const std::vector<int64_t>& dims,
for (auto e : dims) { for (auto e : dims) {
PADDLE_ENFORCE_LT(e, PADDLE_ENFORCE_LT(e,
dim_size, dim_size,
paddle::platform::errors::InvalidArgument( pten::errors::InvalidArgument(
"ReduceOp: invalid axis, when x_dims is %d, " "ReduceOp: invalid axis, when x_dims is %d, "
"axis[i] should less than x_dims, but got %d.", "axis[i] should less than x_dims, but got %d.",
dim_size, dim_size,
...@@ -1057,7 +1057,7 @@ static ...@@ -1057,7 +1057,7 @@ static
int reduce_num, int reduce_num,
const paddle::platform::Place& place, const paddle::platform::Place& place,
gpuStream_t stream) { gpuStream_t stream) {
PADDLE_THROW(paddle::platform::errors::InvalidArgument( PADDLE_THROW(pten::errors::InvalidArgument(
"Tx should not be float16 when using cub::DeviceReduce::Reduce().")); "Tx should not be float16 when using cub::DeviceReduce::Reduce()."));
} }
......
paddle/utils/CMakeLists.txt 0 → 100644
浏览文件 @ 6ca49164
add_subdirectory(string)
paddle/fluid/string/CMakeLists.txt → paddle/utils/string/CMakeLists.txt
浏览文件 @ 6ca49164
cc_library(stringpiece SRCS piece.cc DEPS flags) cc_library(stringpiece SRCS piece.cc DEPS flags)
cc_library(pretty_log SRCS pretty_log.cc DEPS flags) cc_library(pretty_log SRCS pretty_log.cc DEPS flags)
cc_library(string_helper SRCS string_helper.cc DEPS flags) cc_library(string_helper SRCS string_helper.cc DEPS flags)
cc_test(stringpiece_test SRCS piece_test.cc DEPS stringpiece glog gflags) cc_test(stringpiece_test SRCS piece_test.cc DEPS stringpiece gflags)
cc_test(stringprintf_test SRCS printf_test.cc DEPS glog gflags) cc_test(stringprintf_test SRCS printf_test.cc DEPS gflags)
cc_test(to_string_test SRCS to_string_test.cc) cc_test(to_string_test SRCS to_string_test.cc)
cc_test(split_test SRCS split_test.cc) cc_test(split_test SRCS split_test.cc)
cc_test(string_helper_test SRCS string_helper_test.cc DEPS string_helper) cc_test(string_helper_test SRCS string_helper_test.cc DEPS string_helper)
paddle/fluid/string/piece.cc → paddle/utils/string/piece.cc
浏览文件 @ 6ca49164
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "paddle/fluid/string/piece.h" #include "paddle/utils/string/piece.h"
#include <string.h> #include <string.h>
#include <algorithm> #include <algorithm>
...@@ -76,9 +76,11 @@ bool HasPrefix(Piece s, Piece x) { ...@@ -76,9 +76,11 @@ bool HasPrefix(Piece s, Piece x) {
} }
bool HasSuffix(Piece s, Piece x) { bool HasSuffix(Piece s, Piece x) {
return !x.len() ? true : ((s.len() >= x.len()) && return !x.len()
(memcmp(s.data() + (s.len() - x.len()), x.data(), ? true
x.len()) == 0)); : ((s.len() >= x.len()) &&
(memcmp(s.data() + (s.len() - x.len()), x.data(), x.len()) ==
0));
} }
Piece SkipPrefix(Piece s, size_t n) { Piece SkipPrefix(Piece s, size_t n) {
......
paddle/utils/string/piece.h 0 → 100644
浏览文件 @ 6ca49164
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ostream>
#include <string>
namespace paddle {
namespace string {
// Piece points into a std::string object but doesn't own the
// string. It is for efficient access to strings. Like Go's string
// type. Not that Piece doesn't mutate the underlying string,
// so it is thread-safe given that the underlying string doesn't
// change. Because Piece contains a little data members, and
// its syntax is simple as it doesn't own/manage the string, it is
// cheap to construct Pieces and pass them around.
class Piece {
public:
static const size_t npos = static_cast<size_t>(-1);
// We provide non-explicit singleton constructors so users can
// pass in a "const char*" or a "string" wherever a "Piece"
// is expected. These constructors ensure that if data_ is NULL,
// size_ is 0.
Piece();
Piece(const char* d, size_t n);
Piece(const char* d); // NOLINT: accept C string into Piece.
Piece(const std::string& s); // NOLINT: accept C++ string into Piece.
const char* data() const { return data_; }
size_t len() const { return size_; }
char operator[](size_t n) const;
// Piece doesn't own the string, so both iterator and const
// iterator are const char* indeed.
typedef const char* const_iterator;
typedef const char* iterator;
iterator begin() const { return data_; }
iterator end() const { return data_ + size_; }
// Return a string that contains the copy of the referenced data.
std::string ToString() const { return std::string(data_, size_); }
private:
const char* data_;
size_t size_;
// Intentionally copyable
};
int Compare(Piece a, Piece b);
bool operator==(Piece x, Piece y);
bool operator!=(Piece x, Piece y);
bool operator<(Piece x, Piece y);
bool operator>(Piece x, Piece y);
bool operator<=(Piece x, Piece y);
bool operator>=(Piece x, Piece y);
bool HasPrefix(Piece s, Piece prefix);
bool HasSuffix(Piece s, Piece suffix);
Piece SkipPrefix(Piece s, size_t n);
Piece SkipSuffix(Piece s, size_t n);
// Skip the prefix (or suffix) if it matches with the string.
Piece TrimPrefix(Piece s, Piece prefix);
Piece TrimSuffix(Piece s, Piece suffix);
// Returns if s contains sub. Any s except for empty s contains an
// empty sub.
bool Contains(Piece s, Piece sub);
// Return the first occurrence of sub in s, or npos. If both s and
// sub is empty, it returns npos; otherwise, if only sub is empty, it
// returns 0.
size_t Index(Piece s, Piece sub);
// Return the first occurrence of c in s[pos:end], or npos.
size_t Find(Piece s, char c, size_t pos);
// Search range is [0..pos] inclusive. If pos == npos, search everything.
size_t RFind(Piece s, char c, size_t pos);
Piece SubStr(Piece s, size_t pos, size_t n);
// allow Piece to be logged
std::ostream& operator<<(std::ostream& o, Piece piece);
} // namespace string
} // namespace paddle
paddle/fluid/string/piece_test.cc → paddle/utils/string/piece_test.cc
浏览文件 @ 6ca49164
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "paddle/fluid/string/piece.h" #include "paddle/utils/string/piece.h"
#include "gtest/gtest.h" #include "gtest/gtest.h"
......
paddle/fluid/string/pretty_log.cc → paddle/utils/string/pretty_log.cc
浏览文件 @ 6ca49164
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "paddle/fluid/string/pretty_log.h" #include "paddle/utils/string/pretty_log.h"
#include "gflags/gflags.h" #include "gflags/gflags.h"
DEFINE_bool(color, true, "Whether to turn on pretty log"); DEFINE_bool(color, true, "Whether to turn on pretty log");
......
paddle/utils/string/pretty_log.h 0 → 100644
浏览文件 @ 6ca49164
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <iostream>
#include <sstream>
#include <string>
#include <utility>
#include "gflags/gflags.h"
#include "paddle/utils/string/printf.h"
DECLARE_bool(color);
namespace paddle {
namespace string {
inline std::string black() { return FLAGS_color ? "\e[30m" : ""; }
inline std::string red() { return FLAGS_color ? "\e[31m" : ""; }
inline std::string b_red() { return FLAGS_color ? "\e[41m" : ""; }
inline std::string green() { return FLAGS_color ? "\e[32m" : ""; }
inline std::string yellow() { return FLAGS_color ? "\e[33m" : ""; }
inline std::string blue() { return FLAGS_color ? "\e[34m" : ""; }
inline std::string purple() { return FLAGS_color ? "\e[35m" : ""; }
inline std::string cyan() { return FLAGS_color ? "\e[36m" : ""; }
inline std::string light_gray() { return FLAGS_color ? "\e[37m" : ""; }
inline std::string white() { return FLAGS_color ? "\e[37m" : ""; }
inline std::string light_red() { return FLAGS_color ? "\e[91m" : ""; }
inline std::string dim() { return FLAGS_color ? "\e[2m" : ""; }
inline std::string bold() { return FLAGS_color ? "\e[1m" : ""; }
inline std::string underline() { return FLAGS_color ? "\e[4m" : ""; }
inline std::string blink() { return FLAGS_color ? "\e[5m" : ""; }
inline std::string reset() { return FLAGS_color ? "\e[0m" : ""; }
using TextBlock = std::pair<std::string, std::string>;
struct Style {
static std::string info() { return black(); }
static std::string warn() { return b_red(); }
static std::string suc() { return green(); }
static std::string H1() { return bold() + purple(); }
static std::string H2() { return green(); }
static std::string H3() { return green(); }
static std::string detail() { return light_gray(); }
};
template <typename... Args>
static void PrettyLogEndl(const std::string &style,
const char *fmt,
const Args &... args) {
std::cerr << style << Sprintf(fmt, args...) << reset() << std::endl;
}
template <typename... Args>
static void PrettyLog(const std::string &style,
const char *fmt,
const Args &... args) {
std::cerr << style << Sprintf(fmt, args...) << reset();
}
template <typename... Args>
static void PrettyLogInfo(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::info(), fmt, args...);
}
template <typename... Args>
static void PrettyLogDetail(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::detail(), fmt, args...);
}
template <typename... Args>
static void PrettyLogH1(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::H1(), fmt, args...);
}
template <typename... Args>
static void PrettyLogH2(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::H2(), fmt, args...);
}
} // namespace string
} // namespace paddle
paddle/utils/string/printf.h 0 → 100644
浏览文件 @ 6ca49164
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Compared with std::stringstream, there are primary purpose of
// string::Printf:
//
// 1. Type-safe printing, with why and how explained in
// http://www.drdobbs.com/stringprintf-a-typesafe-printf-family-fo/184401999.
// Implementation includes
//
// https://github.com/c42f/tinyformat
// boost::format
// std::stringstream
//
// std::stringstream is not convenient enough in many cases. For example:
//
// std::cout << std::setprecision(2) << std::fixed << 1.23456 << "\n";
//
// boost::format is the most convenient one. We can have
//
// std::cout << format("%2% %1%") % 36 % 77;
//
// or
//
// format fmter("%2% %1%");
// fmter % 36; fmter % 77;
// std::cout << fmter.c_str();
//
// But the overloading of % might be overkilling and it would be
// more efficient if it can write to std::cout directly.
//
// tinyformat has an interface compatible with the C-printf style,
// and it can writes to a stream or returns a std::string:
//
// std::cout << tfm::printf(
// "%s, %s %d, %.2d:%.2d\n",
// weekday, month, day, hour, min);
//
// or
//
// tfm::format(std::cout,
// "%s, %s %d, %.2d:%.2d\n",
// weekday, month, day, hour, min);
//
// 2. High-performance -- most printed strings are not too long and
// doens't need dynamic memory allocation. Many StringPrintf
// implementations doesn't enforce type-safe, but are
// high-performance, including
//
// https://developers.google.com/optimization/reference/base/stringprintf/
// https://github.com/adobe/chromium/blob/master/base/stringprintf.h
// https://github.com/google/protobuf/blob/master/src/google/protobuf/stubs/stringprintf.h
//
// According to
// https://github.com/c42f/tinyformat#compile-time-and-code-bloat,
// boost::format runs too slow and results in large executable binary
// files. So here we port tinyformat.
#pragma once
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "paddle/utils/string/tinyformat/tinyformat.h" // https://github.com/c42f/tinyformat
namespace paddle {
namespace string {
template <typename... Args>
void Fprintf(std::ostream& out, const char* fmt, const Args&... args) {
tinyformat::vformat(out, fmt, tinyformat::makeFormatList(args...));
}
inline std::string Sprintf() { return ""; }
template <typename... Args>
std::string Sprintf(const Args&... args) {
std::ostringstream oss;
Fprintf(oss, "%s", args...);
return oss.str();
}
template <typename... Args>
std::string Sprintf(const char* fmt, const Args&... args) {
std::ostringstream oss;
Fprintf(oss, fmt, args...);
return oss.str();
}
template <typename... Args>
void Printf(const char* fmt, const Args&... args) {
Fprintf(std::cout, fmt, args...);
}
inline std::string HumanReadableSize(double f_size) {
size_t i = 0;
double orig = f_size;
const std::vector<std::string> units(
{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"});
while (f_size >= 1024) {
f_size /= 1024;
i++;
}
if (i >= units.size()) {
return Sprintf("%fB", orig);
}
return Sprintf("%f%s", f_size, units[i]);
}
} // namespace string
} // namespace paddle
paddle/fluid/string/printf_test.cc → paddle/utils/string/printf_test.cc
浏览文件 @ 6ca49164
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "paddle/fluid/string/printf.h" #include "paddle/utils/string/printf.h"
#include <string> #include <string>
...@@ -25,7 +25,7 @@ TEST(StringPrintf, StringPrintf) { ...@@ -25,7 +25,7 @@ TEST(StringPrintf, StringPrintf) {
int hour = 14; int hour = 14;
int min = 44; int min = 44;
EXPECT_EQ(std::string("Wednesday, July 27, 14:44"), EXPECT_EQ(std::string("Wednesday, July 27, 14:44"),
paddle::string::Sprintf("%s, %s %d, %.2d:%.2d", weekday, month, day, paddle::string::Sprintf(
hour, min)); "%s, %s %d, %.2d:%.2d", weekday, month, day, hour, min));
EXPECT_EQ(std::string(""), paddle::string::Sprintf()); EXPECT_EQ(std::string(""), paddle::string::Sprintf());
} }
paddle/utils/string/split.h 0 → 100644
浏览文件 @ 6ca49164
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <sstream>
#include <string>
#include <vector>
namespace paddle {
namespace string {
static inline std::vector<std::string> Split(std::string const& original,
char separator) {
std::vector<std::string> results;
std::string token;
std::istringstream is(original);
while (std::getline(is, token, separator)) {
if (!token.empty()) {
results.push_back(token);
}
}
return results;
}
} // namespace string
} // namespace paddle
paddle/fluid/string/split_test.cc → paddle/utils/string/split_test.cc
浏览文件 @ 6ca49164
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "paddle/fluid/string/split.h" #include "paddle/utils/string/split.h"
#include <string> #include <string>
......
paddle/fluid/string/string_helper.cc → paddle/utils/string/string_helper.cc
浏览文件 @ 6ca49164
...@@ -12,15 +12,13 @@ ...@@ -12,15 +12,13 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "paddle/fluid/string/string_helper.h" #include "paddle/utils/string/string_helper.h"
#include <ctype.h> #include <ctype.h>
#include <stdio.h> #include <stdio.h>
#include <cstring> #include <cstring>
#include <string> #include <string>
#include "glog/logging.h"
namespace paddle { namespace paddle {
namespace string { namespace string {
...@@ -75,7 +73,9 @@ char* LineFileReader::getdelim(FILE* f, char delim) { ...@@ -75,7 +73,9 @@ char* LineFileReader::getdelim(FILE* f, char delim) {
return _buffer; return _buffer;
} else { } else {
_length = 0; _length = 0;
CHECK(feof(f)); int code = feof(f);
(void)code;
assert(code);
return NULL; return NULL;
} }
#else #else
......
paddle/utils/string/string_helper.h 0 → 100644
浏览文件 @ 6ca49164
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <assert.h>
#include <ctype.h>
#include <stdio.h>
#include <cstring>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
namespace paddle {
namespace string {
inline size_t count_spaces(const char* s) {
size_t count = 0;
while (*s != 0 && isspace(*s++)) {
count++;
}
return count;
}
inline size_t count_nonspaces(const char* s) {
size_t count = 0;
while (*s != 0 && !isspace(*s++)) {
count++;
}
return count;
}
template <class... ARGS>
void format_string_append(std::string& str, // NOLINT
const char* fmt, // NOLINT
ARGS&&... args) {
int len = snprintf(NULL, 0, fmt, args...);
assert(len == 0);
size_t oldlen = str.length();
str.resize(oldlen + len + 1);
int new_len =
snprintf(&str[oldlen], (size_t)len + 1, fmt, args...); // NOLINT
(void)new_len;
assert(new_len == len);
str.resize(oldlen + len);
}
template <class... ARGS>
void format_string_append(std::string& str, // NOLINT
const std::string& fmt, // NOLINT
ARGS&&... args) {
format_string_append(str, fmt.c_str(), args...);
}
template <class... ARGS>
std::string format_string(const char* fmt, ARGS&&... args) {
std::string str;
format_string_append(str, fmt, args...);
return str;
}
template <class... ARGS>
std::string format_string(const std::string& fmt, ARGS&&... args) {
return format_string(fmt.c_str(), args...);
}
// remove leading and tailing spaces
std::string trim_spaces(const std::string& str);
// erase all spaces in str
std::string erase_spaces(const std::string& str);
inline int str_to_float(const char* str, float* v) {
const char* head = str;
char* cursor = NULL;
int index = 0;
while (*(head += count_spaces(head)) != 0) {
v[index++] = std::strtof(head, &cursor);
if (head == cursor) {
break;
}
head = cursor;
}
return index;
}
// checks whether the test string is a suffix of the input string.
bool ends_with(std::string const& input, std::string const& test);
// split string by delim
template <class T = std::string>
std::vector<T> split_string(const std::string& str, const std::string& delim) {
size_t pre_pos = 0;
size_t pos = 0;
std::string tmp_str;
std::vector<T> res_list;
res_list.clear();
if (str.empty()) {
return res_list;
}
while ((pos = str.find(delim, pre_pos)) != std::string::npos) {
tmp_str.assign(str, pre_pos, pos - pre_pos);
res_list.push_back(tmp_str);
pre_pos = pos + 1;
}
tmp_str.assign(str, pre_pos, str.length() - pre_pos);
if (!tmp_str.empty()) {
res_list.push_back(tmp_str);
}
return res_list;
}
// split string by spaces. Leading and tailing spaces are ignored. Consecutive
// spaces are treated as one delim.
template <class T = std::string>
std::vector<T> split_string(const std::string& str) {
std::vector<T> list;
const char* p;
int pre_pos = 0;
int pos = 0;
std::string tmp_str;
if (str.empty()) {
return list;
}
for (p = str.c_str(); *p != 0;) {
if (!isspace(*p)) {
pos = pre_pos;
p++;
while (*p != 0 && !isspace(*p)) {
pos++;
p++;
}
tmp_str.assign(str, pre_pos, pos - pre_pos + 1);
list.push_back(tmp_str);
pre_pos = pos + 1;
} else {
pre_pos++;
p++;
}
}
return list;
}
template <class Container>
std::string join_strings(const Container& strs, char delim) {
std::string str;
size_t i = 0;
for (auto& elem : strs) {
if (i > 0) {
str += delim;
}
std::stringstream ss;
ss << elem;
str += ss.str();
++i;
}
return str;
}
template <class Container>
std::string join_strings(const Container& strs, const std::string& delim) {
std::string str;
size_t i = 0;
for (auto& elem : strs) {
if (i > 0) {
str += delim;
}
std::stringstream ss;
ss << elem;
str += ss.str();
++i;
}
return str;
}
template <class Container, class DelimT, class ConvertFunc>
std::string join_strings(const Container& strs,
DelimT&& delim,
ConvertFunc&& func) {
std::stringstream ss;
size_t i = 0;
for (const auto& elem : strs) {
if (i > 0) {
ss << delim;
}
ss << func(elem);
++i;
}
return ss.str();
}
// A helper class for reading lines from file. A line buffer is maintained. It
// doesn't need to know the maximum possible length of a line.
class LineFileReader {
public:
LineFileReader() {}
LineFileReader(LineFileReader&&) = delete;
LineFileReader(const LineFileReader&) = delete;
~LineFileReader() { ::free(_buffer); }
char* getline(FILE* f) { return this->getdelim(f, '\n'); }
char* getdelim(FILE* f, char delim);
char* get() { return _buffer; }
size_t length() { return _length; }
private:
char* _buffer = NULL;
size_t _buf_size = 0;
size_t _length = 0;
};
} // end namespace string
} // end namespace paddle
paddle/fluid/string/string_helper_test.cc → paddle/utils/string/string_helper_test.cc
浏览文件 @ 6ca49164
...@@ -12,7 +12,7 @@ ...@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "paddle/fluid/string/string_helper.h" #include "paddle/utils/string/string_helper.h"
#include <string> #include <string>
......
paddle/fluid/string/tinyformat/tinyformat.h → paddle/utils/string/tinyformat/tinyformat.h
浏览文件 @ 6ca49164
...@@ -170,7 +170,8 @@ struct is_convertible { ...@@ -170,7 +170,8 @@ struct is_convertible {
// Format the value by casting to type fmtT. This default implementation // Format the value by casting to type fmtT. This default implementation
// should never be called. // should never be called.
template <typename T, typename fmtT, template <typename T,
typename fmtT,
bool convertible = is_convertible<T, fmtT>::value> bool convertible = is_convertible<T, fmtT>::value>
struct formatValueAsType { struct formatValueAsType {
static void invoke(std::ostream & /*out*/, const T & /*value*/) { assert(0); } static void invoke(std::ostream & /*out*/, const T & /*value*/) { assert(0); }
...@@ -240,8 +241,11 @@ TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(char) ...@@ -240,8 +241,11 @@ TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(char)
/// operator<< to format the type T, with special cases for the %c and %p /// operator<< to format the type T, with special cases for the %c and %p
/// conversions. /// conversions.
template <typename T> template <typename T>
inline void formatValue(std::ostream &out, const char * /*fmtBegin*/, inline void formatValue(std::ostream &out,
const char *fmtEnd, int ntrunc, const T &value) { const char * /*fmtBegin*/,
const char *fmtEnd,
int ntrunc,
const T &value) {
// The mess here is to support the %c and %p conversions: if these // The mess here is to support the %c and %p conversions: if these
// conversions are active we try to convert the type to a char or const // conversions are active we try to convert the type to a char or const
// void* respectively and format that instead of the value itself. For the // void* respectively and format that instead of the value itself. For the
...@@ -250,35 +254,39 @@ inline void formatValue(std::ostream &out, const char * /*fmtBegin*/, ...@@ -250,35 +254,39 @@ inline void formatValue(std::ostream &out, const char * /*fmtBegin*/,
const bool canConvertToChar = detail::is_convertible<T, char>::value; const bool canConvertToChar = detail::is_convertible<T, char>::value;
const bool canConvertToVoidPtr = const bool canConvertToVoidPtr =
detail::is_convertible<T, const void *>::value; detail::is_convertible<T, const void *>::value;
if (canConvertToChar && *(fmtEnd - 1) == 'c') if (canConvertToChar && *(fmtEnd - 1) == 'c') {
detail::formatValueAsType<T, char>::invoke(out, value); detail::formatValueAsType<T, char>::invoke(out, value);
else if (canConvertToVoidPtr && *(fmtEnd - 1) == 'p') } else if (canConvertToVoidPtr && *(fmtEnd - 1) == 'p') {
detail::formatValueAsType<T, const void *>::invoke(out, value); detail::formatValueAsType<T, const void *>::invoke(out, value);
else if (ntrunc >= 0) { } else if (ntrunc >= 0) {
// Take care not to overread C strings in truncating conversions like // Take care not to overread C strings in truncating conversions like
// "%.4s" where at most 4 characters may be read. // "%.4s" where at most 4 characters may be read.
detail::formatTruncated(out, value, ntrunc); detail::formatTruncated(out, value, ntrunc);
} else } else {
out << value; out << value;
}
} }
// Overloaded version for char types to support printing as an integer // Overloaded version for char types to support printing as an integer
#define TINYFORMAT_DEFINE_FORMATVALUE_CHAR(charType) \ #define TINYFORMAT_DEFINE_FORMATVALUE_CHAR(charType) \
inline void formatValue(std::ostream &out, const char * /*fmtBegin*/, \ inline void formatValue(std::ostream &out, \
const char *fmtEnd, int /**/, charType value) { \ const char * /*fmtBegin*/, \
switch (*(fmtEnd - 1)) { \ const char *fmtEnd, \
case 'u': \ int /**/, \
case 'd': \ charType value) { \
case 'i': \ switch (*(fmtEnd - 1)) { \
case 'o': \ case 'u': \
case 'X': \ case 'd': \
case 'x': \ case 'i': \
out << static_cast<int>(value); \ case 'o': \
break; \ case 'X': \
default: \ case 'x': \
out << value; \ out << static_cast<int>(value); \
break; \ break; \
} \ default: \
out << value; \
break; \
} \
} }
// per 3.9.1: char, signed char and unsigned char are all distinct types // per 3.9.1: char, signed char and unsigned char are all distinct types
TINYFORMAT_DEFINE_FORMATVALUE_CHAR(char) TINYFORMAT_DEFINE_FORMATVALUE_CHAR(char)
...@@ -466,7 +474,7 @@ cog.outl('#define TINYFORMAT_FOREACH_ARGNUM(m) \\\n ' + ...@@ -466,7 +474,7 @@ cog.outl('#define TINYFORMAT_FOREACH_ARGNUM(m) \\\n ' +
#define TINYFORMAT_FOREACH_ARGNUM(m) \ #define TINYFORMAT_FOREACH_ARGNUM(m) \
m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) m(10) m(11) m(12) m(13) m(14) \ m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) m(10) m(11) m(12) m(13) m(14) \
m(15) m(16) m(15) m(16)
//[[[end]]] // [[[end]]]
namespace detail { namespace detail {
...@@ -476,15 +484,17 @@ namespace detail { ...@@ -476,15 +484,17 @@ namespace detail {
// whereas a naive implementation based on inheritance does not. // whereas a naive implementation based on inheritance does not.
class FormatArg { class FormatArg {
public: public:
FormatArg() {} FormatArg() {} // NOLINT
template <typename T> template <typename T>
FormatArg(const T &value) FormatArg(const T &value) // NOLINT
: m_value(static_cast<const void *>(&value)), : m_value(static_cast<const void *>(&value)),
m_formatImpl(&formatImpl<T>), m_formatImpl(&formatImpl<T>),
m_toIntImpl(&toIntImpl<T>) {} m_toIntImpl(&toIntImpl<T>) {}
void format(std::ostream &out, const char *fmtBegin, const char *fmtEnd, void format(std::ostream &out,
const char *fmtBegin,
const char *fmtEnd,
int ntrunc) const { int ntrunc) const {
m_formatImpl(out, fmtBegin, fmtEnd, ntrunc, m_value); m_formatImpl(out, fmtBegin, fmtEnd, ntrunc, m_value);
} }
...@@ -493,8 +503,11 @@ class FormatArg { ...@@ -493,8 +503,11 @@ class FormatArg {
private: private:
template <typename T> template <typename T>
static void formatImpl(std::ostream &out, const char *fmtBegin, static void formatImpl(std::ostream &out,
const char *fmtEnd, int ntrunc, const void *value) { const char *fmtBegin,
const char *fmtEnd,
int ntrunc,
const void *value) {
formatValue(out, fmtBegin, fmtEnd, ntrunc, *static_cast<const T *>(value)); formatValue(out, fmtBegin, fmtEnd, ntrunc, *static_cast<const T *>(value));
} }
...@@ -504,14 +517,17 @@ class FormatArg { ...@@ -504,14 +517,17 @@ class FormatArg {
} }
const void *m_value; const void *m_value;
void (*m_formatImpl)(std::ostream &out, const char *fmtBegin, void (*m_formatImpl)(std::ostream &out,
const char *fmtEnd, int ntrunc, const void *value); const char *fmtBegin,
const char *fmtEnd,
int ntrunc,
const void *value);
int (*m_toIntImpl)(const void *value); int (*m_toIntImpl)(const void *value);
}; };
// Parse and return an integer from the string c, as atoi() // Parse and return an integer from the string c, as atoi()
// On return, c is set to one past the end of the integer. // On return, c is set to one past the end of the integer.
inline int parseIntAndAdvance(const char *&c) { inline int parseIntAndAdvance(const char *&c) { // NOLINT
int i = 0; int i = 0;
for (; *c >= '0' && *c <= '9'; ++c) i = 10 * i + (*c - '0'); for (; *c >= '0' && *c <= '9'; ++c) i = 10 * i + (*c - '0');
return i; return i;
...@@ -553,11 +569,13 @@ inline const char *printFormatStringLiteral(std::ostream &out, ...@@ -553,11 +569,13 @@ inline const char *printFormatStringLiteral(std::ostream &out,
// and ntrunc (for truncating conversions). argIndex is incremented if // and ntrunc (for truncating conversions). argIndex is incremented if
// necessary to pull out variable width and precision . The function returns a // necessary to pull out variable width and precision . The function returns a
// pointer to the character after the end of the current format spec. // pointer to the character after the end of the current format spec.
inline const char *streamStateFromFormat(std::ostream &out, inline const char *streamStateFromFormat(std::ostream &out, // NOLINT
bool &spacePadPositive, int &ntrunc, bool &spacePadPositive, // NOLINT
int &ntrunc, // NOLINT
const char *fmtStart, const char *fmtStart,
const detail::FormatArg *formatters, const detail::FormatArg *formatters,
int &argIndex, int numFormatters) { int &argIndex, // NOLINT
int numFormatters) {
if (*fmtStart != '%') { if (*fmtStart != '%') {
TINYFORMAT_ERROR( TINYFORMAT_ERROR(
"tinyformat: Not enough conversion specifiers in format string"); "tinyformat: Not enough conversion specifiers in format string");
...@@ -733,8 +751,10 @@ inline const char *streamStateFromFormat(std::ostream &out, ...@@ -733,8 +751,10 @@ inline const char *streamStateFromFormat(std::ostream &out,
} }
//------------------------------------------------------------------------------ //------------------------------------------------------------------------------
inline void formatImpl(std::ostream &out, const char *fmt, inline void formatImpl(std::ostream &out,
const detail::FormatArg *formatters, int numFormatters) { const char *fmt,
const detail::FormatArg *formatters,
int numFormatters) {
// Saved stream state // Saved stream state
std::streamsize origWidth = out.width(); std::streamsize origWidth = out.width();
std::streamsize origPrecision = out.precision(); std::streamsize origPrecision = out.precision();
...@@ -746,9 +766,13 @@ inline void formatImpl(std::ostream &out, const char *fmt, ...@@ -746,9 +766,13 @@ inline void formatImpl(std::ostream &out, const char *fmt,
fmt = printFormatStringLiteral(out, fmt); fmt = printFormatStringLiteral(out, fmt);
bool spacePadPositive = false; bool spacePadPositive = false;
int ntrunc = -1; int ntrunc = -1;
const char *fmtEnd = const char *fmtEnd = streamStateFromFormat(out,
streamStateFromFormat(out, spacePadPositive, ntrunc, fmt, formatters, spacePadPositive,
argIndex, numFormatters); ntrunc,
fmt,
formatters,
argIndex,
numFormatters);
if (argIndex >= numFormatters) { if (argIndex >= numFormatters) {
// Check args remain after reading any variable width/precision // Check args remain after reading any variable width/precision
TINYFORMAT_ERROR("tinyformat: Not enough format arguments"); TINYFORMAT_ERROR("tinyformat: Not enough format arguments");
...@@ -756,9 +780,9 @@ inline void formatImpl(std::ostream &out, const char *fmt, ...@@ -756,9 +780,9 @@ inline void formatImpl(std::ostream &out, const char *fmt,
} }
const FormatArg &arg = formatters[argIndex]; const FormatArg &arg = formatters[argIndex];
// Format the arg into the stream. // Format the arg into the stream.
if (!spacePadPositive) if (!spacePadPositive) {
arg.format(out, fmt, fmtEnd, ntrunc); arg.format(out, fmt, fmtEnd, ntrunc);
else { } else {
// The following is a special case with no direct correspondence // The following is a special case with no direct correspondence
// between stream formatting and the printf() behaviour. Simulate // between stream formatting and the printf() behaviour. Simulate
// it crudely by formatting into a temporary string stream and // it crudely by formatting into a temporary string stream and
...@@ -801,7 +825,8 @@ class FormatList { ...@@ -801,7 +825,8 @@ class FormatList {
FormatList(detail::FormatArg *formatters, int N) FormatList(detail::FormatArg *formatters, int N)
: m_formatters(formatters), m_N(N) {} : m_formatters(formatters), m_N(N) {}
friend void vformat(std::ostream &out, const char *fmt, friend void vformat(std::ostream &out,
const char *fmt,
const FormatList &list); const FormatList &list);
private: private:
...@@ -819,7 +844,7 @@ template <int N> ...@@ -819,7 +844,7 @@ template <int N>
class FormatListN : public FormatList { class FormatListN : public FormatList {
public: public:
template <typename... Args> template <typename... Args>
FormatListN(const Args &... args) FormatListN(const Args &... args) // NOLINT
: FormatList(&m_formatterStore[0], N), : FormatList(&m_formatterStore[0], N),
m_formatterStore{FormatArg(args)...} { m_formatterStore{FormatArg(args)...} {
static_assert(sizeof...(args) == N, "Number of args must be N"); static_assert(sizeof...(args) == N, "Number of args must be N");
...@@ -850,7 +875,7 @@ class FormatListN<0> : public FormatList { ...@@ -850,7 +875,7 @@ class FormatListN<0> : public FormatList {
template <typename... Args> template <typename... Args>
detail::FormatListN<sizeof...(Args)> makeFormatList(const Args &... args) { detail::FormatListN<sizeof...(Args)> makeFormatList(const Args &... args) {
return detail::FormatListN<sizeof...(args)>(args...); return detail::FormatListN<sizeof...(args)>(args...);
} } // NOLINT
/// Format list of arguments to the stream according to the given format string. /// Format list of arguments to the stream according to the given format string.
/// ///
......
paddle/utils/string/to_string.h 0 → 100644
浏览文件 @ 6ca49164
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <sstream>
#include <string>
#include <type_traits>
#include <typeindex>
#include <vector>
namespace paddle {
namespace string {
inline std::ostream& operator<<(std::ostream& s, const std::type_index& t) {
s << t.name();
return s;
}
template <typename T,
typename std::enable_if<!std::is_enum<T>::value, int>::type = 0>
inline std::string to_string(T v) {
std::ostringstream sout;
sout << v;
return sout.str();
}
template <typename T,
typename std::enable_if<std::is_enum<T>::value, int>::type = 0>
inline std::string to_string(T v) {
return std::to_string(static_cast<int>(v));
}
template <>
inline std::string to_string(std::type_index t) {
return t.name();
}
// Faster std::string/const char* type
template <>
inline std::string to_string(std::string v) {
return v;
}
template <>
inline std::string to_string(const char* v) {
return std::string(v);
}
} // namespace string
} // namespace paddle
paddle/fluid/string/to_string_test.cc → paddle/utils/string/to_string_test.cc
浏览文件 @ 6ca49164
...@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. ...@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and See the License for the specific language governing permissions and
limitations under the License. */ limitations under the License. */
#include "paddle/fluid/string/to_string.h" #include "paddle/utils/string/to_string.h"
#include <gtest/gtest.h> #include <gtest/gtest.h>
constexpr char kOutputString[] = "User Defined Output"; constexpr char kOutputString[] = "User Defined Output";
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或