Skip to content

P
Paddle

Crayon鑫 / Paddle
与 Fork 源项目一致

Fork自 PaddlePaddle / Paddle

通知 1
Star 1
Fork 0
P
Paddle
未验证 提交 6ca49164 编写于 作者: X xiongkun 提交者： GitHub
浏览文件
操作

[PTen] Migrate string tinyformat errors and part of enforce into pten (#39051) 

* transfer: string tinyformat errors and part of enforce into pten

* remove comment

* fix by code review

* assert is not compile in -DNDEBUG

* add string as dependences of paddle_inference
上级 d43655ba
展开全部 隐藏空白更改
内联 并排
Showing with 1664 addition and 1320 deletion
cmake/inference_lib.cmake
浏览文件 @ 6ca49164
......@@ -189,6 +189,7 @@ copy(inference_lib_dist
copy_part_of_thrid_party(inference_lib_dist ${PADDLE_INFERENCE_INSTALL_DIR})
set(src_dir "${PADDLE_SOURCE_DIR}/paddle/fluid")
if(WIN32)
if(WITH_STATIC_LIB)
set(paddle_inference_lib $<TARGET_FILE_DIR:paddle_inference>/libpaddle_inference.lib
......@@ -304,7 +305,7 @@ copy(fluid_lib_dist
)
set(module "platform")
set(platform_lib_deps profiler_proto error_codes_proto)
set(platform_lib_deps profiler_proto errors)
if(WITH_GPU)
set(platform_lib_deps ${platform_lib_deps} external_error_proto)
endif(WITH_GPU)
......@@ -317,7 +318,7 @@ copy(fluid_lib_dist
set(module "string")
copy(fluid_lib_dist
SRCS ${src_dir}/${module}/*.h ${src_dir}/${module}/tinyformat/*.h
SRCS ${PADDLE_SOURCE_DIR}/paddle/utils/${module}/*.h ${PADDLE_SOURCE_DIR}/paddle/utils/${module}/tinyformat/*.h
DSTS ${dst_dir}/${module} ${dst_dir}/${module}/tinyformat
)
......
paddle/CMakeLists.txt
浏览文件 @ 6ca49164
add_subdirectory(utils)
add_subdirectory(scripts)
add_subdirectory(testing)
set(PYTHON_TESTS_DIR ${PADDLE_BINARY_DIR}/python/paddle/fluid/tests CACHE INTERNAL "python tests directory")
......
paddle/fluid/CMakeLists.txt
浏览文件 @ 6ca49164
......@@ -4,7 +4,6 @@ add_subdirectory(distributed)
add_subdirectory(framework)
add_subdirectory(imperative)
add_subdirectory(operators)
add_subdirectory(string)
add_subdirectory(pybind)
add_subdirectory(eager)
# NOTE: please add subdirectory inference at last.
......
paddle/fluid/framework/op_call_stack.h
浏览文件 @ 6ca49164
......@@ -19,12 +19,6 @@ limitations under the License. */
#include "paddle/fluid/framework/type_defs.h"
#include "paddle/fluid/platform/enforce.h"
namespace paddle {
namespace platform {
struct EnforceNotMet;
} // namespace platform
} // namespace paddle
namespace paddle {
namespace framework {
......
paddle/fluid/framework/op_proto_maker_test.cc
浏览文件 @ 6ca49164
......@@ -18,12 +18,6 @@ limitations under the License. */
#include "gtest/gtest-test-part.h"
#include "gtest/gtest.h"
namespace paddle {
namespace platform {
struct EnforceNotMet;
} // namespace platform
} // namespace paddle
class TestAttrProtoMaker : public paddle::framework::OpProtoAndCheckerMaker {
public:
void Make() {
......
paddle/fluid/framework/type_defs.h
浏览文件 @ 6ca49164
......@@ -21,8 +21,8 @@ limitations under the License. */
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include "paddle/fluid/imperative/type_defs.h"
#include "paddle/fluid/platform/variant.h"
#include "paddle/pten/core/type_defs.h"
#include "paddle/utils/small_vector.h"
namespace paddle {
......@@ -39,14 +39,6 @@ class InferNoNeedBufferVarsFN;
using VariableNameMap = std::map<std::string, std::vector<std::string>>;
using VariableValueMap = std::map<std::string, std::vector<Variable*>>;
// The order should be as same as framework.proto
using Attribute = boost::variant<
boost::blank, int, float, std::string, std::vector<int>, std::vector<float>,
std::vector<std::string>, bool, std::vector<bool>, BlockDesc*, int64_t,
std::vector<BlockDesc*>, std::vector<int64_t>, std::vector<double>>;
using AttributeMap = std::unordered_map<std::string, Attribute>;
#ifdef PADDLE_WITH_ASCEND_CL
using NPUAttribute =
boost::variant<boost::blank, int, float, std::string, std::vector<int>,
......
paddle/fluid/imperative/type_defs.h
浏览文件 @ 6ca49164
......@@ -13,47 +13,4 @@ See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <map>
#include <memory>
#include <string>
#include <vector>
namespace paddle {
namespace imperative {
class VariableWrapper;
class SavedVariableWrapperList;
class VarBase;
class OpBase;
class GradOpNode;
class Tracer;
using WeakNameVarBaseMap =
std::map<std::string, std::vector<std::weak_ptr<VarBase>>>;
namespace details {
template <typename T>
struct NameVarMapTrait {};
template <>
struct NameVarMapTrait<VarBase> {
using Type = std::map<std::string, std::vector<std::shared_ptr<VarBase>>>;
};
template <>
struct NameVarMapTrait<VariableWrapper> {
using Type = std::map<std::string, SavedVariableWrapperList>;
};
} // namespace details
template <typename T>
using NameVarMap = typename details::NameVarMapTrait<T>::Type;
using NameVarBaseMap = NameVarMap<VarBase>;
using NameVariableWrapperMap = NameVarMap<VariableWrapper>;
using VariableWrapperList = std::vector<std::shared_ptr<VariableWrapper>>;
} // namespace imperative
} // namespace paddle
#include "paddle/pten/core/type_defs.h"
paddle/fluid/inference/CMakeLists.txt
浏览文件 @ 6ca49164
......@@ -36,6 +36,7 @@ endif()
# fluid_modules exclude API-interface of inference/api and inference/capi_exp
get_property(fluid_modules GLOBAL PROPERTY FLUID_MODULES)
get_property(pten_modules GLOBAL PROPERTY PTEN_MODULES)
set(utils_modules stringpiece pretty_log string_helper)
add_subdirectory(api)
......@@ -46,9 +47,9 @@ set(STATIC_INFERENCE_API paddle_inference_api analysis_predictor
analysis_config paddle_pass_builder activation_functions ${mkldnn_quantizer_cfg})
#TODO(wilber, T8T9): Do we still need to support windows gpu static library?
if(WIN32 AND WITH_GPU)
cc_library(paddle_inference DEPS ${fluid_modules} ${pten_modules} ${STATIC_INFERENCE_API})
cc_library(paddle_inference DEPS ${fluid_modules} ${pten_modules} ${STATIC_INFERENCE_API} ${utils_modules})
else()
create_static_lib(paddle_inference ${fluid_modules} ${pten_modules} ${STATIC_INFERENCE_API})
create_static_lib(paddle_inference ${fluid_modules} ${pten_modules} ${STATIC_INFERENCE_API} ${utils_modules})
if(WITH_IPU)
target_link_libraries(paddle_inference -Wl,--allow-multiple-definition popart_canonicalization_utils)
endif()
......
paddle/fluid/platform/CMakeLists.txt
浏览文件 @ 6ca49164
proto_library(profiler_proto SRCS profiler.proto DEPS framework_proto simple_threadpool)
proto_library(error_codes_proto SRCS error_codes.proto)
if(WITH_GPU)
proto_library(external_error_proto SRCS external_error.proto)
endif(WITH_GPU)
if (WITH_PYTHON)
py_proto_compile(profiler_py_proto SRCS profiler.proto)
add_custom_target(profiler_py_proto_init ALL COMMAND ${CMAKE_COMMAND} -E touch __init__.py)
......@@ -28,10 +26,9 @@ endif()
cc_library(flags SRCS flags.cc DEPS gflags boost)
cc_library(denormal SRCS denormal.cc DEPS)
cc_library(errors SRCS errors.cc DEPS error_codes_proto)
cc_test(errors_test SRCS errors_test.cc DEPS errors enforce)
set(enforce_deps flags errors boost flags)
set(enforce_deps flags errors boost flags pten_enforce)
if(WITH_GPU)
set(enforce_deps ${enforce_deps} external_error_proto)
endif()
......
paddle/fluid/platform/enforce.h
浏览文件 @ 6ca49164
此差异已折叠。
paddle/fluid/platform/errors.h
浏览文件 @ 6ca49164
......@@ -13,76 +13,10 @@ See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <memory>
#include <stdexcept>
#include <string>
#include <tuple>
#include <type_traits>
#include "paddle/fluid/platform/error_codes.pb.h"
#include "paddle/fluid/string/printf.h"
#include "paddle/pten/core/errors.h"
namespace paddle {
namespace platform {
typedef ::paddle::platform::error::Code Code;
class ErrorSummary {
public:
// Note(chenweihang): Final deprecated constructor
// This constructor is used to be compatible with
// current existing untyped PADDLE_ENFORCE_*
// PADDLE_ENFORCE
// Note(chenweihang): Windows openblas need this
// constructor for compiling PADDLE_ENFORCE in *.cu,
// this is a bug cause we can't remove this
// constructor now.
template <typename... Args>
explicit ErrorSummary(Args... args) {
code_ = paddle::platform::error::LEGACY;
msg_ = paddle::string::Sprintf(args...);
}
// Note(chenweihang): Only recommended constructor
// No longer supports PADDLE_ENFORCE without type or without error message
explicit ErrorSummary(Code code, std::string msg) : code_(code), msg_(msg) {}
Code code() const { return code_; }
const std::string& error_message() const { return msg_; }
std::string to_string() const;
private:
Code code_;
std::string msg_;
};
namespace errors {
#define REGISTER_ERROR(FUNC, CONST, ...) \
template <typename... Args> \
::paddle::platform::ErrorSummary FUNC(Args... args) { \
return ::paddle::platform::ErrorSummary( \
::paddle::platform::error::CONST, ::paddle::string::Sprintf(args...)); \
}
REGISTER_ERROR(InvalidArgument, INVALID_ARGUMENT)
REGISTER_ERROR(NotFound, NOT_FOUND)
REGISTER_ERROR(OutOfRange, OUT_OF_RANGE)
REGISTER_ERROR(AlreadyExists, ALREADY_EXISTS)
REGISTER_ERROR(ResourceExhausted, RESOURCE_EXHAUSTED)
REGISTER_ERROR(PreconditionNotMet, PRECONDITION_NOT_MET)
REGISTER_ERROR(PermissionDenied, PERMISSION_DENIED)
REGISTER_ERROR(ExecutionTimeout, EXECUTION_TIMEOUT)
REGISTER_ERROR(Unimplemented, UNIMPLEMENTED)
REGISTER_ERROR(Unavailable, UNAVAILABLE)
REGISTER_ERROR(Fatal, FATAL)
REGISTER_ERROR(External, EXTERNAL)
#undef REGISTER_ERROR
} // namespace errors
} // namespace platform
} // namespace paddle
namespace errors = ::pten::errors;
using error = ::pten::ErrorCode;
}
}
paddle/fluid/string/piece.h
浏览文件 @ 6ca49164
......@@ -17,89 +17,4 @@
#include <ostream>
#include <string>
namespace paddle {
namespace string {
// Piece points into a std::string object but doesn't own the
// string. It is for efficient access to strings. Like Go's string
// type. Not that Piece doesn't mutate the underlying string,
// so it is thread-safe given that the underlying string doesn't
// change. Because Piece contains a little data members, and
// its syntax is simple as it doesn't own/manage the string, it is
// cheap to construct Pieces and pass them around.
class Piece {
public:
static const size_t npos = static_cast<size_t>(-1);
// We provide non-explicit singleton constructors so users can
// pass in a "const char*" or a "string" wherever a "Piece"
// is expected. These constructors ensure that if data_ is NULL,
// size_ is 0.
Piece();
Piece(const char* d, size_t n);
Piece(const char* d); // NOLINT: accept C string into Piece.
Piece(const std::string& s); // NOLINT: accept C++ string into Piece.
const char* data() const { return data_; }
size_t len() const { return size_; }
char operator[](size_t n) const;
// Piece doesn't own the string, so both iterator and const
// iterator are const char* indeed.
typedef const char* const_iterator;
typedef const char* iterator;
iterator begin() const { return data_; }
iterator end() const { return data_ + size_; }
// Return a string that contains the copy of the referenced data.
std::string ToString() const { return std::string(data_, size_); }
private:
const char* data_;
size_t size_;
// Intentionally copyable
};
int Compare(Piece a, Piece b);
bool operator==(Piece x, Piece y);
bool operator!=(Piece x, Piece y);
bool operator<(Piece x, Piece y);
bool operator>(Piece x, Piece y);
bool operator<=(Piece x, Piece y);
bool operator>=(Piece x, Piece y);
bool HasPrefix(Piece s, Piece prefix);
bool HasSuffix(Piece s, Piece suffix);
Piece SkipPrefix(Piece s, size_t n);
Piece SkipSuffix(Piece s, size_t n);
// Skip the prefix (or suffix) if it matches with the string.
Piece TrimPrefix(Piece s, Piece prefix);
Piece TrimSuffix(Piece s, Piece suffix);
// Returns if s contains sub. Any s except for empty s contains an
// empty sub.
bool Contains(Piece s, Piece sub);
// Return the first occurrence of sub in s, or npos. If both s and
// sub is empty, it returns npos; otherwise, if only sub is empty, it
// returns 0.
size_t Index(Piece s, Piece sub);
// Return the first occurrence of c in s[pos:end], or npos.
size_t Find(Piece s, char c, size_t pos);
// Search range is [0..pos] inclusive. If pos == npos, search everything.
size_t RFind(Piece s, char c, size_t pos);
Piece SubStr(Piece s, size_t pos, size_t n);
// allow Piece to be logged
std::ostream& operator<<(std::ostream& o, Piece piece);
} // namespace string
} // namespace paddle
#include "paddle/utils/string/piece.h"
paddle/fluid/string/pretty_log.h
浏览文件 @ 6ca49164
......@@ -19,70 +19,4 @@
#include <utility>
#include "gflags/gflags.h"
#include "paddle/fluid/string/printf.h"
DECLARE_bool(color);
namespace paddle {
namespace string {
inline std::string black() { return FLAGS_color ? "\e[30m" : ""; }
inline std::string red() { return FLAGS_color ? "\e[31m" : ""; }
inline std::string b_red() { return FLAGS_color ? "\e[41m" : ""; }
inline std::string green() { return FLAGS_color ? "\e[32m" : ""; }
inline std::string yellow() { return FLAGS_color ? "\e[33m" : ""; }
inline std::string blue() { return FLAGS_color ? "\e[34m" : ""; }
inline std::string purple() { return FLAGS_color ? "\e[35m" : ""; }
inline std::string cyan() { return FLAGS_color ? "\e[36m" : ""; }
inline std::string light_gray() { return FLAGS_color ? "\e[37m" : ""; }
inline std::string white() { return FLAGS_color ? "\e[37m" : ""; }
inline std::string light_red() { return FLAGS_color ? "\e[91m" : ""; }
inline std::string dim() { return FLAGS_color ? "\e[2m" : ""; }
inline std::string bold() { return FLAGS_color ? "\e[1m" : ""; }
inline std::string underline() { return FLAGS_color ? "\e[4m" : ""; }
inline std::string blink() { return FLAGS_color ? "\e[5m" : ""; }
inline std::string reset() { return FLAGS_color ? "\e[0m" : ""; }
using TextBlock = std::pair<std::string, std::string>;
struct Style {
static std::string info() { return black(); }
static std::string warn() { return b_red(); }
static std::string suc() { return green(); }
static std::string H1() { return bold() + purple(); }
static std::string H2() { return green(); }
static std::string H3() { return green(); }
static std::string detail() { return light_gray(); }
};
template <typename... Args>
static void PrettyLogEndl(const std::string &style, const char *fmt,
const Args &... args) {
std::cerr << style << Sprintf(fmt, args...) << reset() << std::endl;
}
template <typename... Args>
static void PrettyLog(const std::string &style, const char *fmt,
const Args &... args) {
std::cerr << style << Sprintf(fmt, args...) << reset();
}
template <typename... Args>
static void PrettyLogInfo(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::info(), fmt, args...);
}
template <typename... Args>
static void PrettyLogDetail(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::detail(), fmt, args...);
}
template <typename... Args>
static void PrettyLogH1(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::H1(), fmt, args...);
}
template <typename... Args>
static void PrettyLogH2(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::H2(), fmt, args...);
}
} // namespace string
} // namespace paddle
#include "paddle/utils/string/pretty_log.h"
paddle/fluid/string/printf.h
浏览文件 @ 6ca49164
......@@ -12,113 +12,5 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Compared with std::stringstream, there are primary purpose of
// string::Printf:
//
// 1. Type-safe printing, with why and how explained in
// http://www.drdobbs.com/stringprintf-a-typesafe-printf-family-fo/184401999.
// Implementation includes
//
// https://github.com/c42f/tinyformat
// boost::format
// std::stringstream
//
// std::stringstream is not convenient enough in many cases. For example:
//
// std::cout << std::setprecision(2) << std::fixed << 1.23456 << "\n";
//
// boost::format is the most convenient one. We can have
//
// std::cout << format("%2% %1%") % 36 % 77;
//
// or
//
// format fmter("%2% %1%");
// fmter % 36; fmter % 77;
// std::cout << fmter.c_str();
//
// But the overloading of % might be overkilling and it would be
// more efficient if it can write to std::cout directly.
//
// tinyformat has an interface compatible with the C-printf style,
// and it can writes to a stream or returns a std::string:
//
// std::cout << tfm::printf(
// "%s, %s %d, %.2d:%.2d\n",
// weekday, month, day, hour, min);
//
// or
//
// tfm::format(std::cout,
// "%s, %s %d, %.2d:%.2d\n",
// weekday, month, day, hour, min);
//
// 2. High-performance -- most printed strings are not too long and
// doens't need dynamic memory allocation. Many StringPrintf
// implementations doesn't enforce type-safe, but are
// high-performance, including
//
// https://developers.google.com/optimization/reference/base/stringprintf/
// https://github.com/adobe/chromium/blob/master/base/stringprintf.h
// https://github.com/google/protobuf/blob/master/src/google/protobuf/stubs/stringprintf.h
//
// According to
// https://github.com/c42f/tinyformat#compile-time-and-code-bloat,
// boost::format runs too slow and results in large executable binary
// files. So here we port tinyformat.
#pragma once
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "tinyformat/tinyformat.h" // https://github.com/c42f/tinyformat
namespace paddle {
namespace string {
template <typename... Args>
void Fprintf(std::ostream& out, const char* fmt, const Args&... args) {
tinyformat::vformat(out, fmt, tinyformat::makeFormatList(args...));
}
inline std::string Sprintf() { return ""; }
template <typename... Args>
std::string Sprintf(const Args&... args) {
std::ostringstream oss;
Fprintf(oss, "%s", args...);
return oss.str();
}
template <typename... Args>
std::string Sprintf(const char* fmt, const Args&... args) {
std::ostringstream oss;
Fprintf(oss, fmt, args...);
return oss.str();
}
template <typename... Args>
void Printf(const char* fmt, const Args&... args) {
Fprintf(std::cout, fmt, args...);
}
inline std::string HumanReadableSize(double f_size) {
size_t i = 0;
double orig = f_size;
const std::vector<std::string> units(
{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"});
while (f_size >= 1024) {
f_size /= 1024;
i++;
}
if (i >= units.size()) {
return Sprintf("%fB", orig);
}
return Sprintf("%f%s", f_size, units[i]);
}
} // namespace string
} // namespace paddle
#include "paddle/utils/string/printf.h"
paddle/fluid/string/split.h
浏览文件 @ 6ca49164
......@@ -17,21 +17,4 @@ limitations under the License. */
#include <string>
#include <vector>
namespace paddle {
namespace string {
static inline std::vector<std::string> Split(std::string const& original,
char separator) {
std::vector<std::string> results;
std::string token;
std::istringstream is(original);
while (std::getline(is, token, separator)) {
if (!token.empty()) {
results.push_back(token);
}
}
return results;
}
} // namespace string
} // namespace paddle
#include "paddle/utils/string/split.h"
paddle/fluid/string/string_helper.h
浏览文件 @ 6ca49164
......@@ -14,219 +14,4 @@
#pragma once
#include <ctype.h>
#include <stdio.h>
#include <cstring>
#include <string>
#include <utility>
#include <vector>
#include "glog/logging.h"
namespace paddle {
namespace string {
inline size_t count_spaces(const char* s) {
size_t count = 0;
while (*s != 0 && isspace(*s++)) {
count++;
}
return count;
}
inline size_t count_nonspaces(const char* s) {
size_t count = 0;
while (*s != 0 && !isspace(*s++)) {
count++;
}
return count;
}
template <class... ARGS>
void format_string_append(std::string& str, const char* fmt, // NOLINT
ARGS&&... args) {
int len = snprintf(NULL, 0, fmt, args...);
CHECK_GE(len, 0);
size_t oldlen = str.length();
str.resize(oldlen + len + 1);
CHECK(snprintf(&str[oldlen], (size_t)len + 1, fmt, args...) == // NOLINT
len);
str.resize(oldlen + len);
}
template <class... ARGS>
void format_string_append(std::string& str, const std::string& fmt, // NOLINT
ARGS&&... args) {
format_string_append(str, fmt.c_str(), args...);
}
template <class... ARGS>
std::string format_string(const char* fmt, ARGS&&... args) {
std::string str;
format_string_append(str, fmt, args...);
return str;
}
template <class... ARGS>
std::string format_string(const std::string& fmt, ARGS&&... args) {
return format_string(fmt.c_str(), args...);
}
// remove leading and tailing spaces
std::string trim_spaces(const std::string& str);
// erase all spaces in str
std::string erase_spaces(const std::string& str);
inline int str_to_float(const char* str, float* v) {
const char* head = str;
char* cursor = NULL;
int index = 0;
while (*(head += count_spaces(head)) != 0) {
v[index++] = std::strtof(head, &cursor);
if (head == cursor) {
break;
}
head = cursor;
}
return index;
}
// checks whether the test string is a suffix of the input string.
bool ends_with(std::string const& input, std::string const& test);
// split string by delim
template <class T = std::string>
std::vector<T> split_string(const std::string& str, const std::string& delim) {
size_t pre_pos = 0;
size_t pos = 0;
std::string tmp_str;
std::vector<T> res_list;
res_list.clear();
if (str.empty()) {
return res_list;
}
while ((pos = str.find(delim, pre_pos)) != std::string::npos) {
tmp_str.assign(str, pre_pos, pos - pre_pos);
res_list.push_back(tmp_str);
pre_pos = pos + 1;
}
tmp_str.assign(str, pre_pos, str.length() - pre_pos);
if (!tmp_str.empty()) {
res_list.push_back(tmp_str);
}
return res_list;
}
// split string by spaces. Leading and tailing spaces are ignored. Consecutive
// spaces are treated as one delim.
template <class T = std::string>
std::vector<T> split_string(const std::string& str) {
std::vector<T> list;
const char* p;
int pre_pos = 0;
int pos = 0;
std::string tmp_str;
if (str.empty()) {
return list;
}
for (p = str.c_str(); *p != 0;) {
if (!isspace(*p)) {
pos = pre_pos;
p++;
while (*p != 0 && !isspace(*p)) {
pos++;
p++;
}
tmp_str.assign(str, pre_pos, pos - pre_pos + 1);
list.push_back(tmp_str);
pre_pos = pos + 1;
} else {
pre_pos++;
p++;
}
}
return list;
}
template <class Container>
std::string join_strings(const Container& strs, char delim) {
std::string str;
size_t i = 0;
for (auto& elem : strs) {
if (i > 0) {
str += delim;
}
std::stringstream ss;
ss << elem;
str += ss.str();
++i;
}
return str;
}
template <class Container>
std::string join_strings(const Container& strs, const std::string& delim) {
std::string str;
size_t i = 0;
for (auto& elem : strs) {
if (i > 0) {
str += delim;
}
std::stringstream ss;
ss << elem;
str += ss.str();
++i;
}
return str;
}
template <class Container, class DelimT, class ConvertFunc>
std::string join_strings(const Container& strs, DelimT&& delim,
ConvertFunc&& func) {
std::stringstream ss;
size_t i = 0;
for (const auto& elem : strs) {
if (i > 0) {
ss << delim;
}
ss << func(elem);
++i;
}
return ss.str();
}
// A helper class for reading lines from file. A line buffer is maintained. It
// doesn't need to know the maximum possible length of a line.
class LineFileReader {
public:
LineFileReader() {}
LineFileReader(LineFileReader&&) = delete;
LineFileReader(const LineFileReader&) = delete;
~LineFileReader() { ::free(_buffer); }
char* getline(FILE* f) { return this->getdelim(f, '\n'); }
char* getdelim(FILE* f, char delim);
char* get() { return _buffer; }
size_t length() { return _length; }
private:
char* _buffer = NULL;
size_t _buf_size = 0;
size_t _length = 0;
};
} // end namespace string
} // end namespace paddle
#include "paddle/utils/string/string_helper.h"
paddle/fluid/string/to_string.h
浏览文件 @ 6ca49164
......@@ -13,48 +13,4 @@ See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <sstream>
#include <string>
#include <type_traits>
#include <typeindex>
#include <vector>
namespace paddle {
namespace string {
inline std::ostream& operator<<(std::ostream& s, const std::type_index& t) {
s << t.name();
return s;
}
template <typename T,
typename std::enable_if<!std::is_enum<T>::value, int>::type = 0>
inline std::string to_string(T v) {
std::ostringstream sout;
sout << v;
return sout.str();
}
template <typename T,
typename std::enable_if<std::is_enum<T>::value, int>::type = 0>
inline std::string to_string(T v) {
return std::to_string(static_cast<int>(v));
}
template <>
inline std::string to_string(std::type_index t) {
return t.name();
}
// Faster std::string/const char* type
template <>
inline std::string to_string(std::string v) {
return v;
}
template <>
inline std::string to_string(const char* v) {
return std::string(v);
}
} // namespace string
} // namespace paddle
#include "paddle/utils/string/to_string.h"
paddle/pten/api/lib/CMakeLists.txt
浏览文件 @ 6ca49164
......@@ -3,11 +3,11 @@ add_subdirectory(utils)
cc_library(ext_compat_utils SRCS ext_compat_utils.cc DEPS place)
if (WITH_GPU)
nv_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils enforce)
nv_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils pten_enforce)
elseif (WITH_ROCM)
hip_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils enforce)
hip_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils pten_enforce)
else()
cc_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils enforce)
cc_library(pten_tensor SRCS tensor.cc DEPS tensor_base dense_tensor pten_api_utils ext_compat_utils pten_enforce)
endif()
cc_library(kernel_dispatch SRCS kernel_dispatch.cc DEPS pten_tensor pten_context kernel_factory)
......
paddle/pten/api/lib/op_meta_info.cc
浏览文件 @ 6ca49164
......@@ -19,7 +19,7 @@ limitations under the License. */
#include <vector>
#include "paddle/fluid/framework/custom_operator.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/core/enforce.h"
namespace paddle {
......@@ -74,7 +74,7 @@ OpMetaInfoBuilder::OpMetaInfoBuilder(std::string&& name, size_t index) {
PADDLE_ENFORCE_EQ(
info_vector.size(),
index_,
platform::errors::PreconditionNotMet(
pten::errors::PreconditionNotMet(
"The operator %s's meta info register failed. "
"Please make sure you call marcos as order `PD_BUILD_OP`, "
"`PD_BUILD_GRAD_OP`, `PD_BUILD_DOUBLE_GRAD_OP`.",
......@@ -88,7 +88,7 @@ OpMetaInfoBuilder::OpMetaInfoBuilder(std::string&& name, size_t index) {
case 2:
name_ = name_ + "_grad_grad";
default:
PADDLE_THROW(platform::errors::InvalidArgument(
PADDLE_THROW(pten::errors::InvalidArgument(
"Not support index `%d` when construct OpMetaInfoBuilder, "
"now only support `0, 1, 2`.",
index_));
......@@ -130,7 +130,7 @@ OpMetaInfoBuilder& OpMetaInfoBuilder::SetInferDtypeFn(InferDtypeFunc func) {
PADDLE_ENFORCE_EQ(
index_,
0UL,
platform::errors::Unimplemented(
pten::errors::Unimplemented(
"Currently, the InferDtypeFn setting of Grad Op is not supported, "
"And backward Tensor `X@GRAD` will use the dtype of forward Tensor "
"`X` by default."));
......
paddle/pten/api/lib/tensor.cc
浏览文件 @ 6ca49164
......@@ -48,12 +48,12 @@ limitations under the License. */
* or the corresponding components will be re-implemented.
*/
#include "paddle/fluid/memory/memory.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/fluid/platform/place.h"
#include "paddle/fluid/platform/stream/cuda_stream.h"
#include "paddle/pten/common/complex.h"
#include "paddle/pten/common/float16.h"
#include "paddle/pten/core/ddim.h"
#include "paddle/pten/core/enforce.h"
namespace paddle {
namespace experimental {
......@@ -68,7 +68,7 @@ Tensor cast(const Tensor &x, DataType out_dtype);
Tensor::Tensor(std::shared_ptr<pten::TensorBase> tensor_impl)
: impl_(std::move(tensor_impl)) {
PADDLE_ENFORCE_NOT_NULL(impl_,
platform::errors::InvalidArgument(
pten::errors::InvalidArgument(
"TensorImpl with nullptr is not supported"));
}
......@@ -113,7 +113,7 @@ void Tensor::reshape(const std::vector<int64_t> &shape) {
std::dynamic_pointer_cast<pten::DenseTensor>(impl_)->set_meta(
pten::DenseTensorMeta(dtype(), framework::make_ddim(shape)));
} else {
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(pten::errors::Unimplemented(
"Only support reshape operation on DenseTensor now."));
}
}
......@@ -177,8 +177,8 @@ T *Tensor::mutable_data(const PlaceType &place) {
PADDLE_ENFORCE_EQ(
platform::is_same_place(inner_place, impl_->place()),
true,
platform::errors::Unimplemented("Modification of tensor place through "
"mutable_data is not supported now"));
pten::errors::Unimplemented("Modification of tensor place through "
"mutable_data is not supported now"));
}
if (is_dense_tensor()) {
return std::dynamic_pointer_cast<pten::DenseTensor>(impl_)->mutable_data<T>(
......@@ -236,7 +236,7 @@ Tensor::data<paddle::platform::bfloat16>() const;
template <typename T>
T *Tensor::data() {
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(pten::errors::Unimplemented(
"It is not currently supported to directly obtain the modifiable data "
"address through the tensor::data<T>() method, please use the "
"tensor::mutable_data<T>() method."));
......@@ -267,7 +267,7 @@ Tensor Tensor::slice(int64_t begin_idx, int64_t end_idx) const {
begin_idx,
end_idx))));
} else {
PADDLE_THROW(platform::errors::Unimplemented(
PADDLE_THROW(pten::errors::Unimplemented(
"Only support slice operation on DenseTensor now."));
}
}
......
paddle/pten/core/CMakeLists.txt
浏览文件 @ 6ca49164
......@@ -9,12 +9,16 @@ else()
cc_library(convert_utils SRCS convert_utils.cc DEPS data_type place)
endif()
cc_library(kernel_factory SRCS kernel_factory.cc DEPS enforce convert_utils)
cc_library(kernel_context SRCS kernel_context.cc DEPS enforce pten_context)
cc_library(errors SRCS errors.cc)
set(pten_enforce_deps errors flags)
cc_library(pten_enforce INTERFACE SRCS enforce.cc DEPS ${pten_enforce_deps})
cc_library(tensor_base SRCS tensor_base.cc allocator.cc storage.cc DEPS enforce)
cc_library(tensor_meta SRCS tensor_meta.cc DEPS enforce mixed_vector)
cc_library(lod_utils SRCS lod_utils.cc DEPS enforce mixed_vector)
cc_library(kernel_factory SRCS kernel_factory.cc DEPS pten_enforce convert_utils)
cc_library(kernel_context SRCS kernel_context.cc DEPS pten_enforce pten_context)
cc_library(tensor_base SRCS tensor_base.cc allocator.cc storage.cc DEPS pten_enforce)
cc_library(tensor_meta SRCS tensor_meta.cc DEPS pten_enforce mixed_vector)
cc_library(lod_utils SRCS lod_utils.cc DEPS pten_enforce mixed_vector)
cc_library(dense_tensor SRCS dense_tensor.cc DEPS convert_utils tensor_meta tensor_base)
cc_library(pten_device_context SRCS device_context.cc DEPS tensor_base )
......@@ -25,7 +29,6 @@ cc_test(unroll_array_ops_test SRCS unroll_array_ops_test.cc)
cc_library(ddim SRCS ddim.cc DEPS eigen3 boost enforce)
cc_library(selected_rows SRCS selected_rows.cc DEPS dense_tensor mixed_vector enforce ddim)
# Will remove once we implemented MKLDNN_Tensor
if(WITH_MKLDNN)
add_dependencies(dense_tensor mkldnn)
......
paddle/pten/core/compat/CMakeLists.txt
浏览文件 @ 6ca49164
cc_library(arg_map_context SRCS arg_map_context.cc DEPS enforce)
cc_library(arg_map_context SRCS arg_map_context.cc DEPS pten_enforce)
paddle/pten/core/compat/arg_map_context.cc
浏览文件 @ 6ca49164
......@@ -15,9 +15,9 @@ limitations under the License. */
#include "paddle/pten/core/compat/arg_map_context.h"
#include "paddle/fluid/string/string_helper.h"
#include "paddle/pten/core/enforce.h"
namespace pten {
std::ostream& operator<<(std::ostream& os, KernelSignature signature) {
os << "Kernel Signature - name: " << signature.name << "; inputs: "
<< paddle::string::join_strings(std::get<0>(signature.args), ", ")
......
paddle/pten/core/enforce.cc 0 → 100644
浏览文件 @ 6ca49164
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/pten/core/enforce.h"
paddle/pten/core/enforce.h 0 → 100644
浏览文件 @ 6ca49164
此差异已折叠。
paddle/fluid/platform/errors.cc → paddle/pten/core/errors.cc
浏览文件 @ 6ca49164
......@@ -12,54 +12,50 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/fluid/platform/errors.h"
#include "paddle/pten/core/errors.h"
#include <stdexcept>
namespace paddle {
namespace platform {
typedef ::paddle::platform::error::Code Code;
std::string error_name(Code code) {
namespace pten {
std::string error_name(ErrorCode code) {
switch (code) {
case paddle::platform::error::LEGACY:
case ErrorCode::LEGACY:
return "Error";
break;
case paddle::platform::error::INVALID_ARGUMENT:
case ErrorCode::INVALID_ARGUMENT:
return "InvalidArgumentError";
break;
case paddle::platform::error::NOT_FOUND:
case ErrorCode::NOT_FOUND:
return "NotFoundError";
break;
case paddle::platform::error::OUT_OF_RANGE:
case ErrorCode::OUT_OF_RANGE:
return "OutOfRangeError";
break;
case paddle::platform::error::ALREADY_EXISTS:
case ErrorCode::ALREADY_EXISTS:
return "AlreadyExistsError";
break;
case paddle::platform::error::RESOURCE_EXHAUSTED:
case ErrorCode::RESOURCE_EXHAUSTED:
return "ResourceExhaustedError";
break;
case paddle::platform::error::PRECONDITION_NOT_MET:
case ErrorCode::PRECONDITION_NOT_MET:
return "PreconditionNotMetError";
break;
case paddle::platform::error::PERMISSION_DENIED:
case ErrorCode::PERMISSION_DENIED:
return "PermissionDeniedError";
break;
case paddle::platform::error::EXECUTION_TIMEOUT:
case ErrorCode::EXECUTION_TIMEOUT:
return "ExecutionTimeoutError";
break;
case paddle::platform::error::UNIMPLEMENTED:
case ErrorCode::UNIMPLEMENTED:
return "UnimplementedError";
break;
case paddle::platform::error::UNAVAILABLE:
case ErrorCode::UNAVAILABLE:
return "UnavailableError";
break;
case paddle::platform::error::FATAL:
case ErrorCode::FATAL:
return "FatalError";
break;
case paddle::platform::error::EXTERNAL:
case ErrorCode::EXTERNAL:
return "ExternalError";
break;
default:
......@@ -74,6 +70,4 @@ std::string ErrorSummary::to_string() const {
result += error_message();
return result;
}
} // namespace platform
} // namespace paddle
} // namespace pten
paddle/fluid/platform/error_codes.proto → paddle/pten/core/errors.h
浏览文件 @ 6ca49164
/* Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
/* Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
......@@ -12,69 +12,135 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
syntax = "proto2";
option optimize_for = LITE_RUNTIME;
package paddle.platform.error;
#pragma once
enum Code {
#include <memory>
#include <stdexcept>
#include <string>
#include <tuple>
#include <type_traits>
#include "paddle/utils/string/printf.h"
namespace pten {
enum ErrorCode {
// Legacy error.
// Error type string: "Error"
LEGACY = 0;
LEGACY = 0,
// Client specified an invalid argument.
// Error type string: "InvalidArgumentError"
INVALID_ARGUMENT = 1;
INVALID_ARGUMENT = 1,
// Some requested entity (e.g., file or directory) was not found.
// Error type string: "NotFoundError"
NOT_FOUND = 2;
NOT_FOUND = 2,
// Operation tried to iterate past the valid input range. E.g., seeking or
// reading past end of file.
// Error type string: "OutOfRangeError"
OUT_OF_RANGE = 3;
OUT_OF_RANGE = 3,
// Some entity that we attempted to create (e.g., file or directory)
// already exists.
// Error type string: "AlreadyExistsError"
ALREADY_EXISTS = 4;
ALREADY_EXISTS = 4,
// Some resource has been exhausted, perhaps a per-user quota, or
// perhaps the entire file system is out of space.
// Error type string: "ResourceExhaustedError"
RESOURCE_EXHAUSTED = 5;
RESOURCE_EXHAUSTED = 5,
// Operation was rejected because the system is not in a state
// required for the operation's execution.
// Error type string: "PreconditionNotMetError"
PRECONDITION_NOT_MET = 6;
PRECONDITION_NOT_MET = 6,
// The caller does not have permission to execute the specified
// operation.
// Error type string: "PermissionDeniedError"
PERMISSION_DENIED = 7;
PERMISSION_DENIED = 7,
// Deadline expired before operation could complete.
// Error type string: "ExecutionTimeout"
EXECUTION_TIMEOUT = 8;
EXECUTION_TIMEOUT = 8,
// Operation is not implemented or not supported/enabled in this service.
// Error type string: "UnimpelmentedError"
UNIMPLEMENTED = 9;
UNIMPLEMENTED = 9,
// The service is currently unavailable. This is a most likely a
// transient condition and may be corrected by retrying with
// a backoff.
// Error type string: "UnavailableError"
UNAVAILABLE = 10;
UNAVAILABLE = 10,
// Fatal errors. Means some invariant expected by the underlying
// system has been broken. If you see one of these errors,
// something is very broken.
// Error type string: "FatalError"
FATAL = 11;
FATAL = 11,
// Third-party library error.
// Error type string: "ExternalError"
EXTERNAL = 12;
}
EXTERNAL = 12,
};
class ErrorSummary {
public:
// Note(chenweihang): Final deprecated constructor
// This constructor is used to be compatible with
// current existing untyped PADDLE_ENFORCE_*
// PADDLE_ENFORCE
// Note(chenweihang): Windows openblas need this
// constructor for compiling PADDLE_ENFORCE in *.cu,
// this is a bug cause we can't remove this
// constructor now.
template <typename... Args>
explicit ErrorSummary(Args... args) {
code_ = pten::ErrorCode::LEGACY;
msg_ = paddle::string::Sprintf(args...);
}
// Note(chenweihang): Only recommended constructor
// No longer supports PADDLE_ENFORCE without type or without error message
explicit ErrorSummary(ErrorCode code, std::string msg)
: code_(code), msg_(msg) {}
ErrorCode code() const { return code_; }
const std::string& error_message() const { return msg_; }
std::string to_string() const;
private:
ErrorCode code_;
std::string msg_;
};
namespace errors {
#define REGISTER_ERROR(FUNC, CONST, ...) \
template <typename... Args> \
::pten::ErrorSummary FUNC(Args... args) { \
return ::pten::ErrorSummary(::pten::CONST, \
::paddle::string::Sprintf(args...)); \
}
REGISTER_ERROR(InvalidArgument, ErrorCode::INVALID_ARGUMENT)
REGISTER_ERROR(NotFound, ErrorCode::NOT_FOUND)
REGISTER_ERROR(OutOfRange, ErrorCode::OUT_OF_RANGE)
REGISTER_ERROR(AlreadyExists, ErrorCode::ALREADY_EXISTS)
REGISTER_ERROR(ResourceExhausted, ErrorCode::RESOURCE_EXHAUSTED)
REGISTER_ERROR(PreconditionNotMet, ErrorCode::PRECONDITION_NOT_MET)
REGISTER_ERROR(PermissionDenied, ErrorCode::PERMISSION_DENIED)
REGISTER_ERROR(ExecutionTimeout, ErrorCode::EXECUTION_TIMEOUT)
REGISTER_ERROR(Unimplemented, ErrorCode::UNIMPLEMENTED)
REGISTER_ERROR(Unavailable, ErrorCode::UNAVAILABLE)
REGISTER_ERROR(Fatal, ErrorCode::FATAL)
REGISTER_ERROR(External, ErrorCode::EXTERNAL)
#undef REGISTER_ERROR
} // namespace errors
} // namespace pten
paddle/pten/core/kernel_context.h
浏览文件 @ 6ca49164
......@@ -24,7 +24,7 @@
// See Note [ Why still include the fluid headers? ]
#include "paddle/fluid/platform/device_context.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/core/enforce.h"
namespace pten {
......@@ -123,7 +123,7 @@ class KernelContext {
try {
return paddle::any_cast<AttrType>(attrs_.at(idx));
} catch (paddle::bad_any_cast&) {
PADDLE_THROW(paddle::platform::errors::InvalidArgument(
PADDLE_THROW(pten::errors::InvalidArgument(
"Attribute cast error in Op Kernel Context."));
}
}
......
paddle/pten/core/kernel_factory.cc
浏览文件 @ 6ca49164
......@@ -15,7 +15,7 @@
#include "paddle/pten/core/kernel_factory.h"
// See Note [ Why still include the fluid headers? ]
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/core/enforce.h"
namespace pten {
......@@ -64,8 +64,8 @@ const Kernel& KernelFactory::SelectKernelOrThrowError(
auto iter = kernels_.find(kernel_name);
PADDLE_ENFORCE_NE(iter,
kernels_.end(),
paddle::platform::errors::NotFound(
"The kernel `%s` is not registered.", kernel_name));
pten::errors::NotFound("The kernel `%s` is not registered.",
kernel_name));
auto kernel_iter = iter->second.find(kernel_key);
// TODO(chenweihang): polish refind impl here
......@@ -78,7 +78,7 @@ const Kernel& KernelFactory::SelectKernelOrThrowError(
PADDLE_ENFORCE_NE(
kernel_iter,
iter->second.end(),
paddle::platform::errors::NotFound(
pten::errors::NotFound(
"The kernel with key %s of kernel `%s` is not registered.",
kernel_key,
kernel_name));
......
paddle/pten/core/kernel_factory.h
浏览文件 @ 6ca49164
......@@ -27,7 +27,7 @@
#include "paddle/pten/core/kernel_def.h"
// See Note [ Why still include the fluid headers? ]
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/core/enforce.h"
#include "paddle/utils/flat_hash_map.h"
#include "paddle/utils/small_vector.h"
......
paddle/pten/core/kernel_registry.h
浏览文件 @ 6ca49164
......@@ -26,7 +26,7 @@
#include "paddle/pten/core/kernel_utils.h"
#include "paddle/pten/core/macros.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/core/enforce.h"
namespace pten {
......
paddle/pten/core/kernel_utils.h
浏览文件 @ 6ca49164
......@@ -22,7 +22,7 @@
#include "paddle/pten/core/kernel_def.h"
// See Note [ Why still include the fluid headers? ]
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/core/enforce.h"
namespace pten {
......
paddle/pten/core/type_defs.h 0 → 100644
浏览文件 @ 6ca49164
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <map>
#include <memory>
#include <string>
#include <unordered_map>
#include <unordered_set>
#include <vector>
#include <boost/variant.hpp>
namespace paddle {
namespace framework {
// The order should be as same as framework.proto
// NOTE(xiongkun): we extract from framework/typedef.h to ensure we can transfer
// enforce.h
class BlockDesc;
using Attribute = boost::variant<boost::blank,
int,
float,
std::string,
std::vector<int>,
std::vector<float>,
std::vector<std::string>,
bool,
std::vector<bool>,
BlockDesc*,
int64_t,
std::vector<BlockDesc*>,
std::vector<int64_t>,
std::vector<double>>;
using AttributeMap = std::unordered_map<std::string, Attribute>;
} // namespace framework
namespace imperative {
class VariableWrapper;
class SavedVariableWrapperList;
class VarBase;
class OpBase;
class GradOpNode;
class Tracer;
using WeakNameVarBaseMap =
std::map<std::string, std::vector<std::weak_ptr<VarBase>>>;
namespace details {
template <typename T>
struct NameVarMapTrait {};
template <>
struct NameVarMapTrait<VarBase> {
using Type = std::map<std::string, std::vector<std::shared_ptr<VarBase>>>;
};
template <>
struct NameVarMapTrait<VariableWrapper> {
using Type = std::map<std::string, SavedVariableWrapperList>;
};
} // namespace details
template <typename T>
using NameVarMap = typename details::NameVarMapTrait<T>::Type;
using NameVarBaseMap = NameVarMap<VarBase>;
using NameVariableWrapperMap = NameVarMap<VariableWrapper>;
using VariableWrapperList = std::vector<std::shared_ptr<VariableWrapper>>;
} // namespace imperative
} // namespace paddle
paddle/pten/core/utils/intrusive_ptr.h
浏览文件 @ 6ca49164
......@@ -16,7 +16,7 @@ limitations under the License. */
#include <utility>
#include "glog/logging.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/core/enforce.h"
namespace pten {
......@@ -62,7 +62,7 @@ class intrusive_ptr {
T& operator*() const {
PADDLE_ENFORCE_NOT_NULL(
px,
paddle::platform::errors::PreconditionNotMet(
pten::errors::PreconditionNotMet(
"The pointer must be non-null before the dereference operation."));
return *px;
}
......@@ -70,7 +70,7 @@ class intrusive_ptr {
T* operator->() const {
PADDLE_ENFORCE_NOT_NULL(
px,
paddle::platform::errors::PreconditionNotMet(
pten::errors::PreconditionNotMet(
"The pointer must be non-null before the dereference operation."));
return px;
}
......
paddle/pten/kernels/funcs/elementwise_functor.h
浏览文件 @ 6ca49164
......@@ -14,8 +14,8 @@ limitations under the License. */
#pragma once
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/common/float16.h"
#include "paddle/pten/core/enforce.h"
#include "paddle/pten/core/hostdevice.h"
namespace pten {
......
paddle/pten/kernels/gpu/math_kernel.cu
浏览文件 @ 6ca49164
......@@ -27,10 +27,10 @@ limitations under the License. */
namespace cub = hipcub;
#endif
#include "paddle/fluid/platform/enforce.h"
#include "paddle/pten/common/complex.h"
#include "paddle/pten/common/float16.h"
#include "paddle/pten/core/convert_utils.h"
#include "paddle/pten/core/enforce.h"
#include "paddle/pten/core/kernel_registry.h"
namespace pten {
......
paddle/pten/kernels/gpu/reduce.h
浏览文件 @ 6ca49164
......@@ -37,10 +37,10 @@ namespace cub = hipcub;
#include "paddle/fluid/operators/kernel_primitives/kernel_primitives.h"
#include "paddle/fluid/platform/device/gpu/gpu_device_function.h"
#include "paddle/fluid/platform/device/gpu/gpu_info.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/fluid/platform/fast_divmod.h"
#include "paddle/fluid/string/string_helper.h"
#include "paddle/pten/core/array.h"
#include "paddle/pten/core/enforce.h"
#include "paddle/pten/api/ext/dispatch.h"
#include "paddle/pten/backends/gpu/gpu_context.h"
......@@ -94,7 +94,7 @@ static inline void CheckReduceRank(int reduce_rank, int rank) {
if (rank % 2 == 0) {
PADDLE_ENFORCE_EQ(reduce_rank,
rank / 2,
paddle::platform::errors::InvalidArgument(
pten::errors::InvalidArgument(
"ReduceOp: invalid reduce rank. When rank = %d, "
"reduce_rank must be %d, but got %d.",
rank,
......@@ -106,7 +106,7 @@ static inline void CheckReduceRank(int reduce_rank, int rank) {
PADDLE_ENFORCE_EQ(
reduce_rank == lower_rank || reduce_rank == upper_rank,
true,
paddle::platform::errors::InvalidArgument(
pten::errors::InvalidArgument(
"ReduceOp: invalid reduce rank. When rank = %d, reduce_rank "
"must be %d or %d, but got %d.",
rank,
......@@ -122,7 +122,7 @@ static inline pten::framework::Array<T, ElementCount> VectorToArray(
const VectorLikeType& vec) {
PADDLE_ENFORCE_LE(vec.size(),
ElementCount,
paddle::platform::errors::InvalidArgument(
pten::errors::InvalidArgument(
"Cub reduce Array: size not match. Received "
"vec.size() %d > ElementCount %d.",
vec.size(),
......@@ -149,7 +149,7 @@ static inline std::vector<int> GetReduceDim(const std::vector<int64_t>& dims,
for (auto e : dims) {
PADDLE_ENFORCE_LT(e,
dim_size,
paddle::platform::errors::InvalidArgument(
pten::errors::InvalidArgument(
"ReduceOp: invalid axis, when x_dims is %d, "
"axis[i] should less than x_dims, but got %d.",
dim_size,
......@@ -1057,7 +1057,7 @@ static
int reduce_num,
const paddle::platform::Place& place,
gpuStream_t stream) {
PADDLE_THROW(paddle::platform::errors::InvalidArgument(
PADDLE_THROW(pten::errors::InvalidArgument(
"Tx should not be float16 when using cub::DeviceReduce::Reduce()."));
}
......
paddle/utils/CMakeLists.txt 0 → 100644
浏览文件 @ 6ca49164
add_subdirectory(string)
paddle/fluid/string/CMakeLists.txt → paddle/utils/string/CMakeLists.txt
浏览文件 @ 6ca49164
cc_library(stringpiece SRCS piece.cc DEPS flags)
cc_library(pretty_log SRCS pretty_log.cc DEPS flags)
cc_library(string_helper SRCS string_helper.cc DEPS flags)
cc_test(stringpiece_test SRCS piece_test.cc DEPS stringpiece glog gflags)
cc_test(stringprintf_test SRCS printf_test.cc DEPS glog gflags)
cc_test(stringpiece_test SRCS piece_test.cc DEPS stringpiece gflags)
cc_test(stringprintf_test SRCS printf_test.cc DEPS gflags)
cc_test(to_string_test SRCS to_string_test.cc)
cc_test(split_test SRCS split_test.cc)
cc_test(string_helper_test SRCS string_helper_test.cc DEPS string_helper)
paddle/fluid/string/piece.cc → paddle/utils/string/piece.cc
浏览文件 @ 6ca49164
......@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/string/piece.h"
#include "paddle/utils/string/piece.h"
#include <string.h>
#include <algorithm>
......@@ -76,9 +76,11 @@ bool HasPrefix(Piece s, Piece x) {
}
bool HasSuffix(Piece s, Piece x) {
return !x.len() ? true : ((s.len() >= x.len()) &&
(memcmp(s.data() + (s.len() - x.len()), x.data(),
x.len()) == 0));
return !x.len()
? true
: ((s.len() >= x.len()) &&
(memcmp(s.data() + (s.len() - x.len()), x.data(), x.len()) ==
0));
}
Piece SkipPrefix(Piece s, size_t n) {
......
paddle/utils/string/piece.h 0 → 100644
浏览文件 @ 6ca49164
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <ostream>
#include <string>
namespace paddle {
namespace string {
// Piece points into a std::string object but doesn't own the
// string. It is for efficient access to strings. Like Go's string
// type. Not that Piece doesn't mutate the underlying string,
// so it is thread-safe given that the underlying string doesn't
// change. Because Piece contains a little data members, and
// its syntax is simple as it doesn't own/manage the string, it is
// cheap to construct Pieces and pass them around.
class Piece {
public:
static const size_t npos = static_cast<size_t>(-1);
// We provide non-explicit singleton constructors so users can
// pass in a "const char*" or a "string" wherever a "Piece"
// is expected. These constructors ensure that if data_ is NULL,
// size_ is 0.
Piece();
Piece(const char* d, size_t n);
Piece(const char* d); // NOLINT: accept C string into Piece.
Piece(const std::string& s); // NOLINT: accept C++ string into Piece.
const char* data() const { return data_; }
size_t len() const { return size_; }
char operator[](size_t n) const;
// Piece doesn't own the string, so both iterator and const
// iterator are const char* indeed.
typedef const char* const_iterator;
typedef const char* iterator;
iterator begin() const { return data_; }
iterator end() const { return data_ + size_; }
// Return a string that contains the copy of the referenced data.
std::string ToString() const { return std::string(data_, size_); }
private:
const char* data_;
size_t size_;
// Intentionally copyable
};
int Compare(Piece a, Piece b);
bool operator==(Piece x, Piece y);
bool operator!=(Piece x, Piece y);
bool operator<(Piece x, Piece y);
bool operator>(Piece x, Piece y);
bool operator<=(Piece x, Piece y);
bool operator>=(Piece x, Piece y);
bool HasPrefix(Piece s, Piece prefix);
bool HasSuffix(Piece s, Piece suffix);
Piece SkipPrefix(Piece s, size_t n);
Piece SkipSuffix(Piece s, size_t n);
// Skip the prefix (or suffix) if it matches with the string.
Piece TrimPrefix(Piece s, Piece prefix);
Piece TrimSuffix(Piece s, Piece suffix);
// Returns if s contains sub. Any s except for empty s contains an
// empty sub.
bool Contains(Piece s, Piece sub);
// Return the first occurrence of sub in s, or npos. If both s and
// sub is empty, it returns npos; otherwise, if only sub is empty, it
// returns 0.
size_t Index(Piece s, Piece sub);
// Return the first occurrence of c in s[pos:end], or npos.
size_t Find(Piece s, char c, size_t pos);
// Search range is [0..pos] inclusive. If pos == npos, search everything.
size_t RFind(Piece s, char c, size_t pos);
Piece SubStr(Piece s, size_t pos, size_t n);
// allow Piece to be logged
std::ostream& operator<<(std::ostream& o, Piece piece);
} // namespace string
} // namespace paddle
paddle/fluid/string/piece_test.cc → paddle/utils/string/piece_test.cc
浏览文件 @ 6ca49164
......@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/string/piece.h"
#include "paddle/utils/string/piece.h"
#include "gtest/gtest.h"
......
paddle/fluid/string/pretty_log.cc → paddle/utils/string/pretty_log.cc
浏览文件 @ 6ca49164
......@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/string/pretty_log.h"
#include "paddle/utils/string/pretty_log.h"
#include "gflags/gflags.h"
DEFINE_bool(color, true, "Whether to turn on pretty log");
......
paddle/utils/string/pretty_log.h 0 → 100644
浏览文件 @ 6ca49164
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <iostream>
#include <sstream>
#include <string>
#include <utility>
#include "gflags/gflags.h"
#include "paddle/utils/string/printf.h"
DECLARE_bool(color);
namespace paddle {
namespace string {
inline std::string black() { return FLAGS_color ? "\e[30m" : ""; }
inline std::string red() { return FLAGS_color ? "\e[31m" : ""; }
inline std::string b_red() { return FLAGS_color ? "\e[41m" : ""; }
inline std::string green() { return FLAGS_color ? "\e[32m" : ""; }
inline std::string yellow() { return FLAGS_color ? "\e[33m" : ""; }
inline std::string blue() { return FLAGS_color ? "\e[34m" : ""; }
inline std::string purple() { return FLAGS_color ? "\e[35m" : ""; }
inline std::string cyan() { return FLAGS_color ? "\e[36m" : ""; }
inline std::string light_gray() { return FLAGS_color ? "\e[37m" : ""; }
inline std::string white() { return FLAGS_color ? "\e[37m" : ""; }
inline std::string light_red() { return FLAGS_color ? "\e[91m" : ""; }
inline std::string dim() { return FLAGS_color ? "\e[2m" : ""; }
inline std::string bold() { return FLAGS_color ? "\e[1m" : ""; }
inline std::string underline() { return FLAGS_color ? "\e[4m" : ""; }
inline std::string blink() { return FLAGS_color ? "\e[5m" : ""; }
inline std::string reset() { return FLAGS_color ? "\e[0m" : ""; }
using TextBlock = std::pair<std::string, std::string>;
struct Style {
static std::string info() { return black(); }
static std::string warn() { return b_red(); }
static std::string suc() { return green(); }
static std::string H1() { return bold() + purple(); }
static std::string H2() { return green(); }
static std::string H3() { return green(); }
static std::string detail() { return light_gray(); }
};
template <typename... Args>
static void PrettyLogEndl(const std::string &style,
const char *fmt,
const Args &... args) {
std::cerr << style << Sprintf(fmt, args...) << reset() << std::endl;
}
template <typename... Args>
static void PrettyLog(const std::string &style,
const char *fmt,
const Args &... args) {
std::cerr << style << Sprintf(fmt, args...) << reset();
}
template <typename... Args>
static void PrettyLogInfo(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::info(), fmt, args...);
}
template <typename... Args>
static void PrettyLogDetail(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::detail(), fmt, args...);
}
template <typename... Args>
static void PrettyLogH1(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::H1(), fmt, args...);
}
template <typename... Args>
static void PrettyLogH2(const char *fmt, const Args &... args) {
PrettyLogEndl(Style::H2(), fmt, args...);
}
} // namespace string
} // namespace paddle
paddle/utils/string/printf.h 0 → 100644
浏览文件 @ 6ca49164
// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Compared with std::stringstream, there are primary purpose of
// string::Printf:
//
// 1. Type-safe printing, with why and how explained in
// http://www.drdobbs.com/stringprintf-a-typesafe-printf-family-fo/184401999.
// Implementation includes
//
// https://github.com/c42f/tinyformat
// boost::format
// std::stringstream
//
// std::stringstream is not convenient enough in many cases. For example:
//
// std::cout << std::setprecision(2) << std::fixed << 1.23456 << "\n";
//
// boost::format is the most convenient one. We can have
//
// std::cout << format("%2% %1%") % 36 % 77;
//
// or
//
// format fmter("%2% %1%");
// fmter % 36; fmter % 77;
// std::cout << fmter.c_str();
//
// But the overloading of % might be overkilling and it would be
// more efficient if it can write to std::cout directly.
//
// tinyformat has an interface compatible with the C-printf style,
// and it can writes to a stream or returns a std::string:
//
// std::cout << tfm::printf(
// "%s, %s %d, %.2d:%.2d\n",
// weekday, month, day, hour, min);
//
// or
//
// tfm::format(std::cout,
// "%s, %s %d, %.2d:%.2d\n",
// weekday, month, day, hour, min);
//
// 2. High-performance -- most printed strings are not too long and
// doens't need dynamic memory allocation. Many StringPrintf
// implementations doesn't enforce type-safe, but are
// high-performance, including
//
// https://developers.google.com/optimization/reference/base/stringprintf/
// https://github.com/adobe/chromium/blob/master/base/stringprintf.h
// https://github.com/google/protobuf/blob/master/src/google/protobuf/stubs/stringprintf.h
//
// According to
// https://github.com/c42f/tinyformat#compile-time-and-code-bloat,
// boost::format runs too slow and results in large executable binary
// files. So here we port tinyformat.
#pragma once
#include <iostream>
#include <sstream>
#include <string>
#include <vector>
#include "paddle/utils/string/tinyformat/tinyformat.h" // https://github.com/c42f/tinyformat
namespace paddle {
namespace string {
template <typename... Args>
void Fprintf(std::ostream& out, const char* fmt, const Args&... args) {
tinyformat::vformat(out, fmt, tinyformat::makeFormatList(args...));
}
inline std::string Sprintf() { return ""; }
template <typename... Args>
std::string Sprintf(const Args&... args) {
std::ostringstream oss;
Fprintf(oss, "%s", args...);
return oss.str();
}
template <typename... Args>
std::string Sprintf(const char* fmt, const Args&... args) {
std::ostringstream oss;
Fprintf(oss, fmt, args...);
return oss.str();
}
template <typename... Args>
void Printf(const char* fmt, const Args&... args) {
Fprintf(std::cout, fmt, args...);
}
inline std::string HumanReadableSize(double f_size) {
size_t i = 0;
double orig = f_size;
const std::vector<std::string> units(
{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"});
while (f_size >= 1024) {
f_size /= 1024;
i++;
}
if (i >= units.size()) {
return Sprintf("%fB", orig);
}
return Sprintf("%f%s", f_size, units[i]);
}
} // namespace string
} // namespace paddle
paddle/fluid/string/printf_test.cc → paddle/utils/string/printf_test.cc
浏览文件 @ 6ca49164
......@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/string/printf.h"
#include "paddle/utils/string/printf.h"
#include <string>
......@@ -25,7 +25,7 @@ TEST(StringPrintf, StringPrintf) {
int hour = 14;
int min = 44;
EXPECT_EQ(std::string("Wednesday, July 27, 14:44"),
paddle::string::Sprintf("%s, %s %d, %.2d:%.2d", weekday, month, day,
hour, min));
paddle::string::Sprintf(
"%s, %s %d, %.2d:%.2d", weekday, month, day, hour, min));
EXPECT_EQ(std::string(""), paddle::string::Sprintf());
}
paddle/utils/string/split.h 0 → 100644
浏览文件 @ 6ca49164
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <sstream>
#include <string>
#include <vector>
namespace paddle {
namespace string {
static inline std::vector<std::string> Split(std::string const& original,
char separator) {
std::vector<std::string> results;
std::string token;
std::istringstream is(original);
while (std::getline(is, token, separator)) {
if (!token.empty()) {
results.push_back(token);
}
}
return results;
}
} // namespace string
} // namespace paddle
paddle/fluid/string/split_test.cc → paddle/utils/string/split_test.cc
浏览文件 @ 6ca49164
......@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/string/split.h"
#include "paddle/utils/string/split.h"
#include <string>
......
paddle/fluid/string/string_helper.cc → paddle/utils/string/string_helper.cc
浏览文件 @ 6ca49164
......@@ -12,15 +12,13 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/string/string_helper.h"
#include "paddle/utils/string/string_helper.h"
#include <ctype.h>
#include <stdio.h>
#include <cstring>
#include <string>
#include "glog/logging.h"
namespace paddle {
namespace string {
......@@ -75,7 +73,9 @@ char* LineFileReader::getdelim(FILE* f, char delim) {
return _buffer;
} else {
_length = 0;
CHECK(feof(f));
int code = feof(f);
(void)code;
assert(code);
return NULL;
}
#else
......
paddle/utils/string/string_helper.h 0 → 100644
浏览文件 @ 6ca49164
// Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#pragma once
#include <assert.h>
#include <ctype.h>
#include <stdio.h>
#include <cstring>
#include <sstream>
#include <string>
#include <utility>
#include <vector>
namespace paddle {
namespace string {
inline size_t count_spaces(const char* s) {
size_t count = 0;
while (*s != 0 && isspace(*s++)) {
count++;
}
return count;
}
inline size_t count_nonspaces(const char* s) {
size_t count = 0;
while (*s != 0 && !isspace(*s++)) {
count++;
}
return count;
}
template <class... ARGS>
void format_string_append(std::string& str, // NOLINT
const char* fmt, // NOLINT
ARGS&&... args) {
int len = snprintf(NULL, 0, fmt, args...);
assert(len == 0);
size_t oldlen = str.length();
str.resize(oldlen + len + 1);
int new_len =
snprintf(&str[oldlen], (size_t)len + 1, fmt, args...); // NOLINT
(void)new_len;
assert(new_len == len);
str.resize(oldlen + len);
}
template <class... ARGS>
void format_string_append(std::string& str, // NOLINT
const std::string& fmt, // NOLINT
ARGS&&... args) {
format_string_append(str, fmt.c_str(), args...);
}
template <class... ARGS>
std::string format_string(const char* fmt, ARGS&&... args) {
std::string str;
format_string_append(str, fmt, args...);
return str;
}
template <class... ARGS>
std::string format_string(const std::string& fmt, ARGS&&... args) {
return format_string(fmt.c_str(), args...);
}
// remove leading and tailing spaces
std::string trim_spaces(const std::string& str);
// erase all spaces in str
std::string erase_spaces(const std::string& str);
inline int str_to_float(const char* str, float* v) {
const char* head = str;
char* cursor = NULL;
int index = 0;
while (*(head += count_spaces(head)) != 0) {
v[index++] = std::strtof(head, &cursor);
if (head == cursor) {
break;
}
head = cursor;
}
return index;
}
// checks whether the test string is a suffix of the input string.
bool ends_with(std::string const& input, std::string const& test);
// split string by delim
template <class T = std::string>
std::vector<T> split_string(const std::string& str, const std::string& delim) {
size_t pre_pos = 0;
size_t pos = 0;
std::string tmp_str;
std::vector<T> res_list;
res_list.clear();
if (str.empty()) {
return res_list;
}
while ((pos = str.find(delim, pre_pos)) != std::string::npos) {
tmp_str.assign(str, pre_pos, pos - pre_pos);
res_list.push_back(tmp_str);
pre_pos = pos + 1;
}
tmp_str.assign(str, pre_pos, str.length() - pre_pos);
if (!tmp_str.empty()) {
res_list.push_back(tmp_str);
}
return res_list;
}
// split string by spaces. Leading and tailing spaces are ignored. Consecutive
// spaces are treated as one delim.
template <class T = std::string>
std::vector<T> split_string(const std::string& str) {
std::vector<T> list;
const char* p;
int pre_pos = 0;
int pos = 0;
std::string tmp_str;
if (str.empty()) {
return list;
}
for (p = str.c_str(); *p != 0;) {
if (!isspace(*p)) {
pos = pre_pos;
p++;
while (*p != 0 && !isspace(*p)) {
pos++;
p++;
}
tmp_str.assign(str, pre_pos, pos - pre_pos + 1);
list.push_back(tmp_str);
pre_pos = pos + 1;
} else {
pre_pos++;
p++;
}
}
return list;
}
template <class Container>
std::string join_strings(const Container& strs, char delim) {
std::string str;
size_t i = 0;
for (auto& elem : strs) {
if (i > 0) {
str += delim;
}
std::stringstream ss;
ss << elem;
str += ss.str();
++i;
}
return str;
}
template <class Container>
std::string join_strings(const Container& strs, const std::string& delim) {
std::string str;
size_t i = 0;
for (auto& elem : strs) {
if (i > 0) {
str += delim;
}
std::stringstream ss;
ss << elem;
str += ss.str();
++i;
}
return str;
}
template <class Container, class DelimT, class ConvertFunc>
std::string join_strings(const Container& strs,
DelimT&& delim,
ConvertFunc&& func) {
std::stringstream ss;
size_t i = 0;
for (const auto& elem : strs) {
if (i > 0) {
ss << delim;
}
ss << func(elem);
++i;
}
return ss.str();
}
// A helper class for reading lines from file. A line buffer is maintained. It
// doesn't need to know the maximum possible length of a line.
class LineFileReader {
public:
LineFileReader() {}
LineFileReader(LineFileReader&&) = delete;
LineFileReader(const LineFileReader&) = delete;
~LineFileReader() { ::free(_buffer); }
char* getline(FILE* f) { return this->getdelim(f, '\n'); }
char* getdelim(FILE* f, char delim);
char* get() { return _buffer; }
size_t length() { return _length; }
private:
char* _buffer = NULL;
size_t _buf_size = 0;
size_t _length = 0;
};
} // end namespace string
} // end namespace paddle
paddle/fluid/string/string_helper_test.cc → paddle/utils/string/string_helper_test.cc
浏览文件 @ 6ca49164
......@@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/string/string_helper.h"
#include "paddle/utils/string/string_helper.h"
#include <string>
......
paddle/fluid/string/tinyformat/tinyformat.h → paddle/utils/string/tinyformat/tinyformat.h
浏览文件 @ 6ca49164
......@@ -170,7 +170,8 @@ struct is_convertible {
// Format the value by casting to type fmtT. This default implementation
// should never be called.
template <typename T, typename fmtT,
template <typename T,
typename fmtT,
bool convertible = is_convertible<T, fmtT>::value>
struct formatValueAsType {
static void invoke(std::ostream & /*out*/, const T & /*value*/) { assert(0); }
......@@ -240,8 +241,11 @@ TINYFORMAT_DEFINE_FORMAT_TRUNCATED_CSTR(char)
/// operator<< to format the type T, with special cases for the %c and %p
/// conversions.
template <typename T>
inline void formatValue(std::ostream &out, const char * /*fmtBegin*/,
const char *fmtEnd, int ntrunc, const T &value) {
inline void formatValue(std::ostream &out,
const char * /*fmtBegin*/,
const char *fmtEnd,
int ntrunc,
const T &value) {
// The mess here is to support the %c and %p conversions: if these
// conversions are active we try to convert the type to a char or const
// void* respectively and format that instead of the value itself. For the
......@@ -250,35 +254,39 @@ inline void formatValue(std::ostream &out, const char * /*fmtBegin*/,
const bool canConvertToChar = detail::is_convertible<T, char>::value;
const bool canConvertToVoidPtr =
detail::is_convertible<T, const void *>::value;
if (canConvertToChar && *(fmtEnd - 1) == 'c')
if (canConvertToChar && *(fmtEnd - 1) == 'c') {
detail::formatValueAsType<T, char>::invoke(out, value);
else if (canConvertToVoidPtr && *(fmtEnd - 1) == 'p')
} else if (canConvertToVoidPtr && *(fmtEnd - 1) == 'p') {
detail::formatValueAsType<T, const void *>::invoke(out, value);
else if (ntrunc >= 0) {
} else if (ntrunc >= 0) {
// Take care not to overread C strings in truncating conversions like
// "%.4s" where at most 4 characters may be read.
detail::formatTruncated(out, value, ntrunc);
} else
} else {
out << value;
}
}
// Overloaded version for char types to support printing as an integer
#define TINYFORMAT_DEFINE_FORMATVALUE_CHAR(charType) \
inline void formatValue(std::ostream &out, const char * /*fmtBegin*/, \
const char *fmtEnd, int /**/, charType value) { \
switch (*(fmtEnd - 1)) { \
case 'u': \
case 'd': \
case 'i': \
case 'o': \
case 'X': \
case 'x': \
out << static_cast<int>(value); \
break; \
default: \
out << value; \
break; \
} \
#define TINYFORMAT_DEFINE_FORMATVALUE_CHAR(charType) \
inline void formatValue(std::ostream &out, \
const char * /*fmtBegin*/, \
const char *fmtEnd, \
int /**/, \
charType value) { \
switch (*(fmtEnd - 1)) { \
case 'u': \
case 'd': \
case 'i': \
case 'o': \
case 'X': \
case 'x': \
out << static_cast<int>(value); \
break; \
default: \
out << value; \
break; \
} \
}
// per 3.9.1: char, signed char and unsigned char are all distinct types
TINYFORMAT_DEFINE_FORMATVALUE_CHAR(char)
......@@ -466,7 +474,7 @@ cog.outl('#define TINYFORMAT_FOREACH_ARGNUM(m) \\\n ' +
#define TINYFORMAT_FOREACH_ARGNUM(m) \
m(1) m(2) m(3) m(4) m(5) m(6) m(7) m(8) m(9) m(10) m(11) m(12) m(13) m(14) \
m(15) m(16)
//[[[end]]]
// [[[end]]]
namespace detail {
......@@ -476,15 +484,17 @@ namespace detail {
// whereas a naive implementation based on inheritance does not.
class FormatArg {
public:
FormatArg() {}
FormatArg() {} // NOLINT
template <typename T>
FormatArg(const T &value)
FormatArg(const T &value) // NOLINT
: m_value(static_cast<const void *>(&value)),
m_formatImpl(&formatImpl<T>),
m_toIntImpl(&toIntImpl<T>) {}
void format(std::ostream &out, const char *fmtBegin, const char *fmtEnd,
void format(std::ostream &out,
const char *fmtBegin,
const char *fmtEnd,
int ntrunc) const {
m_formatImpl(out, fmtBegin, fmtEnd, ntrunc, m_value);
}
......@@ -493,8 +503,11 @@ class FormatArg {
private:
template <typename T>
static void formatImpl(std::ostream &out, const char *fmtBegin,
const char *fmtEnd, int ntrunc, const void *value) {
static void formatImpl(std::ostream &out,
const char *fmtBegin,
const char *fmtEnd,
int ntrunc,
const void *value) {
formatValue(out, fmtBegin, fmtEnd, ntrunc, *static_cast<const T *>(value));
}
......@@ -504,14 +517,17 @@ class FormatArg {
}
const void *m_value;
void (*m_formatImpl)(std::ostream &out, const char *fmtBegin,
const char *fmtEnd, int ntrunc, const void *value);
void (*m_formatImpl)(std::ostream &out,
const char *fmtBegin,
const char *fmtEnd,
int ntrunc,
const void *value);
int (*m_toIntImpl)(const void *value);
};
// Parse and return an integer from the string c, as atoi()
// On return, c is set to one past the end of the integer.
inline int parseIntAndAdvance(const char *&c) {
inline int parseIntAndAdvance(const char *&c) { // NOLINT
int i = 0;
for (; *c >= '0' && *c <= '9'; ++c) i = 10 * i + (*c - '0');
return i;
......@@ -553,11 +569,13 @@ inline const char *printFormatStringLiteral(std::ostream &out,
// and ntrunc (for truncating conversions). argIndex is incremented if
// necessary to pull out variable width and precision . The function returns a
// pointer to the character after the end of the current format spec.
inline const char *streamStateFromFormat(std::ostream &out,
bool &spacePadPositive, int &ntrunc,
inline const char *streamStateFromFormat(std::ostream &out, // NOLINT
bool &spacePadPositive, // NOLINT
int &ntrunc, // NOLINT
const char *fmtStart,
const detail::FormatArg *formatters,
int &argIndex, int numFormatters) {
int &argIndex, // NOLINT
int numFormatters) {
if (*fmtStart != '%') {
TINYFORMAT_ERROR(
"tinyformat: Not enough conversion specifiers in format string");
......@@ -733,8 +751,10 @@ inline const char *streamStateFromFormat(std::ostream &out,
}
//------------------------------------------------------------------------------
inline void formatImpl(std::ostream &out, const char *fmt,
const detail::FormatArg *formatters, int numFormatters) {
inline void formatImpl(std::ostream &out,
const char *fmt,
const detail::FormatArg *formatters,
int numFormatters) {
// Saved stream state
std::streamsize origWidth = out.width();
std::streamsize origPrecision = out.precision();
......@@ -746,9 +766,13 @@ inline void formatImpl(std::ostream &out, const char *fmt,
fmt = printFormatStringLiteral(out, fmt);
bool spacePadPositive = false;
int ntrunc = -1;
const char *fmtEnd =
streamStateFromFormat(out, spacePadPositive, ntrunc, fmt, formatters,
argIndex, numFormatters);
const char *fmtEnd = streamStateFromFormat(out,
spacePadPositive,
ntrunc,
fmt,
formatters,
argIndex,
numFormatters);
if (argIndex >= numFormatters) {
// Check args remain after reading any variable width/precision
TINYFORMAT_ERROR("tinyformat: Not enough format arguments");
......@@ -756,9 +780,9 @@ inline void formatImpl(std::ostream &out, const char *fmt,
}
const FormatArg &arg = formatters[argIndex];
// Format the arg into the stream.
if (!spacePadPositive)
if (!spacePadPositive) {
arg.format(out, fmt, fmtEnd, ntrunc);
else {
} else {
// The following is a special case with no direct correspondence
// between stream formatting and the printf() behaviour. Simulate
// it crudely by formatting into a temporary string stream and
......@@ -801,7 +825,8 @@ class FormatList {
FormatList(detail::FormatArg *formatters, int N)
: m_formatters(formatters), m_N(N) {}
friend void vformat(std::ostream &out, const char *fmt,
friend void vformat(std::ostream &out,
const char *fmt,
const FormatList &list);
private:
......@@ -819,7 +844,7 @@ template <int N>
class FormatListN : public FormatList {
public:
template <typename... Args>
FormatListN(const Args &... args)
FormatListN(const Args &... args) // NOLINT
: FormatList(&m_formatterStore[0], N),
m_formatterStore{FormatArg(args)...} {
static_assert(sizeof...(args) == N, "Number of args must be N");
......@@ -850,7 +875,7 @@ class FormatListN<0> : public FormatList {
template <typename... Args>
detail::FormatListN<sizeof...(Args)> makeFormatList(const Args &... args) {
return detail::FormatListN<sizeof...(args)>(args...);
}
} // NOLINT
/// Format list of arguments to the stream according to the given format string.
///
......
paddle/utils/string/to_string.h 0 → 100644
浏览文件 @ 6ca49164
/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#pragma once
#include <sstream>
#include <string>
#include <type_traits>
#include <typeindex>
#include <vector>
namespace paddle {
namespace string {
inline std::ostream& operator<<(std::ostream& s, const std::type_index& t) {
s << t.name();
return s;
}
template <typename T,
typename std::enable_if<!std::is_enum<T>::value, int>::type = 0>
inline std::string to_string(T v) {
std::ostringstream sout;
sout << v;
return sout.str();
}
template <typename T,
typename std::enable_if<std::is_enum<T>::value, int>::type = 0>
inline std::string to_string(T v) {
return std::to_string(static_cast<int>(v));
}
template <>
inline std::string to_string(std::type_index t) {
return t.name();
}
// Faster std::string/const char* type
template <>
inline std::string to_string(std::string v) {
return v;
}
template <>
inline std::string to_string(const char* v) {
return std::string(v);
}
} // namespace string
} // namespace paddle
paddle/fluid/string/to_string_test.cc → paddle/utils/string/to_string_test.cc
浏览文件 @ 6ca49164
......@@ -12,7 +12,7 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License. */
#include "paddle/fluid/string/to_string.h"
#include "paddle/utils/string/to_string.h"
#include <gtest/gtest.h>
constexpr char kOutputString[] = "User Defined Output";
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或