From 6fb28796f5bc55d275035ff42d86447fe7c1df82 Mon Sep 17 00:00:00 2001 From: dzhwinter Date: Mon, 3 Sep 2018 15:45:38 +0800 Subject: [PATCH] memory (#13143) --- paddle/fluid/operators/save_combine_op.cc | 32 +----- paddle/fluid/operators/save_op.cc | 32 +----- paddle/fluid/platform/port.h | 131 ++++++++++++++++++++-- 3 files changed, 126 insertions(+), 69 deletions(-) diff --git a/paddle/fluid/operators/save_combine_op.cc b/paddle/fluid/operators/save_combine_op.cc index cfee920708..5b05f757c0 100644 --- a/paddle/fluid/operators/save_combine_op.cc +++ b/paddle/fluid/operators/save_combine_op.cc @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. */ #include -#include #include #include #include @@ -23,40 +22,11 @@ limitations under the License. */ #include "paddle/fluid/framework/lod_tensor.h" #include "paddle/fluid/framework/op_registry.h" #include "paddle/fluid/platform/device_context.h" +#include "paddle/fluid/platform/port.h" namespace paddle { namespace operators { -// TODO(sidgoyal78): These function are needed by other files (save_op), move -// them to paddle::filesystem namespace. (as noted by yuyang18 in save_op). -constexpr char kSEP = '/'; -static bool FileExists(const std::string &filepath) { - struct stat buffer; - return (stat(filepath.c_str(), &buffer) == 0); -} - -static std::string DirName(const std::string &filepath) { - auto pos = filepath.rfind(kSEP); - if (pos == std::string::npos) { - return ""; - } - return filepath.substr(0, pos); -} - -static void MkDir(const char *path) { - if (mkdir(path, 0755)) { - PADDLE_ENFORCE_EQ(errno, EEXIST, "%s mkdir failed!", path); - } -} - -static void MkDirRecursively(const char *fullpath) { - if (*fullpath == '\0') return; // empty string - if (FileExists(fullpath)) return; - - MkDirRecursively(DirName(fullpath).c_str()); - MkDir(fullpath); -} - class SaveCombineOp : public framework::OperatorBase { public: SaveCombineOp(const std::string &type, diff --git a/paddle/fluid/operators/save_op.cc b/paddle/fluid/operators/save_op.cc index 85de37416b..e79cffcf49 100644 --- a/paddle/fluid/operators/save_op.cc +++ b/paddle/fluid/operators/save_op.cc @@ -13,7 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. */ #include -#include #include #include @@ -25,6 +24,7 @@ limitations under the License. */ #include "paddle/fluid/framework/selected_rows.h" #include "paddle/fluid/framework/variable.h" #include "paddle/fluid/platform/device_context.h" +#include "paddle/fluid/platform/port.h" namespace paddle { namespace operators { @@ -33,36 +33,6 @@ namespace operators { // to directory specified. constexpr char LOOKUP_TABLE_PATH[] = "kLookupTablePath"; -// TODO(yuyang18): If the functions below are needed by other files, move them -// to paddle::filesystem namespace. -constexpr char kSEP = '/'; -static bool FileExists(const std::string &filepath) { - struct stat buffer; - return (stat(filepath.c_str(), &buffer) == 0); -} - -static std::string DirName(const std::string &filepath) { - auto pos = filepath.rfind(kSEP); - if (pos == std::string::npos) { - return ""; - } - return filepath.substr(0, pos); -} - -static void MkDir(const char *path) { - if (mkdir(path, 0755)) { - PADDLE_ENFORCE_EQ(errno, EEXIST, "%s mkdir failed!", path); - } -} - -static void MkDirRecursively(const char *fullpath) { - if (*fullpath == '\0') return; // empty string - if (FileExists(fullpath)) return; - - MkDirRecursively(DirName(fullpath).c_str()); - MkDir(fullpath); -} - class SaveOp : public framework::OperatorBase { public: SaveOp(const std::string &type, const framework::VariableNameMap &inputs, diff --git a/paddle/fluid/platform/port.h b/paddle/fluid/platform/port.h index a0a2d29500..cf9f4aa95b 100644 --- a/paddle/fluid/platform/port.h +++ b/paddle/fluid/platform/port.h @@ -14,24 +14,141 @@ #pragma once +#include #include + +#include #include +#define GLOG_NO_ABBREVIATED_SEVERITIES // msvc conflict logging with windows.h +#include "glog/logging.h" + #if !defined(_WIN32) -#include // for dladdr -#include // for backtrace +#define UNUSED __attribute__((unused)) +#include // dladdr +#include // backtrace +#include +#include // std::accumulate #else -#include -#include +#include // _popen, _pclose +#include +#if defined(_WIN32) +#include // std::accumulate in msvc +#endif +// windows version of __attribute__((unused)) +#define UNUSED __pragma(warning(suppress : 4100)) -static void* dlsym(void* handle, const char* symbol_name) { +#ifndef S_ISDIR // windows port for sys/stat.h +#define S_ISDIR(mode) (((mode)&S_IFMT) == S_IFDIR) +#endif // S_ISDIR + +static void *dlsym(void *handle, const char *symbol_name) { FARPROC found_symbol; found_symbol = GetProcAddress((HMODULE)handle, symbol_name); if (found_symbol == NULL) { throw std::runtime_error(std::string(symbol_name) + " not found."); } - return reinterpret_cast(found_symbol); + return reinterpret_cast(found_symbol); } -#endif +static void *dlopen(const char *filename, int flag) { + std::string file_name(filename); + file_name.replace(0, file_name.size() - 1, '/', '\\'); + HMODULE hModule = LoadLibrary(file_name.c_str()); + if (!hModule) { + throw std::runtime_error(file_name + " not found."); + } + return reinterpret_cast(hModule); +} + +#endif // !_WIN32 + +static void ExecShellCommand(const std::string &cmd, std::string *message) { + char buffer[128]; +#if !defined(_WIN32) + std::shared_ptr pipe(popen(cmd.c_str(), "r"), pclose); +#else + std::shared_ptr pipe(_popen(cmd.c_str(), "r"), _pclose); +#endif // _WIN32 + if (!pipe) { + LOG(ERROR) << "error running command: " << cmd; + return; + } + while (!feof(pipe.get())) { + if (fgets(buffer, 128, pipe.get()) != nullptr) { + *message += buffer; + } + } +} + +static bool PathExists(const std::string &path) { +#if !defined(_WIN32) + struct stat statbuf; + if (stat(path.c_str(), &statbuf) != -1) { + if (S_ISDIR(statbuf.st_mode)) { + return true; + } + } +#else + struct _stat statbuf; + if (_stat(path.c_str(), &statbuf) != -1) { + if (S_ISDIR(statbuf.st_mode)) { + return true; + } + } +#endif // !_WIN32 + return false; +} + +// TODO(yuyang18): If the functions below are needed by other files, move them +// to paddle::filesystem namespace. +#if !defined(_WIN32) +constexpr char kSEP = '/'; +#else +constexpr char kSEP = '\\'; +#endif // _WIN32 + +static bool FileExists(const std::string &filepath) { +#if !defined(_WIN32) + struct stat buffer; + return (stat(filepath.c_str(), &buffer) == 0); +#else + struct _stat buffer; + return (_stat(filepath.c_str(), &buffer) == 0); +#endif // !_WIN32 +} + +static std::string DirName(const std::string &filepath) { + auto pos = filepath.rfind(kSEP); + if (pos == std::string::npos) { + return ""; + } + return filepath.substr(0, pos); +} + +static void MkDir(const char *path) { + std::string path_error(path); + path_error += " mkdir failed!"; +#if !defined(_WIN32) + if (mkdir(path, 0755)) { + if (errno != EEXIST) { + throw std::runtime_error(path_error); + } + } +#else + CreateDirectory(path, NULL); + auto errorno = GetLastError(); + if (errorno != ERROR_ALREADY_EXISTS) { + throw std::runtime_error(path_error); + } +#endif // !_WIN32 +} + +static void MkDirRecursively(const char *fullpath) { + if (*fullpath == '\0') return; // empty string + if (FileExists(fullpath)) return; + + MkDirRecursively(DirName(fullpath).c_str()); + MkDir(fullpath); +} -- GitLab