提交 365be5d5 编写于 作者: D dongdaxiang

support win32 flag in io.cc shell.cc, fix code style problem in fleet_wrapper,...

support win32 flag in io.cc shell.cc, fix code style problem in fleet_wrapper, fix lodtensor_printer_test problem
test=develop
上级 dc8cf36e
...@@ -235,8 +235,8 @@ void FleetWrapper::PushDenseParamSync( ...@@ -235,8 +235,8 @@ void FleetWrapper::PushDenseParamSync(
} }
} }
int cnt = 1; int cnt = 1;
for (auto& i: dim) { for (auto& i : dim) {
cnt *= i; cnt *= i;
} }
DDim d(std::vector<int64_t>{cnt}.data(), 1); DDim d(std::vector<int64_t>{cnt}.data(), 1);
float* g = tensor->mutable_data<float>(d, place); float* g = tensor->mutable_data<float>(d, place);
...@@ -254,8 +254,7 @@ void FleetWrapper::PushDenseParamSync( ...@@ -254,8 +254,7 @@ void FleetWrapper::PushDenseParamSync(
regions.data(), regions.size(), table_id); regions.data(), regions.size(), table_id);
push_status.wait(); push_status.wait();
auto status = push_status.get(); auto status = push_status.get();
CHECK(status == 0) << "push dense param failed, status[" CHECK(status == 0) << "push dense param failed, status[" << status << "]";
<< status << "]";
} }
#endif #endif
} }
...@@ -346,13 +345,14 @@ void FleetWrapper::PushSparseVarsWithLabelAsync( ...@@ -346,13 +345,14 @@ void FleetWrapper::PushSparseVarsWithLabelAsync(
#endif #endif
} }
int FleetWrapper::RegisterClientToClientMsgHandler( int FleetWrapper::RegisterClientToClientMsgHandler(int msg_type,
int msg_type, MsgHandlerFunc handler) { MsgHandlerFunc handler) {
#ifdef PADDLE_WITH_PSLIB #ifdef PADDLE_WITH_PSLIB
VLOG(3) << "calling FleetWrapper::RegisterClientToClientMsgHandler"; VLOG(3) << "calling FleetWrapper::RegisterClientToClientMsgHandler";
VLOG(3) << "pslib_ptr_=" << pslib_ptr_; VLOG(3) << "pslib_ptr_=" << pslib_ptr_;
VLOG(3) << "_worker_ptr=" << pslib_ptr_->_worker_ptr; VLOG(3) << "_worker_ptr=" << pslib_ptr_->_worker_ptr;
return pslib_ptr_->_worker_ptr->registe_client2client_msg_handler(msg_type, handler); return pslib_ptr_->_worker_ptr->registe_client2client_msg_handler(msg_type,
handler);
#else #else
VLOG(0) << "FleetWrapper::RegisterClientToClientMsgHandler" VLOG(0) << "FleetWrapper::RegisterClientToClientMsgHandler"
<< " does nothing when no pslib"; << " does nothing when no pslib";
...@@ -363,7 +363,8 @@ int FleetWrapper::RegisterClientToClientMsgHandler( ...@@ -363,7 +363,8 @@ int FleetWrapper::RegisterClientToClientMsgHandler(
std::future<int32_t> FleetWrapper::SendClientToClientMsg( std::future<int32_t> FleetWrapper::SendClientToClientMsg(
int msg_type, int to_client_id, const std::string& msg) { int msg_type, int to_client_id, const std::string& msg) {
#ifdef PADDLE_WITH_PSLIB #ifdef PADDLE_WITH_PSLIB
return pslib_ptr_->_worker_ptr->send_client2client_msg(msg_type, to_client_id, msg); return pslib_ptr_->_worker_ptr->send_client2client_msg(msg_type, to_client_id,
msg);
#else #else
VLOG(0) << "FleetWrapper::SendClientToClientMsg" VLOG(0) << "FleetWrapper::SendClientToClientMsg"
<< " does nothing when no pslib"; << " does nothing when no pslib";
......
...@@ -19,6 +19,7 @@ namespace framework { ...@@ -19,6 +19,7 @@ namespace framework {
std::shared_ptr<FILE> shell_fopen(const std::string& path, std::shared_ptr<FILE> shell_fopen(const std::string& path,
const std::string& mode) { const std::string& mode) {
#ifndef _WIN32
if (shell_verbose()) { if (shell_verbose()) {
LOG(INFO) << "Opening file[" << path << "] with mode[" << mode << "]"; LOG(INFO) << "Opening file[" << path << "] with mode[" << mode << "]";
} }
...@@ -34,12 +35,16 @@ std::shared_ptr<FILE> shell_fopen(const std::string& path, ...@@ -34,12 +35,16 @@ std::shared_ptr<FILE> shell_fopen(const std::string& path,
LOG(FATAL) << "fclose fail, path[" << path << "]"; LOG(FATAL) << "fclose fail, path[" << path << "]";
} }
}}; }};
#else
return nullptr;
#endif
} }
// Close all open file descriptors // Close all open file descriptors
// The implementation is async signal safe // The implementation is async signal safe
// Mostly copy from CPython code // Mostly copy from CPython code
static int close_open_fds_internal() { static int close_open_fds_internal() {
#ifndef _WIN32
struct linux_dirent { struct linux_dirent {
long d_ino = 0; // NOLINT long d_ino = 0; // NOLINT
off_t d_off; off_t d_off;
...@@ -86,11 +91,13 @@ static int close_open_fds_internal() { ...@@ -86,11 +91,13 @@ static int close_open_fds_internal() {
} }
close(dir_fd); close(dir_fd);
#endif
return 0; return 0;
} }
static int shell_popen_fork_internal(const char* real_cmd, bool do_read, static int shell_popen_fork_internal(const char* real_cmd, bool do_read,
int parent_end, int child_end) { int parent_end, int child_end) {
#ifndef _WIN32
int child_pid = -1; int child_pid = -1;
// Too frequent calls to fork() makes openmpi very slow. Use vfork() instead. // Too frequent calls to fork() makes openmpi very slow. Use vfork() instead.
// But vfork() is very dangerous. Be careful. // But vfork() is very dangerous. Be careful.
...@@ -119,10 +126,13 @@ static int shell_popen_fork_internal(const char* real_cmd, bool do_read, ...@@ -119,10 +126,13 @@ static int shell_popen_fork_internal(const char* real_cmd, bool do_read,
return -1; return -1;
} }
exit(127); exit(127);
#endif
return 0;
} }
std::shared_ptr<FILE> shell_popen(const std::string& cmd, std::shared_ptr<FILE> shell_popen(const std::string& cmd,
const std::string& mode, int* err_no) { const std::string& mode, int* err_no) {
#ifndef _WIN32
bool do_read = mode == "r"; bool do_read = mode == "r";
bool do_write = mode == "w"; bool do_write = mode == "w";
if (!(do_read || do_write)) { if (!(do_read || do_write)) {
...@@ -170,12 +180,9 @@ std::shared_ptr<FILE> shell_popen(const std::string& cmd, ...@@ -170,12 +180,9 @@ std::shared_ptr<FILE> shell_popen(const std::string& cmd,
*err_no = -1; *err_no = -1;
} }
int wstatus = -1; int wstatus = -1;
// int ret = waitpid(child_pid, &wstatus, 0);
waitpid(child_pid, &wstatus, 0); waitpid(child_pid, &wstatus, 0);
if (wstatus == 0 || wstatus == (128 + SIGPIPE) * 256 || if (wstatus == 0 || wstatus == (128 + SIGPIPE) * 256 ||
(wstatus == -1 && errno == ECHILD)) { (wstatus == -1 && errno == ECHILD)) {
// LOG(INFO) << "status[" << wstatus << "], cmd[" << cmd << "]" <<
// ", err_no[" << *err_no << "]";
} else { } else {
*err_no = -1; *err_no = -1;
LOG(WARNING) << "status[" << wstatus << "], cmd[" << cmd << "]" LOG(WARNING) << "status[" << wstatus << "], cmd[" << cmd << "]"
...@@ -185,10 +192,12 @@ std::shared_ptr<FILE> shell_popen(const std::string& cmd, ...@@ -185,10 +192,12 @@ std::shared_ptr<FILE> shell_popen(const std::string& cmd,
LOG(WARNING) << "errno is ECHILD"; LOG(WARNING) << "errno is ECHILD";
} }
}}; }};
#endif
} }
static int shell_p2open_fork_internal(const char* real_cmd, int pipein_fds[2], static int shell_p2open_fork_internal(const char* real_cmd, int pipein_fds[2],
int pipeout_fds[2]) { int pipeout_fds[2]) {
#ifndef
int child_pid = -1; int child_pid = -1;
if ((child_pid = fork()) < 0) { if ((child_pid = fork()) < 0) {
return -1; return -1;
...@@ -220,10 +229,13 @@ static int shell_p2open_fork_internal(const char* real_cmd, int pipein_fds[2], ...@@ -220,10 +229,13 @@ static int shell_p2open_fork_internal(const char* real_cmd, int pipein_fds[2],
return -1; return -1;
} }
exit(127); exit(127);
#endif
return 0;
} }
std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open( std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open(
const std::string& cmd) { const std::string& cmd) {
#ifndef _WIN32
if (shell_verbose()) { if (shell_verbose()) {
LOG(INFO) << "Opening bidirectional pipe[" << cmd << "]"; LOG(INFO) << "Opening bidirectional pipe[" << cmd << "]";
} }
...@@ -275,9 +287,13 @@ std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open( ...@@ -275,9 +287,13 @@ std::pair<std::shared_ptr<FILE>, std::shared_ptr<FILE>> shell_p2open(
PCHECK((out_fp = fdopen(pipeout_fds[1], "w")) != NULL); PCHECK((out_fp = fdopen(pipeout_fds[1], "w")) != NULL);
return {{in_fp, [child_life](FILE* fp) { PCHECK(fclose(fp) == 0); }}, return {{in_fp, [child_life](FILE* fp) { PCHECK(fclose(fp) == 0); }},
{out_fp, [child_life](FILE* fp) { PCHECK(fclose(fp) == 0); }}}; {out_fp, [child_life](FILE* fp) { PCHECK(fclose(fp) == 0); }}};
#else
return nullptr;
#endif
} }
std::string shell_get_command_output(const std::string& cmd) { std::string shell_get_command_output(const std::string& cmd) {
#ifndef _WIN32
int err_no = 0; int err_no = 0;
do { do {
err_no = 0; err_no = 0;
...@@ -291,7 +307,7 @@ std::string shell_get_command_output(const std::string& cmd) { ...@@ -291,7 +307,7 @@ std::string shell_get_command_output(const std::string& cmd) {
} }
} }
} while (err_no == -1); } while (err_no == -1);
#endif
return ""; return "";
} }
} // end namespace framework } // end namespace framework
......
...@@ -17,30 +17,8 @@ ...@@ -17,30 +17,8 @@
#include "paddle/fluid/framework/variable.h" #include "paddle/fluid/framework/variable.h"
TEST(LodTensorPrinter, PrintVar) { TEST(LodTensorPrinter, PrintVar) {
Scope scope; paddle::framework::Scope scope;
PrintVar(&scope, "NotAVar"); PrintVar(&scope, "NotAVar", "We don't have var");
Variable* v = scope.Var("NotAVar"); paddle::framework::Variable* v = scope.Var("NotAVar");
PrintVar(&scope, "NotAVar"); PrintVar(&scope, "NotAVar", "Now we have a var");
}
TEST(Timer, Start) {
paddle::platform::Timer timeline;
timeline.Start();
sleep(3);
timeline.Pause();
}
TEST(Timer, Pause) {
paddle::platform::Timer timeline;
timeline.Start();
sleep(3);
timeline.Pause();
}
TEST(Timer, Resume) {
paddle::platform::Timer timeline;
timeline.Start();
sleep(3);
timeline.Pause();
timeline.Resume();
} }
...@@ -211,7 +211,7 @@ class LineFileReader { ...@@ -211,7 +211,7 @@ class LineFileReader {
~LineFileReader() { ::free(_buffer); } ~LineFileReader() { ::free(_buffer); }
char* getline(FILE* f) { return this->getdelim(f, '\n'); } char* getline(FILE* f) { return this->getdelim(f, '\n'); }
char* getdelim(FILE* f, char delim) { char* getdelim(FILE* f, char delim) {
ssize_t ret = ::getdelim(&_buffer, &_buf_size, delim, f); int32_t ret = ::getdelim(&_buffer, &_buf_size, delim, f);
if (ret >= 0) { if (ret >= 0) {
if (ret >= 1 && _buffer[ret - 1] == delim) { if (ret >= 1 && _buffer[ret - 1] == delim) {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册