diff --git a/paddle/fluid/framework/async_executor.cc b/paddle/fluid/framework/async_executor.cc index b13eefba2e6ce0613a35c01718c71d5f6df40403..89153d82d078b53d8d5582f0a38d3dafe21cc7eb 100644 --- a/paddle/fluid/framework/async_executor.cc +++ b/paddle/fluid/framework/async_executor.cc @@ -153,11 +153,14 @@ void AsyncExecutor::RunFromFile(const ProgramDesc& main_program, for (auto& th : threads) { th.join(); } + // TODO(guru4elephant): we don't need this + /* #ifdef PADDLE_WITH_PSLIB if (mode == "mpi") { _pull_dense_thread->stop(); } #endif + */ VLOG(3) << "start to run from files in async_executor"; VLOG(3) << "Drop current scope kids"; root_scope_->DropKids(); diff --git a/paddle/fluid/framework/downpour_worker.cc b/paddle/fluid/framework/downpour_worker.cc index dab3113fbc4e9eddc00704725175d7c4060a1287..4ca7842fa261a1b8178438d35ca5d626146663d4 100644 --- a/paddle/fluid/framework/downpour_worker.cc +++ b/paddle/fluid/framework/downpour_worker.cc @@ -210,6 +210,7 @@ void DownpourWorker::TrainFilesWithProfiler() { timeline.Pause(); pull_sparse_time += timeline.ElapsedSec(); total_time += timeline.ElapsedSec(); + timeline.Start(); CollectLabelInfo(i); timeline.Pause(); collect_label_time += timeline.ElapsedSec(); @@ -336,6 +337,16 @@ void DownpourWorker::TrainFilesWithProfiler() { } fprintf(stderr, "mean read time: %fs\n", read_time / batch_cnt); fprintf(stderr, "IO percent: %f\n", read_time / total_time * 100); + fprintf(stderr, "pull sparse time percent: %f\n", + pull_sparse_time / total_time * 100); + fprintf(stderr, "collect label time percent: %f\n", + collect_label_time / total_time * 100); + fprintf(stderr, "fill sparse time percent: %f\n", + fill_sparse_time / total_time * 100); + fprintf(stderr, "push sparse time percent: %f\n", + push_sparse_time / total_time * 100); + fprintf(stderr, "push dense time percent: %f\n", + push_dense_time / total_time * 100); fprintf(stderr, "%6.2f instances/s\n", total_inst / total_time); } } diff --git a/paddle/fluid/framework/fleet/fleet_wrapper.cc b/paddle/fluid/framework/fleet/fleet_wrapper.cc index 6af8ba951895659120a40edc19b9218b515fb743..72fd1a9cf1c57b591bfaee2a201dd8fe43ef8627 100644 --- a/paddle/fluid/framework/fleet/fleet_wrapper.cc +++ b/paddle/fluid/framework/fleet/fleet_wrapper.cc @@ -142,6 +142,7 @@ void FleetWrapper::PullSparseVarsSync( } fea_keys->push_back(static_cast(ids[i])); } + /* fea_values->resize(fea_keys->size() + 1); for (auto& t : *fea_values) { t.resize(fea_value_dim); @@ -150,10 +151,19 @@ void FleetWrapper::PullSparseVarsSync( for (auto& t : *fea_values) { pull_result_ptr.push_back(t.data()); } - auto status = pslib_ptr_->_worker_ptr->pull_sparse( - pull_result_ptr.data(), table_id, fea_keys->data(), fea_keys->size()); - pull_sparse_status.push_back(std::move(status)); + */ } + fea_values->resize(fea_keys->size() + 1); + for (auto& t : *fea_values) { + t.resize(fea_value_dim); + } + std::vector pull_result_ptr; + for (auto& t : *fea_values) { + pull_result_ptr.push_back(t.data()); + } + auto status = pslib_ptr_->_worker_ptr->pull_sparse( + pull_result_ptr.data(), table_id, fea_keys->data(), fea_keys->size()); + pull_sparse_status.push_back(std::move(status)); for (auto& t : pull_sparse_status) { t.wait(); auto status = t.get(); @@ -207,7 +217,7 @@ void FleetWrapper::PullDenseVarsSync( } void FleetWrapper::PushDenseParamSync( - const ProgramDesc& program, const uint64_t table_id, + const Scope& scope, const uint64_t table_id, const std::vector& var_names) { #ifdef PADDLE_WITH_PSLIB auto place = platform::CPUPlace(); diff --git a/paddle/fluid/framework/fleet/fleet_wrapper.h b/paddle/fluid/framework/fleet/fleet_wrapper.h index 40ed3c55113378976a1e0ed96d98aacdf70e79db..07eb670cbe43e6bd2b4444f252b579f6ca40bce1 100644 --- a/paddle/fluid/framework/fleet/fleet_wrapper.h +++ b/paddle/fluid/framework/fleet/fleet_wrapper.h @@ -73,7 +73,7 @@ class FleetWrapper { const std::vector& var_names, std::vector<::std::future>* pull_dense_status); - void PushDenseParamSync(const ProgramDesc& program, const uint64_t table_id, + void PushDenseParamSync(const Scope& scope, const uint64_t table_id, const std::vector& var_names); // Push dense variables to server in async mode diff --git a/paddle/fluid/platform/lodtensor_printer.cc b/paddle/fluid/platform/lodtensor_printer.cc index fb8e761f1a849a1500bcb8fcb978e96cf13dfa7b..a5aa1a4148686b032c52f99497252fde4867438f 100644 --- a/paddle/fluid/platform/lodtensor_printer.cc +++ b/paddle/fluid/platform/lodtensor_printer.cc @@ -41,7 +41,7 @@ void print_lod_tensor(const std::string& var_name, void PrintVar(framework::Scope* scope, const std::string& var_name, const std::string& print_info) { framework::Variable* var = scope->FindVar(var_name); - if (tensor == nullptr) { + if (var == nullptr) { VLOG(1) << "Variable Name " << var_name << " does not exist in your scope"; return; }