diff --git a/paddle/fluid/framework/fleet/fleet_wrapper.cc b/paddle/fluid/framework/fleet/fleet_wrapper.cc index 9bc0029d08d4aa9ee57b3448ad813c304563bca7..f4522fd34d24626e40441b26e0a166ca6503bc06 100644 --- a/paddle/fluid/framework/fleet/fleet_wrapper.cc +++ b/paddle/fluid/framework/fleet/fleet_wrapper.cc @@ -42,13 +42,13 @@ std::shared_ptr FleetWrapper::pslib_ptr_ = NULL; void FleetWrapper::InitServer(const std::string& dist_desc, int index) { #ifdef PADDLE_WITH_PSLIB if (!is_initialized_) { - LOG(WARNING) << "Going to init server"; + VLOG(3) << "Going to init server"; pslib_ptr_ = std::shared_ptr( new paddle::distributed::PSlib()); pslib_ptr_->init_server(dist_desc, index); is_initialized_ = true; } else { - LOG(WARNING) << "Server can be initialized only once"; + VLOG(3) << "Server can be initialized only once"; } #endif } @@ -58,7 +58,7 @@ void FleetWrapper::InitWorker(const std::string& dist_desc, int node_num, int index) { #ifdef PADDLE_WITH_PSLIB if (!is_initialized_) { - LOG(WARNING) << "Going to init server"; + VLOG(3) << "Going to init worker"; pslib_ptr_ = std::shared_ptr( new paddle::distributed::PSlib()); pslib_ptr_->init_worker(dist_desc, @@ -66,21 +66,21 @@ void FleetWrapper::InitWorker(const std::string& dist_desc, node_num, index); is_initialized_ = true; } else { - LOG(WARNING) << "Worker can be initialized only once"; + VLOG(3) << "Worker can be initialized only once"; } #endif } void FleetWrapper::StopServer() { #ifdef PADDLE_WITH_PSLIB - LOG(WARNING) << "Going to stop server"; + VLOG(3) << "Going to stop server"; pslib_ptr_->stop_server(); #endif } uint64_t FleetWrapper::RunServer() { #ifdef PADDLE_WITH_PSLIB - LOG(WARNING) << "Going to run server"; + VLOG(3) << "Going to run server"; return pslib_ptr_->run_server(); #else return 0; @@ -90,7 +90,7 @@ uint64_t FleetWrapper::RunServer() { void FleetWrapper::GatherServers(const std::vector& host_sign_list, int node_num) { #ifdef PADDLE_WITH_PSLIB - LOG(WARNING) << "Going to gather server ips"; + VLOG(3) << "Going to gather server ips"; pslib_ptr_->gather_servers(const_cast(host_sign_list.data()), node_num); #endif diff --git a/paddle/fluid/framework/multi_trainer.cc b/paddle/fluid/framework/multi_trainer.cc index 969d27c8efe5be002acd15f5e432cbb431cc4fc0..6c9fa960841fd95f85eee9a227f43efd93af553f 100644 --- a/paddle/fluid/framework/multi_trainer.cc +++ b/paddle/fluid/framework/multi_trainer.cc @@ -39,6 +39,7 @@ void MultiTrainer::Initialize(const TrainerDesc& trainer_desc) { for (unsigned i = 0; i < trainer_desc.filelist_size(); ++i) { filelist_vec.push_back(trainer_desc.filelist(i)); } + readers_[0]->SetFileList(filelist_vec); } // call only after all resources are set in current trainer diff --git a/python/paddle/fluid/trainer_desc.py b/python/paddle/fluid/trainer_desc.py index 6e66706bb71a43bbc6ee7407821e8bd304ded24d..1805362f9fbd7ef5d5bf53933a8efd0607703bf4 100644 --- a/python/paddle/fluid/trainer_desc.py +++ b/python/paddle/fluid/trainer_desc.py @@ -29,7 +29,9 @@ class TrainerDesc(object): text_format.Parse(f.read(), self.proto_desc) ''' self.proto_desc = trainer_desc_pb2.TrainerDesc() - self.proto_desc.thread_num = 12 + import multiprocessing as mp + # set default thread num == cpu count + self.proto_desc.thread_num = mp.cpu_count() def set_thread(self, thread_num): self.proto_desc.thread_num = thread_num