From be757096dab84b1b8777cb2fe3f55907f4aefb02 Mon Sep 17 00:00:00 2001 From: dongdaxiang Date: Thu, 28 Feb 2019 22:22:28 +0800 Subject: [PATCH] add pybind for fleet --- paddle/fluid/framework/async_executor.cc | 4 -- paddle/fluid/framework/data_feed.h | 3 -- paddle/fluid/framework/fleet/fleet_wrapper.h | 4 +- paddle/fluid/pybind/CMakeLists.txt | 6 ++- paddle/fluid/pybind/fleet_wrapper_py.cc | 52 ++++++++++++++++++++ paddle/fluid/pybind/fleet_wrapper_py.h | 28 +++++++++++ paddle/fluid/pybind/pybind.cc | 2 +- 7 files changed, 87 insertions(+), 12 deletions(-) create mode 100644 paddle/fluid/pybind/fleet_wrapper_py.cc create mode 100644 paddle/fluid/pybind/fleet_wrapper_py.h diff --git a/paddle/fluid/framework/async_executor.cc b/paddle/fluid/framework/async_executor.cc index 67770f77c2..27c06f5aa1 100644 --- a/paddle/fluid/framework/async_executor.cc +++ b/paddle/fluid/framework/async_executor.cc @@ -59,10 +59,6 @@ void AsyncExecutor::GatherServers(const std::vector& host_sign_list, fleet_ptr_->GatherServers(host_sign_list, node_num); } -void AsyncExecutor::InitModel() {} - -void AsyncExecutor::SaveModel(const std::string& path) {} - void AsyncExecutor::RunFromFile(const ProgramDesc& main_program, const std::string& data_feed_desc_str, const std::vector& filelist, diff --git a/paddle/fluid/framework/data_feed.h b/paddle/fluid/framework/data_feed.h index 59ad90afe1..506af02f32 100644 --- a/paddle/fluid/framework/data_feed.h +++ b/paddle/fluid/framework/data_feed.h @@ -243,9 +243,6 @@ class MultiSlotDataFeed virtual bool ParseOneInstance(std::vector* instance); virtual bool ParseOneInstanceFromPipe(std::vector* instance); virtual void PutToFeedVec(const std::vector& ins_vec); - - private: - BatchGenerator batch_gen_; }; } // namespace framework } // namespace paddle diff --git a/paddle/fluid/framework/fleet/fleet_wrapper.h b/paddle/fluid/framework/fleet/fleet_wrapper.h index ba393886c9..edac3e4141 100644 --- a/paddle/fluid/framework/fleet/fleet_wrapper.h +++ b/paddle/fluid/framework/fleet/fleet_wrapper.h @@ -49,6 +49,7 @@ namespace framework { class FleetWrapper { public: virtual ~FleetWrapper() {} + FleetWrapper() {} // Pull sparse variables from server in Sync mode // Param: scope, table_id, var_names, fea_keys // Param: fea_values @@ -123,9 +124,6 @@ class FleetWrapper { private: static std::shared_ptr s_instance_; - private: - FleetWrapper() {} - protected: static bool is_initialized_; DISABLE_COPY_AND_ASSIGN(FleetWrapper); diff --git a/paddle/fluid/pybind/CMakeLists.txt b/paddle/fluid/pybind/CMakeLists.txt index 0991eff0fd..8207f2b72c 100644 --- a/paddle/fluid/pybind/CMakeLists.txt +++ b/paddle/fluid/pybind/CMakeLists.txt @@ -1,11 +1,15 @@ -set(PYBIND_DEPS pybind python proto_desc memory executor async_executor prune +set(PYBIND_DEPS pybind python proto_desc memory executor async_executor fleet_wrapper prune feed_fetch_method pass_builder parallel_executor profiler layer scope_pool tracer analysis_predictor imperative_profiler) if(WITH_PYTHON) list(APPEND PYBIND_DEPS py_func_op) endif() +<<<<<<< HEAD set(PYBIND_SRCS pybind.cc exception.cc protobuf.cc const_value.cc recordio.cc reader_py.cc async_executor_py.cc imperative.cc ir.cc inference_api.cc) +======= +set(PYBIND_SRCS pybind.cc exception.cc protobuf.cc const_value.cc recordio.cc async_executor_py.cc fleet_wrapper_py.cc imperative.cc ir.cc inference_api.cc) +>>>>>>> add pybind for fleet if(WITH_PYTHON) if(WITH_AMD_GPU) diff --git a/paddle/fluid/pybind/fleet_wrapper_py.cc b/paddle/fluid/pybind/fleet_wrapper_py.cc new file mode 100644 index 0000000000..65f71096e9 --- /dev/null +++ b/paddle/fluid/pybind/fleet_wrapper_py.cc @@ -0,0 +1,52 @@ +/* Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */ +#include + +#ifdef _POSIX_C_SOURCE +#undef _POSIX_C_SOURCE +#endif + +#ifdef _XOPEN_SOURCE +#undef _XOPEN_SOURCE +#endif + +#include +#include + +#include "google/protobuf/io/zero_copy_stream_impl.h" +#include "google/protobuf/text_format.h" +#include "paddle/fluid/framework/async_executor.h" +#include "paddle/fluid/framework/data_feed.h" +#include "paddle/fluid/framework/data_feed.pb.h" +#include "paddle/fluid/framework/scope.h" +#include "paddle/fluid/inference/io.h" +#include "paddle/fluid/platform/place.h" +#include "paddle/fluid/platform/variant.h" +#include "paddle/fluid/pybind/fleet_wrapper_py.h" + +namespace py = pybind11; +namespace pd = paddle::framework; + +namespace paddle { +namespace pybind { +void BindFleetWrapper(py::module* m) { + py::class_(*m, "Fleet") + .def(py::init()) + .def("init_server", &framework::FleetWrapper::InitServer) + .def("init_worker", &framework::FleetWrapper::InitWorker) + .def("stop_server", &framework::FleetWrapper::StopServer) + .def("gather_servers", &framework::FleetWrapper::GatherServers); +} // end FleetWrapper +} // end namespace pybind +} // end namespace paddle diff --git a/paddle/fluid/pybind/fleet_wrapper_py.h b/paddle/fluid/pybind/fleet_wrapper_py.h new file mode 100644 index 0000000000..b2bfa10eec --- /dev/null +++ b/paddle/fluid/pybind/fleet_wrapper_py.h @@ -0,0 +1,28 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "pybind11/pybind11.h" +#include "pybind11/stl.h" + +namespace py = pybind11; + +namespace paddle { +namespace pybind { + +void BindFleetWrapper(py::module* m); + +} // namespace pybind +} // namespace paddle diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index fa978f1c99..e1ef00681c 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -51,6 +51,7 @@ limitations under the License. */ #include "paddle/fluid/pybind/async_executor_py.h" #include "paddle/fluid/pybind/const_value.h" #include "paddle/fluid/pybind/exception.h" +#include "paddle/fluid/pybind/fleet_wrapper_py.h" #include "paddle/fluid/pybind/imperative.h" #include "paddle/fluid/pybind/inference_api.h" #include "paddle/fluid/pybind/ir.h" @@ -59,7 +60,6 @@ limitations under the License. */ #include "paddle/fluid/pybind/reader_py.h" #include "paddle/fluid/pybind/recordio.h" #include "paddle/fluid/pybind/tensor_py.h" - #include "paddle/fluid/string/to_string.h" #ifdef PADDLE_WITH_CUDA -- GitLab