From 52c1431eeed854c55d94666da3f82ad9c6f8308f Mon Sep 17 00:00:00 2001 From: Thunderbrook <52529258+Thunderbrook@users.noreply.github.com> Date: Mon, 29 Jul 2019 14:18:19 +0800 Subject: [PATCH] add clear_model interface in fleetwrapper (#18815) * dump slot * test * proto * dump slot * test * proto * code style * code style * code style * style * add delete after unseen days * add unseen days * code style * conflict solve test=develop * add clear model * code style test=develop * code style test=develop --- paddle/fluid/framework/fleet/fleet_wrapper.cc | 9 +++++++++ paddle/fluid/framework/fleet/fleet_wrapper.h | 1 + paddle/fluid/pybind/fleet_wrapper_py.cc | 1 + .../fleet/parameter_server/pslib/__init__.py | 15 +++++++++++++++ 4 files changed, 26 insertions(+) diff --git a/paddle/fluid/framework/fleet/fleet_wrapper.cc b/paddle/fluid/framework/fleet/fleet_wrapper.cc index 0c5eb0da19..3f4f345912 100644 --- a/paddle/fluid/framework/fleet/fleet_wrapper.cc +++ b/paddle/fluid/framework/fleet/fleet_wrapper.cc @@ -521,6 +521,15 @@ void FleetWrapper::ShrinkSparseTable(int table_id) { #endif } +void FleetWrapper::ClearModel() { +#ifdef PADDLE_WITH_PSLIB + auto ret = pslib_ptr_->_worker_ptr->clear(); + ret.wait(); +#else + VLOG(0) << "FleetWrapper::ClearModel does nothing when no pslib"; +#endif +} + void FleetWrapper::ShrinkDenseTable(int table_id, Scope* scope, std::vector var_list, float decay, int emb_dim) { diff --git a/paddle/fluid/framework/fleet/fleet_wrapper.h b/paddle/fluid/framework/fleet/fleet_wrapper.h index 4779978689..17b58e5759 100644 --- a/paddle/fluid/framework/fleet/fleet_wrapper.h +++ b/paddle/fluid/framework/fleet/fleet_wrapper.h @@ -148,6 +148,7 @@ class FleetWrapper { // mode = 1, save delta feature, which means save diff void SaveModel(const std::string& path, const int mode); + void ClearModel(); void ShrinkSparseTable(int table_id); void ShrinkDenseTable(int table_id, Scope* scope, std::vector var_list, float decay, diff --git a/paddle/fluid/pybind/fleet_wrapper_py.cc b/paddle/fluid/pybind/fleet_wrapper_py.cc index e962587962..36fc0822e8 100644 --- a/paddle/fluid/pybind/fleet_wrapper_py.cc +++ b/paddle/fluid/pybind/fleet_wrapper_py.cc @@ -49,6 +49,7 @@ void BindFleetWrapper(py::module* m) { .def("init_model", &framework::FleetWrapper::PushDenseParamSync) .def("save_model", &framework::FleetWrapper::SaveModel) .def("load_model", &framework::FleetWrapper::LoadModel) + .def("clear_model", &framework::FleetWrapper::ClearModel) .def("stop_server", &framework::FleetWrapper::StopServer) .def("gather_servers", &framework::FleetWrapper::GatherServers) .def("gather_clients", &framework::FleetWrapper::GatherClients) diff --git a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py index b70a4f5558..92f1314816 100644 --- a/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py +++ b/python/paddle/fluid/incubate/fleet/parameter_server/pslib/__init__.py @@ -264,6 +264,21 @@ class PSLib(Fleet): decay, emb_dim) self._role_maker._barrier_worker() + def clear_model(self): + """ + clear_model() will be called by user. It will clear sparse model. + + Examples: + .. code-block:: python + + fleet.clear_model() + + """ + self._role_maker._barrier_worker() + if self._role_maker.is_first_worker(): + self._fleet_ptr.clear_model() + self._role_maker._barrier_worker() + def load_one_table(self, table_id, model_path, **kwargs): """ load pslib model for one table or load params from paddle model -- GitLab