未验证 提交 52c1431e 编写于 作者: T Thunderbrook 提交者: GitHub

add clear_model interface in fleetwrapper (#18815)

* dump slot

* test

* proto

* dump slot

* test

* proto

* code style

* code style

* code style

* style

* add delete after unseen days

* add unseen days

* code style

* conflict solve
test=develop

* add clear model

* code style
test=develop

* code style
test=develop
上级 9a8a7a1d
...@@ -521,6 +521,15 @@ void FleetWrapper::ShrinkSparseTable(int table_id) { ...@@ -521,6 +521,15 @@ void FleetWrapper::ShrinkSparseTable(int table_id) {
#endif #endif
} }
void FleetWrapper::ClearModel() {
#ifdef PADDLE_WITH_PSLIB
auto ret = pslib_ptr_->_worker_ptr->clear();
ret.wait();
#else
VLOG(0) << "FleetWrapper::ClearModel does nothing when no pslib";
#endif
}
void FleetWrapper::ShrinkDenseTable(int table_id, Scope* scope, void FleetWrapper::ShrinkDenseTable(int table_id, Scope* scope,
std::vector<std::string> var_list, std::vector<std::string> var_list,
float decay, int emb_dim) { float decay, int emb_dim) {
......
...@@ -148,6 +148,7 @@ class FleetWrapper { ...@@ -148,6 +148,7 @@ class FleetWrapper {
// mode = 1, save delta feature, which means save diff // mode = 1, save delta feature, which means save diff
void SaveModel(const std::string& path, const int mode); void SaveModel(const std::string& path, const int mode);
void ClearModel();
void ShrinkSparseTable(int table_id); void ShrinkSparseTable(int table_id);
void ShrinkDenseTable(int table_id, Scope* scope, void ShrinkDenseTable(int table_id, Scope* scope,
std::vector<std::string> var_list, float decay, std::vector<std::string> var_list, float decay,
......
...@@ -49,6 +49,7 @@ void BindFleetWrapper(py::module* m) { ...@@ -49,6 +49,7 @@ void BindFleetWrapper(py::module* m) {
.def("init_model", &framework::FleetWrapper::PushDenseParamSync) .def("init_model", &framework::FleetWrapper::PushDenseParamSync)
.def("save_model", &framework::FleetWrapper::SaveModel) .def("save_model", &framework::FleetWrapper::SaveModel)
.def("load_model", &framework::FleetWrapper::LoadModel) .def("load_model", &framework::FleetWrapper::LoadModel)
.def("clear_model", &framework::FleetWrapper::ClearModel)
.def("stop_server", &framework::FleetWrapper::StopServer) .def("stop_server", &framework::FleetWrapper::StopServer)
.def("gather_servers", &framework::FleetWrapper::GatherServers) .def("gather_servers", &framework::FleetWrapper::GatherServers)
.def("gather_clients", &framework::FleetWrapper::GatherClients) .def("gather_clients", &framework::FleetWrapper::GatherClients)
......
...@@ -264,6 +264,21 @@ class PSLib(Fleet): ...@@ -264,6 +264,21 @@ class PSLib(Fleet):
decay, emb_dim) decay, emb_dim)
self._role_maker._barrier_worker() self._role_maker._barrier_worker()
def clear_model(self):
"""
clear_model() will be called by user. It will clear sparse model.
Examples:
.. code-block:: python
fleet.clear_model()
"""
self._role_maker._barrier_worker()
if self._role_maker.is_first_worker():
self._fleet_ptr.clear_model()
self._role_maker._barrier_worker()
def load_one_table(self, table_id, model_path, **kwargs): def load_one_table(self, table_id, model_path, **kwargs):
""" """
load pslib model for one table or load params from paddle model load pslib model for one table or load params from paddle model
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册