未验证 提交 68ea1ad5 编写于 作者: X xujiaqi01 提交者: GitHub

add clear one table (#23089)

* add clear_one_table
* test=develop
上级 4a54a464
...@@ -804,6 +804,15 @@ void FleetWrapper::ClearModel() { ...@@ -804,6 +804,15 @@ void FleetWrapper::ClearModel() {
#endif #endif
} }
void FleetWrapper::ClearOneTable(const uint64_t table_id) {
#ifdef PADDLE_WITH_PSLIB
auto ret = pslib_ptr_->_worker_ptr->clear(table_id);
ret.wait();
#else
VLOG(0) << "FleetWrapper::ClearOneTable does nothing when no pslib";
#endif
}
void FleetWrapper::ShrinkDenseTable(int table_id, Scope* scope, void FleetWrapper::ShrinkDenseTable(int table_id, Scope* scope,
std::vector<std::string> var_list, std::vector<std::string> var_list,
float decay, int emb_dim) { float decay, int emb_dim) {
......
...@@ -220,6 +220,8 @@ class FleetWrapper { ...@@ -220,6 +220,8 @@ class FleetWrapper {
const std::vector<uint64_t>& feasign_list); const std::vector<uint64_t>& feasign_list);
// clear all models, release their memory // clear all models, release their memory
void ClearModel(); void ClearModel();
// clear one table
void ClearOneTable(const uint64_t table_id);
// shrink sparse table // shrink sparse table
void ShrinkSparseTable(int table_id); void ShrinkSparseTable(int table_id);
// shrink dense table // shrink dense table
......
...@@ -59,6 +59,7 @@ void BindFleetWrapper(py::module* m) { ...@@ -59,6 +59,7 @@ void BindFleetWrapper(py::module* m) {
.def("save_cache", &framework::FleetWrapper::SaveCache) .def("save_cache", &framework::FleetWrapper::SaveCache)
.def("load_model", &framework::FleetWrapper::LoadModel) .def("load_model", &framework::FleetWrapper::LoadModel)
.def("clear_model", &framework::FleetWrapper::ClearModel) .def("clear_model", &framework::FleetWrapper::ClearModel)
.def("clear_one_table", &framework::FleetWrapper::ClearOneTable)
.def("stop_server", &framework::FleetWrapper::StopServer) .def("stop_server", &framework::FleetWrapper::StopServer)
.def("finalize_worker", &framework::FleetWrapper::FinalizeWorker) .def("finalize_worker", &framework::FleetWrapper::FinalizeWorker)
.def("gather_servers", &framework::FleetWrapper::GatherServers) .def("gather_servers", &framework::FleetWrapper::GatherServers)
......
...@@ -402,6 +402,23 @@ class PSLib(Fleet): ...@@ -402,6 +402,23 @@ class PSLib(Fleet):
var_list, decay, emb_dim) var_list, decay, emb_dim)
self._role_maker._barrier_worker() self._role_maker._barrier_worker()
def clear_one_table(self, table_id):
"""
clear_one_table() will be called by user. It will clear one table.
Args:
table_id(int): table id
Examples:
.. code-block:: python
fleet.clear_one_table(0)
"""
self._role_maker._barrier_worker()
if self._role_maker.is_first_worker():
self._fleet_ptr.clear_one_table(table_id)
self._role_maker._barrier_worker()
def clear_model(self): def clear_model(self):
""" """
clear_model() will be called by user. It will clear sparse model. clear_model() will be called by user. It will clear sparse model.
......
...@@ -99,6 +99,7 @@ class TestCloudRoleMaker(unittest.TestCase): ...@@ -99,6 +99,7 @@ class TestCloudRoleMaker(unittest.TestCase):
except: except:
print("do not support pslib test, skip") print("do not support pslib test, skip")
return return
fleet.clear_one_table(0)
from paddle.fluid.incubate.fleet.base.role_maker import \ from paddle.fluid.incubate.fleet.base.role_maker import \
MPISymetricRoleMaker MPISymetricRoleMaker
try: try:
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册