未验证 提交 d51ffe86 编写于 作者: X xujiaqi01 提交者: GitHub

fix copy table bug (#22432)

* fix copy table bug of lost some feasign
* test=develop
上级 822e5b36
...@@ -577,13 +577,6 @@ void DownpourWorker::TrainFilesWithProfiler() { ...@@ -577,13 +577,6 @@ void DownpourWorker::TrainFilesWithProfiler() {
timeline.Start(); timeline.Start();
if (copy_table_config_.need_copy()) { if (copy_table_config_.need_copy()) {
VLOG(3) << "copy_sparse_tables_.size " << copy_sparse_tables_.size(); VLOG(3) << "copy_sparse_tables_.size " << copy_sparse_tables_.size();
if (copy_table_config_.sparse_copy_by_feasign()) {
for (size_t i = 0; i < copy_sparse_tables_.size(); ++i) {
uint64_t tid = copy_sparse_tables_[i].first;
feasign_set_[tid].insert(sparse_push_keys_[tid].begin(),
sparse_push_keys_[tid].end());
}
}
if (batch_cnt % copy_table_config_.batch_num() == 0) { if (batch_cnt % copy_table_config_.batch_num() == 0) {
CopySparseTable(); CopySparseTable();
CopyDenseTable(); CopyDenseTable();
...@@ -696,6 +689,18 @@ void DownpourWorker::TrainFilesWithProfiler() { ...@@ -696,6 +689,18 @@ void DownpourWorker::TrainFilesWithProfiler() {
} }
} }
#ifdef PADDLE_WITH_PSLIB
if (copy_table_config_.need_copy()) {
if (copy_table_config_.sparse_copy_by_feasign()) {
for (size_t i = 0; i < copy_sparse_tables_.size(); ++i) {
uint64_t tid = copy_sparse_tables_[i].first;
feasign_set_[tid].insert(sparse_push_keys_[tid].begin(),
sparse_push_keys_[tid].end());
}
}
}
#endif
if (need_to_push_dense_) { if (need_to_push_dense_) {
timeline.Start(); timeline.Start();
for (int i = 0; i < param_.program_config(0).push_dense_table_id_size(); for (int i = 0; i < param_.program_config(0).push_dense_table_id_size();
...@@ -828,13 +833,6 @@ void DownpourWorker::TrainFiles() { ...@@ -828,13 +833,6 @@ void DownpourWorker::TrainFiles() {
int cur_batch; int cur_batch;
while ((cur_batch = device_reader_->Next()) > 0) { while ((cur_batch = device_reader_->Next()) > 0) {
if (copy_table_config_.need_copy()) { if (copy_table_config_.need_copy()) {
if (copy_table_config_.sparse_copy_by_feasign()) {
for (size_t i = 0; i < copy_sparse_tables_.size(); ++i) {
uint64_t tid = copy_sparse_tables_[i].first;
feasign_set_[tid].insert(sparse_push_keys_[tid].begin(),
sparse_push_keys_[tid].end());
}
}
if (batch_cnt % copy_table_config_.batch_num() == 0) { if (batch_cnt % copy_table_config_.batch_num() == 0) {
CopySparseTable(); CopySparseTable();
CopyDenseTable(); CopyDenseTable();
...@@ -918,6 +916,18 @@ void DownpourWorker::TrainFiles() { ...@@ -918,6 +916,18 @@ void DownpourWorker::TrainFiles() {
} }
} }
#ifdef PADDLE_WITH_PSLIB
if (copy_table_config_.need_copy()) {
if (copy_table_config_.sparse_copy_by_feasign()) {
for (size_t i = 0; i < copy_sparse_tables_.size(); ++i) {
uint64_t tid = copy_sparse_tables_[i].first;
feasign_set_[tid].insert(sparse_push_keys_[tid].begin(),
sparse_push_keys_[tid].end());
}
}
}
#endif
if (need_to_push_dense_) { if (need_to_push_dense_) {
for (int i = 0; i < param_.program_config(0).push_dense_table_id_size(); for (int i = 0; i < param_.program_config(0).push_dense_table_id_size();
++i) { ++i) {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册