未验证 提交 94dfb7d7 编写于 作者: S ShenLiang 提交者: GitHub

opt the postprocess, test=develop (#24155)

上级 896bda0c
...@@ -779,6 +779,9 @@ template class DatasetImpl<Record>; ...@@ -779,6 +779,9 @@ template class DatasetImpl<Record>;
void MultiSlotDataset::PostprocessInstance() { void MultiSlotDataset::PostprocessInstance() {
// divide pv instance, and merge to input_channel_ // divide pv instance, and merge to input_channel_
if (enable_pv_merge_) { if (enable_pv_merge_) {
auto fleet_ptr = FleetWrapper::GetInstance();
std::shuffle(input_records_.begin(), input_records_.end(),
fleet_ptr->LocalRandomEngine());
input_channel_->Open(); input_channel_->Open();
input_channel_->Write(std::move(input_records_)); input_channel_->Write(std::move(input_records_));
for (size_t i = 0; i < multi_pv_consume_.size(); ++i) { for (size_t i = 0; i < multi_pv_consume_.size(); ++i) {
...@@ -799,8 +802,8 @@ void MultiSlotDataset::PostprocessInstance() { ...@@ -799,8 +802,8 @@ void MultiSlotDataset::PostprocessInstance() {
multi_consume_channel_[i]->Clear(); multi_consume_channel_[i]->Clear();
} }
input_channel_->Close(); input_channel_->Close();
this->LocalShuffle();
} }
this->LocalShuffle();
} }
void MultiSlotDataset::SetCurrentPhase(int current_phase) { void MultiSlotDataset::SetCurrentPhase(int current_phase) {
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册