From 94dfb7d770ece68cd2c07689bcb3f9fbd6864e04 Mon Sep 17 00:00:00 2001
From: ShenLiang <2282912238@qq.com>
Date: Sun, 26 Apr 2020 15:21:21 +0800
Subject: [PATCH] opt the postprocess, test=develop (#24155)

---
 paddle/fluid/framework/data_set.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/paddle/fluid/framework/data_set.cc b/paddle/fluid/framework/data_set.cc
index b0948c8df53..48cb4131d58 100644
--- a/paddle/fluid/framework/data_set.cc
+++ b/paddle/fluid/framework/data_set.cc
@@ -779,6 +779,9 @@ template class DatasetImpl<Record>;
 void MultiSlotDataset::PostprocessInstance() {
   // divide pv instance, and merge to input_channel_
   if (enable_pv_merge_) {
+    auto fleet_ptr = FleetWrapper::GetInstance();
+    std::shuffle(input_records_.begin(), input_records_.end(),
+                 fleet_ptr->LocalRandomEngine());
     input_channel_->Open();
     input_channel_->Write(std::move(input_records_));
     for (size_t i = 0; i < multi_pv_consume_.size(); ++i) {
@@ -799,8 +802,8 @@ void MultiSlotDataset::PostprocessInstance() {
       multi_consume_channel_[i]->Clear();
     }
     input_channel_->Close();
+    this->LocalShuffle();
   }
-  this->LocalShuffle();
 }
 
 void MultiSlotDataset::SetCurrentPhase(int current_phase) {
-- 
GitLab