diff --git a/paddle/fluid/framework/dist_multi_trainer.cc b/paddle/fluid/framework/dist_multi_trainer.cc
index 636e0a735493a416595a948cb36069993c3a9440..481e12fcd63e77b6d42143f93df69c0f6abe7f25 100644
--- a/paddle/fluid/framework/dist_multi_trainer.cc
+++ b/paddle/fluid/framework/dist_multi_trainer.cc
@@ -73,6 +73,7 @@ void DistMultiTrainer::Finalize() {
   }
   pull_dense_worker_->Stop();
   dataset_ptr_->DestroyReaders();
+  root_scope_->DropKids();
 }
 
 }  // end namespace framework
diff --git a/paddle/fluid/framework/executor.cc b/paddle/fluid/framework/executor.cc
index 501480876b216b36cfe4b6f0e99a7acd7b555193..239a3ce0a84e9d0f4b3395bdbbd3fdae58e8b36a 100644
--- a/paddle/fluid/framework/executor.cc
+++ b/paddle/fluid/framework/executor.cc
@@ -143,8 +143,6 @@ void Executor::RunFromDataset(const ProgramDesc& main_program, Scope* scope,
   trainer->Run();
   VLOG(3) << "Trainer going to finalize";
   trainer->Finalize();
-  VLOG(3) << "Drop current scope kids";
-  scope->DropKids();
   return;
 }
 
diff --git a/paddle/fluid/framework/multi_trainer.cc b/paddle/fluid/framework/multi_trainer.cc
index 409c2f435f84914d2edfed65c421fcd3c57e62e5..3a266e4bda91d5962ce09b241cc5e5671d67a142 100644
--- a/paddle/fluid/framework/multi_trainer.cc
+++ b/paddle/fluid/framework/multi_trainer.cc
@@ -76,6 +76,7 @@ void MultiTrainer::Finalize() {
     th.join();
   }
   dataset_ptr_->DestroyReaders();
+  root_scope_->DropKids();
 }
 
 }  // end namespace framework