diff --git a/paddle/fluid/framework/data_set.cc b/paddle/fluid/framework/data_set.cc
index 0684d5674ad939d9cf806f38da3ed04a7023ec02..b0948c8df53d6b490549734fe1d9bbe837614465 100644
--- a/paddle/fluid/framework/data_set.cc
+++ b/paddle/fluid/framework/data_set.cc
@@ -326,6 +326,8 @@ void DatasetImpl<T>::ReleaseMemory() {
   std::vector<paddle::framework::Channel<PvInstance>>().swap(multi_pv_consume_);
 
   std::vector<std::shared_ptr<paddle::framework::DataFeed>>().swap(readers_);
+  input_records_.clear();
+  std::vector<T>().swap(input_records_);
   VLOG(3) << "DatasetImpl<T>::ReleaseMemory() end";
 }
 
diff --git a/paddle/fluid/framework/data_set.h b/paddle/fluid/framework/data_set.h
index 7adef69a44812d8e06d65892865b86b2c711e98c..bb6210b1f014d62f8174c275b32e1d3ae73fc506 100644
--- a/paddle/fluid/framework/data_set.h
+++ b/paddle/fluid/framework/data_set.h
@@ -272,6 +272,7 @@ class DatasetImpl : public Dataset {
   std::mutex global_index_mutex_;
   int64_t global_index_ = 0;
   std::vector<std::shared_ptr<ThreadPool>> consume_task_pool_;
+  std::vector<T> input_records_;  // only for paddleboxdatafeed
 };
 
 // use std::vector<MultiSlotType> or Record as data type
@@ -296,9 +297,6 @@ class MultiSlotDataset : public DatasetImpl<Record> {
   virtual void GetRandomData(const std::set<uint16_t>& slots_to_replace,
                              std::vector<Record>* result);
   virtual ~MultiSlotDataset() {}
-
- protected:
-  std::vector<Record> input_records_;  // the real data
 };
 
 }  // end namespace framework