diff --git a/paddle/fluid/framework/data_set.cc b/paddle/fluid/framework/data_set.cc
index e9bf392d69868e42d6438253643d5c3db1d81900..c2e8bff348ade126b48cc8f54ba6c355360e665c 100644
--- a/paddle/fluid/framework/data_set.cc
+++ b/paddle/fluid/framework/data_set.cc
@@ -221,9 +221,9 @@ void DatasetImpl<T>::DestroyReaders() {
   }
   std::vector<std::shared_ptr<paddle::framework::DataFeed>>().swap(readers_);
   VLOG(3) << "readers size: " << readers_.size();
-  // if memory_data_ is not empty, which means it's not InMemory mode,
+  // if memory_data_ is empty, which means it's not InMemory mode,
   // so the next epoch should read all data again
-  if (memory_data_.size() != 0) {
+  if (memory_data_.size() == 0) {
     file_idx_ = 0;
   }
 }