diff --git a/paddle/fluid/operators/reader/lod_tensor_blocking_queue.h b/paddle/fluid/operators/reader/lod_tensor_blocking_queue.h
index d97d915ef876f842fc55b54750c3aec7a191753a..17778148f82dd70ec0ed6d7b1dccfc43e1550972 100644
--- a/paddle/fluid/operators/reader/lod_tensor_blocking_queue.h
+++ b/paddle/fluid/operators/reader/lod_tensor_blocking_queue.h
@@ -135,7 +135,6 @@ class OrderedMultiDeviceLoDTensorBlockingQueue {
     for (auto& item : queues_) {
       item->Close();
     }
-    data_index_ = 0;
   }
 
   inline void Kill() {
@@ -157,6 +156,7 @@ class OrderedMultiDeviceLoDTensorBlockingQueue {
       auto cap = (capacity_ + dev_cnt - 1) / dev_cnt;
       item.reset(new LoDTensorBlockingQueue(cap, speed_test_mode_));
     }
+    data_index_ = 0;
   }
 
   inline void SetResetMethod(size_t idx,
@@ -171,8 +171,7 @@ class OrderedMultiDeviceLoDTensorBlockingQueue {
 
  private:
   const std::shared_ptr<LoDTensorBlockingQueue>& CurQueue() {
-    EnforceIsInited();
-    return queues_[data_index_.fetch_add(1) % queues_.size()];
+    return queues_[(data_index_++) % queues_.size()];
   }
 
  private:
@@ -183,7 +182,7 @@ class OrderedMultiDeviceLoDTensorBlockingQueue {
 
  private:
   std::vector<std::shared_ptr<LoDTensorBlockingQueue>> queues_;
-  mutable std::atomic<uint64_t> data_index_{0};
+  mutable uint64_t data_index_{0};
 
   size_t dev_cnt_{0};
   const size_t capacity_;
diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py
index ed6ac17e1b116b22048cb87167385e3d86b97d95..67e6259e8124f8898ef6c53a880ec3ea2505be31 100644
--- a/python/paddle/fluid/reader.py
+++ b/python/paddle/fluid/reader.py
@@ -89,7 +89,7 @@ class DataLoader(object):
                        return_list=False,
                        use_multiprocess=False,
                        drop_last=True,
-                       keep_order=False):
+                       keep_order=True):
         """
         Create a DataLoader object for loading data from Python generator. 
         Data would be prefetched using Python thread and be pushed
@@ -633,7 +633,7 @@ class GeneratorLoader(DataLoaderBase):
                  iterable=True,
                  return_list=False,
                  drop_last=True,
-                 keep_order=False):
+                 keep_order=True):
         self._tensor_reader = None
         self._places = None
         self._thread = None