From a0d14b18a82d7877b5172ef41ebbb801cbeae7f4 Mon Sep 17 00:00:00 2001 From: sneaxiy Date: Mon, 24 Feb 2020 07:57:20 +0000 Subject: [PATCH] Turn on keep_order=True for test, test=develop --- paddle/fluid/operators/reader/lod_tensor_blocking_queue.h | 7 +++---- python/paddle/fluid/reader.py | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/operators/reader/lod_tensor_blocking_queue.h b/paddle/fluid/operators/reader/lod_tensor_blocking_queue.h index d97d915ef87..17778148f82 100644 --- a/paddle/fluid/operators/reader/lod_tensor_blocking_queue.h +++ b/paddle/fluid/operators/reader/lod_tensor_blocking_queue.h @@ -135,7 +135,6 @@ class OrderedMultiDeviceLoDTensorBlockingQueue { for (auto& item : queues_) { item->Close(); } - data_index_ = 0; } inline void Kill() { @@ -157,6 +156,7 @@ class OrderedMultiDeviceLoDTensorBlockingQueue { auto cap = (capacity_ + dev_cnt - 1) / dev_cnt; item.reset(new LoDTensorBlockingQueue(cap, speed_test_mode_)); } + data_index_ = 0; } inline void SetResetMethod(size_t idx, @@ -171,8 +171,7 @@ class OrderedMultiDeviceLoDTensorBlockingQueue { private: const std::shared_ptr& CurQueue() { - EnforceIsInited(); - return queues_[data_index_.fetch_add(1) % queues_.size()]; + return queues_[(data_index_++) % queues_.size()]; } private: @@ -183,7 +182,7 @@ class OrderedMultiDeviceLoDTensorBlockingQueue { private: std::vector> queues_; - mutable std::atomic data_index_{0}; + mutable uint64_t data_index_{0}; size_t dev_cnt_{0}; const size_t capacity_; diff --git a/python/paddle/fluid/reader.py b/python/paddle/fluid/reader.py index ed6ac17e1b1..67e6259e812 100644 --- a/python/paddle/fluid/reader.py +++ b/python/paddle/fluid/reader.py @@ -89,7 +89,7 @@ class DataLoader(object): return_list=False, use_multiprocess=False, drop_last=True, - keep_order=False): + keep_order=True): """ Create a DataLoader object for loading data from Python generator. Data would be prefetched using Python thread and be pushed @@ -633,7 +633,7 @@ class GeneratorLoader(DataLoaderBase): iterable=True, return_list=False, drop_last=True, - keep_order=False): + keep_order=True): self._tensor_reader = None self._places = None self._thread = None -- GitLab