From e67783375d31f7bcd1f5ce2af12dc56cafdb5783 Mon Sep 17 00:00:00 2001 From: Qiao Longfei Date: Sun, 21 Oct 2018 21:06:18 +0800 Subject: [PATCH] code clean --- paddle/fluid/operators/reader/ctr_reader.cc | 28 +----------------- paddle/fluid/operators/reader/ctr_reader.h | 9 ------ .../fluid/operators/reader/ctr_reader_test.cc | 29 ++++++++++--------- 3 files changed, 16 insertions(+), 50 deletions(-) diff --git a/paddle/fluid/operators/reader/ctr_reader.cc b/paddle/fluid/operators/reader/ctr_reader.cc index 47f2c56c6..0002e80a3 100644 --- a/paddle/fluid/operators/reader/ctr_reader.cc +++ b/paddle/fluid/operators/reader/ctr_reader.cc @@ -73,18 +73,6 @@ static inline void parse_line( } } -static void print_map( - std::unordered_map>* map) { - for (auto it = map->begin(); it != map->end(); ++it) { - std::cout << it->first << " -> "; - std::cout << "["; - for (auto& i : it->second) { - std::cout << i << " "; - } - std::cout << "]\n"; - } -} - class Reader { public: virtual ~Reader() {} @@ -162,10 +150,6 @@ void ReadThread(const std::vector& file_list, VLOG(3) << "reader inited"; - uint64_t t0 = GetTimeInSec(); - - int i = 0; - while (reader.HasNext()) { batch_data.clear(); batch_data.reserve(batch_size); @@ -186,7 +170,6 @@ void ReadThread(const std::vector& file_list, break; } } - // print_map(&batch_data[0]); std::vector lod_datas; @@ -224,19 +207,10 @@ void ReadThread(const std::vector& file_list, queue->Push(lod_datas); VLOG(4) << "push one data, queue_size=" << queue->Size(); - - if (i != 0 && i % 100 == 0) { - uint64_t t1 = GetTimeInSec(); - float line_per_s = 100 * batch_size * 1000000 / (t1 - t0); - VLOG(3) << "[" << thread_id << "]" - << " line_per_second = " << line_per_s; - t0 = t1; - } - i++; } (*thread_status)[thread_id] = Stopped; - VLOG(3) << "thread " << thread_id << " exited"; + VLOG(3) << "set status to stopped, thread " << thread_id << " exited"; } } // namespace reader diff --git a/paddle/fluid/operators/reader/ctr_reader.h b/paddle/fluid/operators/reader/ctr_reader.h index d87f81402..244a5e2e7 100644 --- a/paddle/fluid/operators/reader/ctr_reader.h +++ b/paddle/fluid/operators/reader/ctr_reader.h @@ -39,15 +39,6 @@ void ReadThread(const std::vector& file_list, int thread_id, std::vector* thread_status, std::shared_ptr queue); -inline uint64_t GetTimeInSec() { - using clock = std::conditional::type; - return std::chrono::duration_cast( - clock::now().time_since_epoch()) - .count(); -} - class CTRReader : public framework::FileReader { public: explicit CTRReader(const std::shared_ptr& queue, diff --git a/paddle/fluid/operators/reader/ctr_reader_test.cc b/paddle/fluid/operators/reader/ctr_reader_test.cc index a73d54385..0b8a053a8 100644 --- a/paddle/fluid/operators/reader/ctr_reader_test.cc +++ b/paddle/fluid/operators/reader/ctr_reader_test.cc @@ -33,6 +33,7 @@ using paddle::operators::reader::LoDTensorBlockingQueueHolder; using paddle::operators::reader::CTRReader; using paddle::framework::LoDTensor; using paddle::framework::LoD; +using paddle::framework::DDim; using paddle::platform::CPUPlace; static void generatedata(const std::vector& data, @@ -73,17 +74,17 @@ TEST(CTR_READER, read_data) { std::vector label_value = {0, 0, 1, 0, 1, 1, 0, 1, 1, 1}; std::vector>> data_slot_6002{ - {{{0, 1, 2}}, {0, 0}}, - {{{0, 5, 6}}, {10, 11, 12, 13, 14, 0}}, - {{{0, 1, 2}}, {0, 0}}, - {{{0, 1, 2}}, {30, 0}}, - {{{0, 1, 2}}, {40, 0}}}; + {{{0, 1, 2, 7}}, {0, 0, 10, 11, 12, 13, 14}}, + {{{0, 1, 2, 3}}, {0, 0, 0}}, + {{{0, 1, 2, 3}}, {30, 0, 40}}, + {{{0, 1}}, {0}}}; std::vector>> data_slot_6003{ - {{{0, 1, 4}}, {1, 5, 6, 7}}, - {{{0, 1, 5}}, {0, 15, 16, 17, 18}}, - {{{0, 1, 2}}, {0, 0}}, - {{{0, 1, 3}}, {31, 35, 36}}, - {{{0, 1, 4}}, {41, 47, 48, 49}}}; + {{{0, 1, 4, 5}}, {1, 5, 6, 7, 0}}, + {{{0, 4, 5, 6}}, {15, 16, 17, 18, 0, 0}}, + {{{0, 1, 3, 4}}, {31, 35, 36, 41}}, + {{{0, 3}}, {47, 48, 49}}}; + + std::vector label_dims = {{1, 3}, {1, 3}, {1, 3}, {1, 1}}; LoDTensorBlockingQueueHolder queue_holder; int capacity = 64; @@ -91,7 +92,7 @@ TEST(CTR_READER, read_data) { std::shared_ptr queue = queue_holder.GetQueue(); - int batch_size = 2; + int batch_size = 3; int thread_num = 1; std::vector slots = {"6002", "6003"}; std::vector file_list; @@ -103,15 +104,15 @@ TEST(CTR_READER, read_data) { reader.Start(); - size_t batch_num = std::ceil(ctr_data.size() / batch_size) * thread_num; + size_t batch_num = + std::ceil(static_cast(ctr_data.size()) / batch_size) * thread_num; for (size_t i = 0; i < batch_num; ++i) { std::vector out; reader.ReadNext(&out); ASSERT_EQ(out.size(), slots.size() + 1); auto& label_tensor = out.back(); - ASSERT_EQ(label_tensor.dims(), - paddle::framework::make_ddim({1, batch_size})); + ASSERT_EQ(label_tensor.dims(), label_dims[i]); for (size_t j = 0; j < batch_size && i * batch_num + j < ctr_data.size(); ++j) { auto& label = label_tensor.data()[j]; -- GitLab