From c839ec6c470d434578ddecae02caae974ff97228 Mon Sep 17 00:00:00 2001 From: Yi Wang Date: Fri, 6 Apr 2018 13:19:49 -0700 Subject: [PATCH] Update rcordio --- cmake/external/snappystream.cmake | 4 +++- cmake/external/zlib.cmake | 3 ++- paddle/fluid/recordio/chunk.cc | 6 ++++-- paddle/fluid/recordio/chunk_test.cc | 12 +++++------- paddle/fluid/recordio/header_test.cc | 6 ++---- paddle/fluid/recordio/scanner.cc | 4 ++++ paddle/fluid/recordio/scanner.h | 3 +++ paddle/fluid/recordio/writer.cc | 5 +++++ paddle/fluid/recordio/writer.h | 4 +++- paddle/fluid/recordio/writer_scanner_test.cc | 7 ++++--- 10 files changed, 35 insertions(+), 19 deletions(-) diff --git a/cmake/external/snappystream.cmake b/cmake/external/snappystream.cmake index 5377a0b046a..8f7a3bf8eea 100644 --- a/cmake/external/snappystream.cmake +++ b/cmake/external/snappystream.cmake @@ -54,5 +54,7 @@ add_library(snappystream STATIC IMPORTED GLOBAL) set_property(TARGET snappystream PROPERTY IMPORTED_LOCATION "${SNAPPYSTREAM_INSTALL_DIR}/lib/libsnappystream.a") -include_directories(${SNAPPYSTREAM_INCLUDE_DIR}) +include_directories(${SNAPPYSTREAM_INCLUDE_DIR}) # For snappysteam to include its own headers. +include_directories(${THIRD_PARTY_PATH}/install) # For Paddle to include snappy stream headers. + add_dependencies(snappystream extern_snappystream) diff --git a/cmake/external/zlib.cmake b/cmake/external/zlib.cmake index 20b8506e678..c3d73235453 100644 --- a/cmake/external/zlib.cmake +++ b/cmake/external/zlib.cmake @@ -25,7 +25,8 @@ ELSE(WIN32) SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/libz.a" CACHE FILEPATH "zlib library." FORCE) ENDIF(WIN32) -INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR}) +INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR}) # For zlib code to include its own headers. +INCLUDE_DIRECTORIES(${THIRD_PARTY_PATH}/install) # For Paddle code to include zlib.h. ExternalProject_Add( extern_zlib diff --git a/paddle/fluid/recordio/chunk.cc b/paddle/fluid/recordio/chunk.cc index e828cbabefd..e7ebbba452c 100644 --- a/paddle/fluid/recordio/chunk.cc +++ b/paddle/fluid/recordio/chunk.cc @@ -14,11 +14,13 @@ #include "paddle/fluid/recordio/chunk.h" +#include #include #include + #include "paddle/fluid/platform/enforce.h" -#include "snappystream.hpp" -#include "zlib.h" +#include "snappy_stream/include/snappystream.hpp" +#include "zlib/include/zlib.h" namespace paddle { namespace recordio { diff --git a/paddle/fluid/recordio/chunk_test.cc b/paddle/fluid/recordio/chunk_test.cc index 1f0e36a14d3..98ca99b9a01 100644 --- a/paddle/fluid/recordio/chunk_test.cc +++ b/paddle/fluid/recordio/chunk_test.cc @@ -18,29 +18,27 @@ #include "gtest/gtest.h" -using namespace paddle::recordio; - TEST(Chunk, SaveLoad) { - Chunk ch; + paddle::recordio::Chunk ch; ch.Add(std::string("12345", 6)); ch.Add(std::string("123", 4)); std::stringstream ss; - ch.Write(ss, Compressor::kNoCompress); + ch.Write(ss, paddle::recordio::Compressor::kNoCompress); ss.seekg(0); ch.Parse(ss); ASSERT_EQ(ch.NumBytes(), 10U); } TEST(Chunk, Compressor) { - Chunk ch; + paddle::recordio::Chunk ch; ch.Add(std::string("12345", 6)); ch.Add(std::string("123", 4)); ch.Add(std::string("123", 4)); ch.Add(std::string("123", 4)); std::stringstream ss; - ch.Write(ss, Compressor::kSnappy); + ch.Write(ss, paddle::recordio::Compressor::kSnappy); std::stringstream ss2; - ch.Write(ss2, Compressor::kNoCompress); + ch.Write(ss2, paddle::recordio::Compressor::kNoCompress); ASSERT_LE(ss.tellp(), ss2.tellp()); // Compress should contain less data; ch.Clear(); diff --git a/paddle/fluid/recordio/header_test.cc b/paddle/fluid/recordio/header_test.cc index a7d627c3eb4..00f1887dc5e 100644 --- a/paddle/fluid/recordio/header_test.cc +++ b/paddle/fluid/recordio/header_test.cc @@ -18,14 +18,12 @@ #include "gtest/gtest.h" -using namespace paddle::recordio; - TEST(Recordio, ChunkHead) { - Header hdr(0, 1, Compressor::kGzip, 3); + paddle::recordio::Header hdr(0, 1, paddle::recordio::Compressor::kGzip, 3); std::stringstream ss; hdr.Write(ss); ss.seekg(0, std::ios::beg); - Header hdr2; + paddle::recordio::Header hdr2; hdr2.Parse(ss); EXPECT_TRUE(hdr == hdr2); } diff --git a/paddle/fluid/recordio/scanner.cc b/paddle/fluid/recordio/scanner.cc index c22281dc97e..88b4d4001bc 100644 --- a/paddle/fluid/recordio/scanner.cc +++ b/paddle/fluid/recordio/scanner.cc @@ -13,10 +13,14 @@ // limitations under the License. #include "paddle/fluid/recordio/scanner.h" + +#include + #include "paddle/fluid/platform/enforce.h" namespace paddle { namespace recordio { + Scanner::Scanner(std::unique_ptr &&stream) : stream_(std::move(stream)) { Reset(); diff --git a/paddle/fluid/recordio/scanner.h b/paddle/fluid/recordio/scanner.h index 8812e2c9527..34f1b0c78d6 100644 --- a/paddle/fluid/recordio/scanner.h +++ b/paddle/fluid/recordio/scanner.h @@ -16,7 +16,10 @@ #include #include +#include + #include "paddle/fluid/recordio/chunk.h" + namespace paddle { namespace recordio { diff --git a/paddle/fluid/recordio/writer.cc b/paddle/fluid/recordio/writer.cc index 196d66edff8..8046f4ff789 100644 --- a/paddle/fluid/recordio/writer.cc +++ b/paddle/fluid/recordio/writer.cc @@ -12,9 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. #include "paddle/fluid/recordio/writer.h" + +#include + #include "paddle/fluid/platform/enforce.h" + namespace paddle { namespace recordio { + void Writer::Write(const std::string& record) { cur_chunk_.Add(record); if (cur_chunk_.NumRecords() >= max_num_records_in_chunk_) { diff --git a/paddle/fluid/recordio/writer.h b/paddle/fluid/recordio/writer.h index 87349644a7b..ac7e50ee90e 100644 --- a/paddle/fluid/recordio/writer.h +++ b/paddle/fluid/recordio/writer.h @@ -11,8 +11,10 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - #pragma once + +#include + #include "paddle/fluid/recordio/chunk.h" namespace paddle { namespace recordio { diff --git a/paddle/fluid/recordio/writer_scanner_test.cc b/paddle/fluid/recordio/writer_scanner_test.cc index 7e764f0d943..6583df21a20 100644 --- a/paddle/fluid/recordio/writer_scanner_test.cc +++ b/paddle/fluid/recordio/writer_scanner_test.cc @@ -12,9 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "gtest/gtest.h" - #include +#include + +#include "gtest/gtest.h" #include "paddle/fluid/recordio/scanner.h" #include "paddle/fluid/recordio/writer.h" @@ -66,4 +67,4 @@ TEST(WriterScanner, TinyChunk) { ASSERT_EQ(scanner.Next(), "DEFG"); ASSERT_FALSE(scanner.HasNext()); } -} \ No newline at end of file +} -- GitLab