提交 c839ec6c 编写于 作者: Y Yi Wang

Update rcordio

上级 9f3ac225
...@@ -54,5 +54,7 @@ add_library(snappystream STATIC IMPORTED GLOBAL) ...@@ -54,5 +54,7 @@ add_library(snappystream STATIC IMPORTED GLOBAL)
set_property(TARGET snappystream PROPERTY IMPORTED_LOCATION set_property(TARGET snappystream PROPERTY IMPORTED_LOCATION
"${SNAPPYSTREAM_INSTALL_DIR}/lib/libsnappystream.a") "${SNAPPYSTREAM_INSTALL_DIR}/lib/libsnappystream.a")
include_directories(${SNAPPYSTREAM_INCLUDE_DIR}) include_directories(${SNAPPYSTREAM_INCLUDE_DIR}) # For snappysteam to include its own headers.
include_directories(${THIRD_PARTY_PATH}/install) # For Paddle to include snappy stream headers.
add_dependencies(snappystream extern_snappystream) add_dependencies(snappystream extern_snappystream)
...@@ -25,7 +25,8 @@ ELSE(WIN32) ...@@ -25,7 +25,8 @@ ELSE(WIN32)
SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/libz.a" CACHE FILEPATH "zlib library." FORCE) SET(ZLIB_LIBRARIES "${ZLIB_INSTALL_DIR}/lib/libz.a" CACHE FILEPATH "zlib library." FORCE)
ENDIF(WIN32) ENDIF(WIN32)
INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR}) INCLUDE_DIRECTORIES(${ZLIB_INCLUDE_DIR}) # For zlib code to include its own headers.
INCLUDE_DIRECTORIES(${THIRD_PARTY_PATH}/install) # For Paddle code to include zlib.h.
ExternalProject_Add( ExternalProject_Add(
extern_zlib extern_zlib
......
...@@ -14,11 +14,13 @@ ...@@ -14,11 +14,13 @@
#include "paddle/fluid/recordio/chunk.h" #include "paddle/fluid/recordio/chunk.h"
#include <algorithm>
#include <memory> #include <memory>
#include <sstream> #include <sstream>
#include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/enforce.h"
#include "snappystream.hpp" #include "snappy_stream/include/snappystream.hpp"
#include "zlib.h" #include "zlib/include/zlib.h"
namespace paddle { namespace paddle {
namespace recordio { namespace recordio {
......
...@@ -18,29 +18,27 @@ ...@@ -18,29 +18,27 @@
#include "gtest/gtest.h" #include "gtest/gtest.h"
using namespace paddle::recordio;
TEST(Chunk, SaveLoad) { TEST(Chunk, SaveLoad) {
Chunk ch; paddle::recordio::Chunk ch;
ch.Add(std::string("12345", 6)); ch.Add(std::string("12345", 6));
ch.Add(std::string("123", 4)); ch.Add(std::string("123", 4));
std::stringstream ss; std::stringstream ss;
ch.Write(ss, Compressor::kNoCompress); ch.Write(ss, paddle::recordio::Compressor::kNoCompress);
ss.seekg(0); ss.seekg(0);
ch.Parse(ss); ch.Parse(ss);
ASSERT_EQ(ch.NumBytes(), 10U); ASSERT_EQ(ch.NumBytes(), 10U);
} }
TEST(Chunk, Compressor) { TEST(Chunk, Compressor) {
Chunk ch; paddle::recordio::Chunk ch;
ch.Add(std::string("12345", 6)); ch.Add(std::string("12345", 6));
ch.Add(std::string("123", 4)); ch.Add(std::string("123", 4));
ch.Add(std::string("123", 4)); ch.Add(std::string("123", 4));
ch.Add(std::string("123", 4)); ch.Add(std::string("123", 4));
std::stringstream ss; std::stringstream ss;
ch.Write(ss, Compressor::kSnappy); ch.Write(ss, paddle::recordio::Compressor::kSnappy);
std::stringstream ss2; std::stringstream ss2;
ch.Write(ss2, Compressor::kNoCompress); ch.Write(ss2, paddle::recordio::Compressor::kNoCompress);
ASSERT_LE(ss.tellp(), ss2.tellp()); // Compress should contain less data; ASSERT_LE(ss.tellp(), ss2.tellp()); // Compress should contain less data;
ch.Clear(); ch.Clear();
......
...@@ -18,14 +18,12 @@ ...@@ -18,14 +18,12 @@
#include "gtest/gtest.h" #include "gtest/gtest.h"
using namespace paddle::recordio;
TEST(Recordio, ChunkHead) { TEST(Recordio, ChunkHead) {
Header hdr(0, 1, Compressor::kGzip, 3); paddle::recordio::Header hdr(0, 1, paddle::recordio::Compressor::kGzip, 3);
std::stringstream ss; std::stringstream ss;
hdr.Write(ss); hdr.Write(ss);
ss.seekg(0, std::ios::beg); ss.seekg(0, std::ios::beg);
Header hdr2; paddle::recordio::Header hdr2;
hdr2.Parse(ss); hdr2.Parse(ss);
EXPECT_TRUE(hdr == hdr2); EXPECT_TRUE(hdr == hdr2);
} }
...@@ -13,10 +13,14 @@ ...@@ -13,10 +13,14 @@
// limitations under the License. // limitations under the License.
#include "paddle/fluid/recordio/scanner.h" #include "paddle/fluid/recordio/scanner.h"
#include <string>
#include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/enforce.h"
namespace paddle { namespace paddle {
namespace recordio { namespace recordio {
Scanner::Scanner(std::unique_ptr<std::istream> &&stream) Scanner::Scanner(std::unique_ptr<std::istream> &&stream)
: stream_(std::move(stream)) { : stream_(std::move(stream)) {
Reset(); Reset();
......
...@@ -16,7 +16,10 @@ ...@@ -16,7 +16,10 @@
#include <fstream> #include <fstream>
#include <memory> #include <memory>
#include <string>
#include "paddle/fluid/recordio/chunk.h" #include "paddle/fluid/recordio/chunk.h"
namespace paddle { namespace paddle {
namespace recordio { namespace recordio {
......
...@@ -12,9 +12,14 @@ ...@@ -12,9 +12,14 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "paddle/fluid/recordio/writer.h" #include "paddle/fluid/recordio/writer.h"
#include <string>
#include "paddle/fluid/platform/enforce.h" #include "paddle/fluid/platform/enforce.h"
namespace paddle { namespace paddle {
namespace recordio { namespace recordio {
void Writer::Write(const std::string& record) { void Writer::Write(const std::string& record) {
cur_chunk_.Add(record); cur_chunk_.Add(record);
if (cur_chunk_.NumRecords() >= max_num_records_in_chunk_) { if (cur_chunk_.NumRecords() >= max_num_records_in_chunk_) {
......
...@@ -11,8 +11,10 @@ ...@@ -11,8 +11,10 @@
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#pragma once #pragma once
#include <string>
#include "paddle/fluid/recordio/chunk.h" #include "paddle/fluid/recordio/chunk.h"
namespace paddle { namespace paddle {
namespace recordio { namespace recordio {
......
...@@ -12,9 +12,10 @@ ...@@ -12,9 +12,10 @@
// See the License for the specific language governing permissions and // See the License for the specific language governing permissions and
// limitations under the License. // limitations under the License.
#include "gtest/gtest.h"
#include <sstream> #include <sstream>
#include <string>
#include "gtest/gtest.h"
#include "paddle/fluid/recordio/scanner.h" #include "paddle/fluid/recordio/scanner.h"
#include "paddle/fluid/recordio/writer.h" #include "paddle/fluid/recordio/writer.h"
...@@ -66,4 +67,4 @@ TEST(WriterScanner, TinyChunk) { ...@@ -66,4 +67,4 @@ TEST(WriterScanner, TinyChunk) {
ASSERT_EQ(scanner.Next(), "DEFG"); ASSERT_EQ(scanner.Next(), "DEFG");
ASSERT_FALSE(scanner.HasNext()); ASSERT_FALSE(scanner.HasNext());
} }
} }
\ No newline at end of file
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册