From 7016979cd55b519e404657f0a3a4015c0208f074 Mon Sep 17 00:00:00 2001 From: dongzhihong Date: Mon, 5 Mar 2018 21:55:27 +0800 Subject: [PATCH] "add crc32 encoder" --- paddle/fluid/recordio/CMakeLists.txt | 1 + paddle/fluid/recordio/chunk.cc | 105 ++ paddle/fluid/recordio/chunk.h | 113 +- paddle/fluid/recordio/chunk_test.cc | 23 + paddle/fluid/recordio/crc32.h | 33 + paddle/fluid/recordio/detail/crc.h | 1899 ++++++++++++++++++++++++++ paddle/fluid/recordio/header.cc | 40 +- paddle/fluid/recordio/header.h | 6 +- paddle/fluid/recordio/header_test.cc | 7 +- paddle/fluid/recordio/io.cc | 4 +- paddle/fluid/recordio/io.h | 13 +- paddle/fluid/recordio/io_test.cc | 2 +- paddle/fluid/recordio/writer.cc | 8 +- paddle/fluid/recordio/writer.h | 2 - paddle/fluid/recordio/writer_test.cc | 2 +- 15 files changed, 2114 insertions(+), 144 deletions(-) create mode 100644 paddle/fluid/recordio/chunk.cc create mode 100644 paddle/fluid/recordio/chunk_test.cc create mode 100644 paddle/fluid/recordio/crc32.h create mode 100644 paddle/fluid/recordio/detail/crc.h diff --git a/paddle/fluid/recordio/CMakeLists.txt b/paddle/fluid/recordio/CMakeLists.txt index 86b4583c7b..5d55709b4b 100644 --- a/paddle/fluid/recordio/CMakeLists.txt +++ b/paddle/fluid/recordio/CMakeLists.txt @@ -2,3 +2,4 @@ cc_library(header SRCS header.cc) cc_test(header_test SRCS header_test.cc DEPS header) cc_library(io SRCS io.cc DEPS stringpiece) cc_test(io_test SRCS io_test.cc DEPS io) +cc_library(chunk SRCS chunk.cc DEPS snappy) diff --git a/paddle/fluid/recordio/chunk.cc b/paddle/fluid/recordio/chunk.cc new file mode 100644 index 0000000000..1ab2c7dd55 --- /dev/null +++ b/paddle/fluid/recordio/chunk.cc @@ -0,0 +1,105 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/recordio/chunk.h" + +#include +#include +#include + +#include "snappy.h" + +#include "paddle/fluid/recordio/crc32.h" + +namespace paddle { +namespace recordio { + +void Chunk::Add(const char* record, size_t length) { + records_.emplace_after(std::move(s)); + num_bytes_ += s.size() * sizeof(char); +} + +bool Chunk::Dump(Stream* fo, Compressor ct) { + // NOTE(dzhwinter): don't check records.numBytes instead, because + // empty records are allowed. + if (records_.size() == 0) return false; + + // pack the record into consecutive memory for compress + std::ostringstream os; + for (auto& record : records_) { + os.write(record.size(), sizeof(size_t)); + os.write(record.data(), static_cast(record.size())); + } + + std::unique_ptr buffer(new char[kDefaultMaxChunkSize]); + size_t compressed = + CompressData(os.str().c_str(), num_bytes_, ct, buffer.get()); + uint32_t checksum = Crc32(buffer.get(), compressed); + Header hdr(records_.size(), checksum, ct, static_cast(compressed)); + hdr.Write(fo); + fo.Write(buffer.get(), compressed); + return true; +} + +void Chunk::Parse(Stream* fi, size_t offset) { + fi->Seek(offset); + Header hdr; + hdr.Parse(fi); + + std::unique_ptr buffer(new char[kDefaultMaxChunkSize]); + fi->Read(buffer.get(), static_cast(hdr.CompressSize())); + uint32_t deflated_size = + DeflateData(buffer.get(), hdr.CompressSize(), hdr.CompressType()); + std::istringstream deflated(std::string(buffer.get(), deflated_size)); + for (size_t i = 0; i < hdr.NumRecords(); ++i) { + uint32_t rs; + deflated >> rs; + std::string record(rs, '\0'); + deflated.read(&record[0], rs); + records_.emplace_back(record); + num_bytes_ += record.size(); + } +} + +size_t CompressData(const char* in, + size_t in_length, + Compressor ct, + char* out) { + size_t compressd_size = 0; + switch (ct) { + case Compressor::kNoCompress: + // do nothing + memcpy(out, in, in_length); + compressd_size = in_length; + break; + case Compressor::kSnappy: + snappy::RawCompress(in, in_length, out, &compressd_size); + break; + } + return compressd_size; +} + +void DeflateData(const char* in, size_t in_length, Compressor ct, char* out) { + switch (c) { + case Compressor::kNoCompress: + memcpy(out, in, in_length); + break; + case Compressor::kSnappy: + snappy::RawUncompress(in, in_length, out); + break; + } +} + +} // namespace recordio +} // namespace paddle diff --git a/paddle/fluid/recordio/chunk.h b/paddle/fluid/recordio/chunk.h index 48626b92fe..975604df3c 100644 --- a/paddle/fluid/recordio/chunk.h +++ b/paddle/fluid/recordio/chunk.h @@ -13,109 +13,36 @@ // limitations under the License. #pragma once - -#include -#include -#include +#include #include -#include -#include -// Chunk -// a chunk contains the Header and optionally compressed records. +#include "paddle/fluid/recordio/header.h" +#include "paddle/fluid/recordio/io.h" + +namespace paddle { +namespace recordio { + +// A Chunk contains the Header and optionally compressed records. class Chunk { public: - Chunk() = default; - void Add(const char* record, size_t length); - void Add(const std::string&); - - bool Dump(std::ostream& os, Compressor ct); - void Parse(std::istream& iss, int64_t offset); - const std::string Record(int i) { return records_[i]; } + Chunk() {} + void Add(const char* record, size_t size); + // dump the chunk into w, and clears the chunk and makes it ready for + // the next add invocation. + bool Dump(Stream* fo, Compressor ct); + void Parse(Stream* fi, size_t offset); size_t NumBytes() { return num_bytes_; } private: - std::vector records_; + std::forward_list records_; // sum of record lengths in bytes. size_t num_bytes_; + DISABLE_COPY_AND_ASSIGN(Chunk); }; -size_t CompressData(const std::stringstream& ss, Compressor ct, char* buffer); - -uint32_t DeflateData(char* buffer, uint32_t size, Compressor c); - -// implementation -void Chunk::Add(const std::string& s) { - num_bytes_ += s.size() * sizeof(char); - records_.emplace_back(std::move(s)); - // records_.resize(records_.size()+1); - // records_[records_.size()-1] = s; -} - -void Chunk::Add(const char* record, size_t length) { - Add(std::string(record, length)); -} - -bool Chunk::Dump(std::ostream& os, Compressor ct) { - if (records_.size() == 0) return false; - - // TODO(dzhwinter): - // we pack the string with same size buffer, - // then compress with another buffer. - // Here can be optimized if it is the bottle-neck. - std::ostringstream oss; - for (auto& record : records_) { - unsigned len = record.size(); - oss << len; - oss << record; - // os.write(std::to_string(len).c_str(), sizeof(unsigned)); - // os.write(record.c_str(), record.size()); - } - std::unique_ptr buffer(new char[kDefaultMaxChunkSize]); - size_t compressed = CompressData(oss.str(), ct, buffer.get()); - - // TODO(dzhwinter): crc32 checksum - size_t checksum = compressed; - - Header hdr(records_.size(), checksum, ct, compressed); - - return true; -} - -void Chunk::Parse(std::istream& iss, int64_t offset) { - iss.seekg(offset, iss.beg); - Header hdr; - hdr.Parse(iss); +size_t CompressData(const char* in, size_t in_length, Compressor ct, char* out); - std::unique_ptr buffer(new char[kDefaultMaxChunkSize]); - iss.read(buffer.get(), static_cast(hdr.CompressSize())); - // TODO(dzhwinter): checksum - uint32_t deflated_size = - DeflateData(buffer.get(), hdr.CompressSize(), hdr.CompressType()); - std::istringstream deflated(std::string(buffer.get(), deflated_size)); - for (size_t i = 0; i < hdr.NumRecords(); ++i) { - uint32_t rs; - deflated >> rs; - std::string record(rs, '\0'); - deflated.read(&record[0], rs); - records_.emplace_back(record); - num_bytes_ += record.size(); - } -} +void DeflateData(const char* in, size_t in_length, Compressor ct, char* out); -uint32_t DeflateData(char* buffer, uint32_t size, Compressor c) { - uint32_t deflated_size = 0; - std::string uncompressed; - switch (c) { - case Compressor::kNoCompress: - deflated_size = size; - break; - case Compressor::kSnappy: - // snappy::Uncompress(buffer, size, &uncompressed); - // deflated_size = uncompressed.size(); - // memcpy(buffer, uncompressed.data(), uncompressed.size() * - // sizeof(char)); - break; - } - return deflated_size; -} +} // namespace recordio +} // namespace paddle diff --git a/paddle/fluid/recordio/chunk_test.cc b/paddle/fluid/recordio/chunk_test.cc new file mode 100644 index 0000000000..8aec47c234 --- /dev/null +++ b/paddle/fluid/recordio/chunk_test.cc @@ -0,0 +1,23 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/recordio/chunk.h" + +#include + +#include "gtest/gtest.h" + +using namespace paddle::recordio; + +TEST(Chunk, SaveLoad) {} diff --git a/paddle/fluid/recordio/crc32.h b/paddle/fluid/recordio/crc32.h new file mode 100644 index 0000000000..77b430356f --- /dev/null +++ b/paddle/fluid/recordio/crc32.h @@ -0,0 +1,33 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// A wrapper on crc library https://github.com/d-bahr/CRCpp +#include + +#include "paddle/fluid/recordio/detail/crc.h" + +namespace paddle { +namespace recordio { + +// usage +// char data[] = "hello,world"; +// crc = Crc32(data, 12); +// Assert_EQ(crc, 68a85159); + +uint32_t Crc32(const char* data, size_t size) { + return CRC::Calculate(data, size, CRC::CRC_32()) +} + +} // namespace recordio +} // namespace paddle diff --git a/paddle/fluid/recordio/detail/crc.h b/paddle/fluid/recordio/detail/crc.h new file mode 100644 index 0000000000..ef8390c34a --- /dev/null +++ b/paddle/fluid/recordio/detail/crc.h @@ -0,0 +1,1899 @@ +// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** + @file CRC.h + @author Daniel Bahr + @version 0.2.0.6 + @copyright + @parblock + CRC++ + Copyright (c) 2016, Daniel Bahr + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright notice, + this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, + this list of conditions and the following disclaimer in the + documentation + and/or other materials provided with the distribution. + + * Neither the name of CRC++ nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" + AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE + IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE + LIABLE + FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + CONSEQUENTIAL + DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + OR + SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER + CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + LIABILITY, + OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF + THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + @endparblock +*/ + +/* + CRC++ can be configured by setting various #defines before #including this + header file: + + #define crcpp_uint8 - Specifies the type + used to store CRCs that have a width of 8 bits or less. + This type is not used + in CRC calculations. Defaults to ::std::uint8_t. + #define crcpp_uint16 - Specifies the type + used to store CRCs that have a width between 9 and 16 bits (inclusive). + This type is not used + in CRC calculations. Defaults to ::std::uint16_t. + #define crcpp_uint32 - Specifies the type + used to store CRCs that have a width between 17 and 32 bits (inclusive). + This type is not used + in CRC calculations. Defaults to ::std::uint32_t. + #define crcpp_uint64 - Specifies the type + used to store CRCs that have a width between 33 and 64 bits (inclusive). + This type is not used + in CRC calculations. Defaults to ::std::uint64_t. + #define crcpp_size - This type is used for + loop iteration and function signatures only. Defaults to ::std::size_t. + #define CRCPP_USE_NAMESPACE - Define to place all + CRC++ code within the ::CRCPP namespace. + #define CRCPP_BRANCHLESS - Define to enable a + branchless CRC implementation. The branchless implementation uses a single + integer + multiplication in the + bit-by-bit calculation instead of a small conditional. The branchless + implementation + may be faster on + processor architectures which support single-instruction integer + multiplication. + #define CRCPP_USE_CPP11 - Define to enables + C++11 features (move semantics, constexpr, static_assert, etc.). + #define CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS - Define to include + definitions for little-used CRCs. +*/ + +#ifndef CRCPP_CRC_H_ +#define CRCPP_CRC_H_ + +#include // Includes CHAR_BIT +#ifdef CRCPP_USE_CPP11 +#include // Includes ::std::size_t +#include // Includes ::std::uint8_t, ::std::uint16_t, ::std::uint32_t, ::std::uint64_t +#else +#include // Includes size_t +#include // Includes uint8_t, uint16_t, uint32_t, uint64_t +#endif +#include // Includes ::std::numeric_limits +#include // Includes ::std::move + +#ifndef crcpp_uint8 +#ifdef CRCPP_USE_CPP11 +/// @brief Unsigned 8-bit integer definition, used primarily for parameter +/// definitions. +#define crcpp_uint8 ::std::uint8_t +#else +/// @brief Unsigned 8-bit integer definition, used primarily for parameter +/// definitions. +#define crcpp_uint8 uint8_t +#endif +#endif + +#ifndef crcpp_uint16 +#ifdef CRCPP_USE_CPP11 +/// @brief Unsigned 16-bit integer definition, used primarily for parameter +/// definitions. +#define crcpp_uint16 ::std::uint16_t +#else +/// @brief Unsigned 16-bit integer definition, used primarily for parameter +/// definitions. +#define crcpp_uint16 uint16_t +#endif +#endif + +#ifndef crcpp_uint32 +#ifdef CRCPP_USE_CPP11 +/// @brief Unsigned 32-bit integer definition, used primarily for parameter +/// definitions. +#define crcpp_uint32 ::std::uint32_t +#else +/// @brief Unsigned 32-bit integer definition, used primarily for parameter +/// definitions. +#define crcpp_uint32 uint32_t +#endif +#endif + +#ifndef crcpp_uint64 +#ifdef CRCPP_USE_CPP11 +/// @brief Unsigned 64-bit integer definition, used primarily for parameter +/// definitions. +#define crcpp_uint64 ::std::uint64_t +#else +/// @brief Unsigned 64-bit integer definition, used primarily for parameter +/// definitions. +#define crcpp_uint64 uint64_t +#endif +#endif + +#ifndef crcpp_size +#ifdef CRCPP_USE_CPP11 +/// @brief Unsigned size definition, used for specifying data sizes. +#define crcpp_size ::std::size_t +#else +/// @brief Unsigned size definition, used for specifying data sizes. +#define crcpp_size size_t +#endif +#endif + +#ifdef CRCPP_USE_CPP11 +/// @brief Compile-time expression definition. +#define crcpp_constexpr constexpr +#else +/// @brief Compile-time expression definition. +#define crcpp_constexpr const +#endif + +#ifdef CRCPP_USE_NAMESPACE +namespace CRCPP { +#endif + +/** + @brief Static class for computing CRCs. + @note This class supports computation of full and multi-part CRCs, using a + bit-by-bit algorithm or a + byte-by-byte lookup table. The CRCs are calculated using as many + optimizations as is reasonable. + If compiling with C++11, the constexpr keyword is used liberally so that + many calculations are + performed at compile-time instead of at runtime. +*/ +class CRC { +public: + // Forward declaration + template + struct Table; + + /** + @brief CRC parameters. + */ + template + struct Parameters { + CRCType polynomial; ///< CRC polynomial + CRCType initialValue; ///< Initial CRC value + CRCType finalXOR; ///< Value to XOR with the final CRC + bool reflectInput; ///< true to reflect all input bytes + bool reflectOutput; ///< true to reflect the output CRC (reflection occurs + /// before the final XOR) + + Table MakeTable() const; + }; + + /** + @brief CRC lookup table. After construction, the CRC parameters are fixed. + @note A CRC table can be used for multiple CRC calculations. + */ + template + struct Table { + // Constructors are intentionally NOT marked explicit. + Table(const Parameters ¶meters); + +#ifdef CRCPP_USE_CPP11 + Table(Parameters &¶meters); +#endif + + const Parameters &GetParameters() const; + + const CRCType *GetTable() const; + + CRCType operator[](unsigned char index) const; + + private: + void InitTable(); + + Parameters + parameters; ///< CRC parameters used to construct the table + CRCType table[1 << CHAR_BIT]; ///< CRC lookup table + }; + + // The number of bits in CRCType must be at least as large as CRCWidth. + // CRCType must be an unsigned integer type or a custom type with operator + // overloads. + template + static CRCType Calculate(const void *data, + crcpp_size size, + const Parameters ¶meters); + + template + static CRCType Calculate(const void *data, + crcpp_size size, + const Parameters ¶meters, + CRCType crc); + + template + static CRCType Calculate(const void *data, + crcpp_size size, + const Table &lookupTable); + + template + static CRCType Calculate(const void *data, + crcpp_size size, + const Table &lookupTable, + CRCType crc); + +// Common CRCs up to 64 bits. +// Note: Check values are the computed CRCs when given an ASCII input of +// "123456789" (without null terminator) +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + static const Parameters &CRC_4_ITU(); + static const Parameters &CRC_5_EPC(); + static const Parameters &CRC_5_ITU(); + static const Parameters &CRC_5_USB(); + static const Parameters &CRC_6_CDMA2000A(); + static const Parameters &CRC_6_CDMA2000B(); + static const Parameters &CRC_6_ITU(); + static const Parameters &CRC_7(); +#endif + static const Parameters &CRC_8(); +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + static const Parameters &CRC_8_EBU(); + static const Parameters &CRC_8_MAXIM(); + static const Parameters &CRC_8_WCDMA(); + static const Parameters &CRC_10(); + static const Parameters &CRC_10_CDMA2000(); + static const Parameters &CRC_11(); + static const Parameters &CRC_12_CDMA2000(); + static const Parameters &CRC_12_DECT(); + static const Parameters &CRC_12_UMTS(); + static const Parameters &CRC_13_BBC(); + static const Parameters &CRC_15(); + static const Parameters &CRC_15_MPT1327(); +#endif + static const Parameters &CRC_16_ARC(); + static const Parameters &CRC_16_BUYPASS(); + static const Parameters &CRC_16_CCITTFALSE(); +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + static const Parameters &CRC_16_CDMA2000(); + static const Parameters &CRC_16_DECTR(); + static const Parameters &CRC_16_DECTX(); + static const Parameters &CRC_16_DNP(); +#endif + static const Parameters &CRC_16_GENIBUS(); + static const Parameters &CRC_16_KERMIT(); +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + static const Parameters &CRC_16_MAXIM(); + static const Parameters &CRC_16_MODBUS(); + static const Parameters &CRC_16_T10DIF(); + static const Parameters &CRC_16_USB(); +#endif + static const Parameters &CRC_16_X25(); + static const Parameters &CRC_16_XMODEM(); +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + static const Parameters &CRC_17_CAN(); + static const Parameters &CRC_21_CAN(); + static const Parameters &CRC_24(); + static const Parameters &CRC_24_FLEXRAYA(); + static const Parameters &CRC_24_FLEXRAYB(); + static const Parameters &CRC_30(); +#endif + static const Parameters &CRC_32(); + static const Parameters &CRC_32_BZIP2(); +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + static const Parameters &CRC_32_C(); +#endif + static const Parameters &CRC_32_MPEG2(); + static const Parameters &CRC_32_POSIX(); +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + static const Parameters &CRC_32_Q(); + static const Parameters &CRC_40_GSM(); + static const Parameters &CRC_64(); +#endif + +#ifdef CRCPP_USE_CPP11 + CRC() = delete; + CRC(const CRC &other) = delete; + CRC &operator=(const CRC &other) = delete; + CRC(CRC &&other) = delete; + CRC &operator=(CRC &&other) = delete; +#endif + +private: +#ifndef CRCPP_USE_CPP11 + CRC(); + CRC(const CRC &other); + CRC &operator=(const CRC &other); +#endif + + template + static IntegerType Reflect(IntegerType value, crcpp_uint16 numBits); + + template + static CRCType Finalize(CRCType remainder, + CRCType finalXOR, + bool reflectOutput); + + template + static CRCType UndoFinalize(CRCType remainder, + CRCType finalXOR, + bool reflectOutput); + + template + static CRCType CalculateRemainder( + const void *data, + crcpp_size size, + const Parameters ¶meters, + CRCType remainder); + + template + static CRCType CalculateRemainder(const void *data, + crcpp_size size, + const Table &lookupTable, + CRCType remainder); + + template + static crcpp_constexpr IntegerType BoundedConstexprValue(IntegerType x); +}; + +/** + @brief Returns a CRC lookup table construct using these CRC parameters. + @note This function primarily exists to allow use of the auto keyword + instead of instantiating + a table directly, since template parameters are not inferred in + constructors. + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return CRC lookup table +*/ +template +inline CRC::Table +CRC::Parameters::MakeTable() const { + // This should take advantage of RVO and optimize out the copy. + return CRC::Table(*this); +} + +/** + @brief Constructs a CRC table from a set of CRC parameters + @param[in] parameters CRC parameters + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC +*/ +template +inline CRC::Table::Table( + const Parameters ¶meters) + : parameters(parameters) { + InitTable(); +} + +#ifdef CRCPP_USE_CPP11 +/** + @brief Constructs a CRC table from a set of CRC parameters + @param[in] parameters CRC parameters + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC +*/ +template +inline CRC::Table::Table( + Parameters &¶meters) + : parameters(::std::move(parameters)) { + InitTable(); +} +#endif + +/** + @brief Gets the CRC parameters used to construct the CRC table + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return CRC parameters +*/ +template +inline const CRC::Parameters + &CRC::Table::GetParameters() const { + return parameters; +} + +/** + @brief Gets the CRC table + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return CRC table +*/ +template +inline const CRCType *CRC::Table::GetTable() const { + return table; +} + +/** + @brief Gets an entry in the CRC table + @param[in] index Index into the CRC table + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return CRC table entry +*/ +template +inline CRCType CRC::Table::operator[]( + unsigned char index) const { + return table[index]; +} + +/** + @brief Initializes a CRC table. + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC +*/ +template +inline void CRC::Table::InitTable() { + // For masking off the bits for the CRC (in the event that the number of bits + // in CRCType is larger than CRCWidth) + static crcpp_constexpr CRCType BIT_MASK( + (CRCType(1) << (CRCWidth - CRCType(1))) | + ((CRCType(1) << (CRCWidth - CRCType(1))) - CRCType(1))); + + static crcpp_constexpr CRCType SHIFT( + CRC::BoundedConstexprValue(CHAR_BIT - CRCWidth)); + + CRCType crc; + unsigned char byte = 0; + + // Loop over each dividend (each possible number storable in an unsigned char) + do { + crc = CRC::CalculateRemainder( + &byte, sizeof(byte), parameters, CRCType(0)); + + // This mask might not be necessary; all unit tests pass with this line + // commented out, + // but that might just be a coincidence based on the CRC parameters used for + // testing. + // In any case, this is harmless to leave in and only adds a single machine + // instruction per loop iteration. + crc &= BIT_MASK; + + if (!parameters.reflectInput && CRCWidth < CHAR_BIT) { + // Undo the special operation at the end of the CalculateRemainder() + // function for non-reflected CRCs < CHAR_BIT. + crc <<= SHIFT; + } + + table[byte] = crc; + } while (++byte); +} + +/** + @brief Computes a CRC. + @param[in] data Data over which CRC will be computed + @param[in] size Size of the data + @param[in] parameters CRC parameters + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return CRC +*/ +template +inline CRCType CRC::Calculate(const void *data, + crcpp_size size, + const Parameters ¶meters) { + CRCType remainder = + CalculateRemainder(data, size, parameters, parameters.initialValue); + + // No need to mask the remainder here; the mask will be applied in the + // Finalize() function. + + return Finalize( + remainder, + parameters.finalXOR, + parameters.reflectInput != parameters.reflectOutput); +} +/** + @brief Appends additional data to a previous CRC calculation. + @note This function can be used to compute multi-part CRCs. + @param[in] data Data over which CRC will be computed + @param[in] size Size of the data + @param[in] parameters CRC parameters + @param[in] crc CRC from a previous calculation + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return CRC +*/ +template +inline CRCType CRC::Calculate(const void *data, + crcpp_size size, + const Parameters ¶meters, + CRCType crc) { + CRCType remainder = UndoFinalize( + crc, + parameters.finalXOR, + parameters.reflectInput != parameters.reflectOutput); + + remainder = CalculateRemainder(data, size, parameters, remainder); + + // No need to mask the remainder here; the mask will be applied in the + // Finalize() function. + + return Finalize( + remainder, + parameters.finalXOR, + parameters.reflectInput != parameters.reflectOutput); +} + +/** + @brief Computes a CRC via a lookup table. + @param[in] data Data over which CRC will be computed + @param[in] size Size of the data + @param[in] lookupTable CRC lookup table + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return CRC +*/ +template +inline CRCType CRC::Calculate(const void *data, + crcpp_size size, + const Table &lookupTable) { + const Parameters ¶meters = lookupTable.GetParameters(); + + CRCType remainder = + CalculateRemainder(data, size, lookupTable, parameters.initialValue); + + // No need to mask the remainder here; the mask will be applied in the + // Finalize() function. + + return Finalize( + remainder, + parameters.finalXOR, + parameters.reflectInput != parameters.reflectOutput); +} + +/** + @brief Appends additional data to a previous CRC calculation using a lookup + table. + @note This function can be used to compute multi-part CRCs. + @param[in] data Data over which CRC will be computed + @param[in] size Size of the data + @param[in] lookupTable CRC lookup table + @param[in] crc CRC from a previous calculation + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return CRC +*/ +template +inline CRCType CRC::Calculate(const void *data, + crcpp_size size, + const Table &lookupTable, + CRCType crc) { + const Parameters ¶meters = lookupTable.GetParameters(); + + CRCType remainder = UndoFinalize( + crc, + parameters.finalXOR, + parameters.reflectInput != parameters.reflectOutput); + + remainder = CalculateRemainder(data, size, lookupTable, remainder); + + // No need to mask the remainder here; the mask will be applied in the + // Finalize() function. + + return Finalize( + remainder, + parameters.finalXOR, + parameters.reflectInput != parameters.reflectOutput); +} + +/** + @brief Reflects (i.e. reverses the bits within) an integer value. + @param[in] value Value to reflect + @param[in] numBits Number of bits in the integer which will be reflected + @tparam IntegerType Integer type of the value being reflected + @return Reflected value +*/ +template +inline IntegerType CRC::Reflect(IntegerType value, crcpp_uint16 numBits) { + IntegerType reversedValue(0); + + for (crcpp_uint16 i = 0; i < numBits; ++i) { + reversedValue = (reversedValue << 1) | (value & 1); + value >>= 1; + } + + return reversedValue; +} + +/** + @brief Computes the final reflection and XOR of a CRC remainder. + @param[in] remainder CRC remainder to reflect and XOR + @param[in] finalXOR Final value to XOR with the remainder + @param[in] reflectOutput true to reflect each byte of the remainder before + the XOR + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return Final CRC +*/ +template +inline CRCType CRC::Finalize(CRCType remainder, + CRCType finalXOR, + bool reflectOutput) { + // For masking off the bits for the CRC (in the event that the number of bits + // in CRCType is larger than CRCWidth) + static crcpp_constexpr CRCType BIT_MASK = + (CRCType(1) << (CRCWidth - CRCType(1))) | + ((CRCType(1) << (CRCWidth - CRCType(1))) - CRCType(1)); + + if (reflectOutput) { + remainder = Reflect(remainder, CRCWidth); + } + + return (remainder ^ finalXOR) & BIT_MASK; +} + +/** + @brief Undoes the process of computing the final reflection and XOR of a CRC + remainder. + @note This function allows for computation of multi-part CRCs + @note Calling UndoFinalize() followed by Finalize() (or vice versa) will + always return the original remainder value: + + CRCType x = ...; + CRCType y = Finalize(x, finalXOR, reflectOutput); + CRCType z = UndoFinalize(y, finalXOR, reflectOutput); + assert(x == z); + + @param[in] crc Reflected and XORed CRC + @param[in] finalXOR Final value XORed with the remainder + @param[in] reflectOutput true if the remainder is to be reflected + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return Un-finalized CRC remainder +*/ +template +inline CRCType CRC::UndoFinalize(CRCType crc, + CRCType finalXOR, + bool reflectOutput) { + // For masking off the bits for the CRC (in the event that the number of bits + // in CRCType is larger than CRCWidth) + static crcpp_constexpr CRCType BIT_MASK = + (CRCType(1) << (CRCWidth - CRCType(1))) | + ((CRCType(1) << (CRCWidth - CRCType(1))) - CRCType(1)); + + crc = (crc & BIT_MASK) ^ finalXOR; + + if (reflectOutput) { + crc = Reflect(crc, CRCWidth); + } + + return crc; +} + +/** + @brief Computes a CRC remainder. + @param[in] data Data over which the remainder will be computed + @param[in] size Size of the data + @param[in] parameters CRC parameters + @param[in] remainder Running CRC remainder. Can be an initial value or the + result of a previous CRC remainder calculation. + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return CRC remainder +*/ +template +inline CRCType CRC::CalculateRemainder( + const void *data, + crcpp_size size, + const Parameters ¶meters, + CRCType remainder) { +#ifdef CRCPP_USE_CPP11 + // This static_assert is put here because this function will always be + // compiled in no matter what + // the template parameters are and whether or not a table lookup or bit-by-bit + // algorithm is used. + static_assert(::std::numeric_limits::digits >= CRCWidth, + "CRCType is too small to contain a CRC of width CRCWidth."); +#else + // Catching this compile-time error is very important. Sadly, the compiler + // error will be very cryptic, but it's + // better than nothing. + enum { + static_assert_failed_CRCType_is_too_small_to_contain_a_CRC_of_width_CRCWidth = + 1 / (::std::numeric_limits::digits >= CRCWidth ? 1 : 0) + }; +#endif + + const unsigned char *current = reinterpret_cast(data); + + // Slightly different implementations based on the parameters. The current + // implementations try to eliminate as much + // computation from the inner loop (looping over each bit) as possible. + if (parameters.reflectInput) { + CRCType polynomial = CRC::Reflect(parameters.polynomial, CRCWidth); + while (size--) { + remainder ^= *current++; + + // An optimizing compiler might choose to unroll this loop. + for (crcpp_size i = 0; i < CHAR_BIT; ++i) { +#ifdef CRCPP_BRANCHLESS + // Clever way to avoid a branch at the expense of a multiplication. This + // code is equivalent to the following: + // if (remainder & 1) + // remainder = (remainder >> 1) ^ polynomial; + // else + // remainder >>= 1; + remainder = (remainder >> 1) ^ ((remainder & 1) * polynomial); +#else + remainder = (remainder & 1) ? ((remainder >> 1) ^ polynomial) + : (remainder >> 1); +#endif + } + } + } else if (CRCWidth >= CHAR_BIT) { + static crcpp_constexpr CRCType CRC_WIDTH_MINUS_ONE(CRCWidth - CRCType(1)); +#ifndef CRCPP_BRANCHLESS + static crcpp_constexpr CRCType CRC_HIGHEST_BIT_MASK(CRCType(1) + << CRC_WIDTH_MINUS_ONE); +#endif + static crcpp_constexpr CRCType SHIFT( + BoundedConstexprValue(CRCWidth - CHAR_BIT)); + + while (size--) { + remainder ^= (static_cast(*current++) << SHIFT); + + // An optimizing compiler might choose to unroll this loop. + for (crcpp_size i = 0; i < CHAR_BIT; ++i) { +#ifdef CRCPP_BRANCHLESS + // Clever way to avoid a branch at the expense of a multiplication. This + // code is equivalent to the following: + // if (remainder & CRC_HIGHEST_BIT_MASK) + // remainder = (remainder << 1) ^ parameters.polynomial; + // else + // remainder <<= 1; + remainder = + (remainder << 1) ^ + (((remainder >> CRC_WIDTH_MINUS_ONE) & 1) * parameters.polynomial); +#else + remainder = (remainder & CRC_HIGHEST_BIT_MASK) + ? ((remainder << 1) ^ parameters.polynomial) + : (remainder << 1); +#endif + } + } + } else { + static crcpp_constexpr CRCType CHAR_BIT_MINUS_ONE(CHAR_BIT - 1); +#ifndef CRCPP_BRANCHLESS + static crcpp_constexpr CRCType CHAR_BIT_HIGHEST_BIT_MASK( + CRCType(1) << CHAR_BIT_MINUS_ONE); +#endif + static crcpp_constexpr CRCType SHIFT( + BoundedConstexprValue(CHAR_BIT - CRCWidth)); + + CRCType polynomial = parameters.polynomial << SHIFT; + remainder <<= SHIFT; + + while (size--) { + remainder ^= *current++; + + // An optimizing compiler might choose to unroll this loop. + for (crcpp_size i = 0; i < CHAR_BIT; ++i) { +#ifdef CRCPP_BRANCHLESS + // Clever way to avoid a branch at the expense of a multiplication. This + // code is equivalent to the following: + // if (remainder & CHAR_BIT_HIGHEST_BIT_MASK) + // remainder = (remainder << 1) ^ polynomial; + // else + // remainder <<= 1; + remainder = (remainder << 1) ^ + (((remainder >> CHAR_BIT_MINUS_ONE) & 1) * polynomial); +#else + remainder = (remainder & CHAR_BIT_HIGHEST_BIT_MASK) + ? ((remainder << 1) ^ polynomial) + : (remainder << 1); +#endif + } + } + + remainder >>= SHIFT; + } + + return remainder; +} + +/** + @brief Computes a CRC remainder using lookup table. + @param[in] data Data over which the remainder will be computed + @param[in] size Size of the data + @param[in] lookupTable CRC lookup table + @param[in] remainder Running CRC remainder. Can be an initial value or the + result of a previous CRC remainder calculation. + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return CRC remainder +*/ +template +inline CRCType CRC::CalculateRemainder( + const void *data, + crcpp_size size, + const Table &lookupTable, + CRCType remainder) { + const unsigned char *current = reinterpret_cast(data); + + if (lookupTable.GetParameters().reflectInput) { + while (size--) { +#if defined(WIN32) || defined(_WIN32) || defined(WINCE) +// Disable warning about data loss when doing (remainder >> CHAR_BIT) when +// remainder is one byte long. The algorithm is still correct in this case, +// though it's possible that one additional machine instruction will be +// executed. +#pragma warning(push) +#pragma warning(disable : 4333) +#endif + remainder = + (remainder >> CHAR_BIT) ^ + lookupTable[static_cast(remainder ^ *current++)]; +#if defined(WIN32) || defined(_WIN32) || defined(WINCE) +#pragma warning(pop) +#endif + } + } else if (CRCWidth >= CHAR_BIT) { + static crcpp_constexpr CRCType SHIFT( + BoundedConstexprValue(CRCWidth - CHAR_BIT)); + + while (size--) { + remainder = (remainder << CHAR_BIT) ^ + lookupTable[static_cast((remainder >> SHIFT) ^ + *current++)]; + } + } else { + static crcpp_constexpr CRCType SHIFT( + BoundedConstexprValue(CHAR_BIT - CRCWidth)); + + remainder <<= SHIFT; + + while (size--) { + // Note: no need to mask here since remainder is guaranteed to fit in a + // single byte. + remainder = + lookupTable[static_cast(remainder ^ *current++)]; + } + + remainder >>= SHIFT; + } + + return remainder; +} + +/** + @brief Function to force a compile-time expression to be >= 0. + @note This function is used to avoid compiler warnings because all constexpr + values are evaluated + in a function even in a branch will never be executed. This also means + we don't need pragmas + to get rid of warnings, but it still can be computed at compile-time. + Win-win! + @param[in] x Compile-time expression to bound + @tparam CRCType Integer type for storing the CRC result + @tparam CRCWidth Number of bits in the CRC + @return Non-negative compile-time expression +*/ +template +inline crcpp_constexpr IntegerType CRC::BoundedConstexprValue(IntegerType x) { + return (x < IntegerType(0)) ? IntegerType(0) : x; +} + +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS +/** + @brief Returns a set of parameters for CRC-4 ITU. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-4 ITU has the following parameters and check value: + - polynomial = 0x3 + - initial value = 0x0 + - final XOR = 0x0 + - reflect input = true + - reflect output = true + - check value = 0x7 + @return CRC-4 ITU parameters +*/ +inline const CRC::Parameters &CRC::CRC_4_ITU() { + static const Parameters parameters = { + 0x3, 0x0, 0x0, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-5 EPC. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-5 EPC has the following parameters and check value: + - polynomial = 0x09 + - initial value = 0x09 + - final XOR = 0x00 + - reflect input = false + - reflect output = false + - check value = 0x00 + @return CRC-5 EPC parameters +*/ +inline const CRC::Parameters &CRC::CRC_5_EPC() { + static const Parameters parameters = { + 0x09, 0x09, 0x00, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-5 ITU. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-5 ITU has the following parameters and check value: + - polynomial = 0x15 + - initial value = 0x00 + - final XOR = 0x00 + - reflect input = true + - reflect output = true + - check value = 0x07 + @return CRC-5 ITU parameters +*/ +inline const CRC::Parameters &CRC::CRC_5_ITU() { + static const Parameters parameters = { + 0x15, 0x00, 0x00, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-5 USB. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-5 USB has the following parameters and check value: + - polynomial = 0x05 + - initial value = 0x1F + - final XOR = 0x1F + - reflect input = true + - reflect output = true + - check value = 0x19 + @return CRC-5 USB parameters +*/ +inline const CRC::Parameters &CRC::CRC_5_USB() { + static const Parameters parameters = { + 0x05, 0x1F, 0x1F, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-6 CDMA2000-A. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-6 CDMA2000-A has the following parameters and check value: + - polynomial = 0x27 + - initial value = 0x3F + - final XOR = 0x00 + - reflect input = false + - reflect output = false + - check value = 0x0D + @return CRC-6 CDMA2000-A parameters +*/ +inline const CRC::Parameters &CRC::CRC_6_CDMA2000A() { + static const Parameters parameters = { + 0x27, 0x3F, 0x00, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-6 CDMA2000-B. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-6 CDMA2000-A has the following parameters and check value: + - polynomial = 0x07 + - initial value = 0x3F + - final XOR = 0x00 + - reflect input = false + - reflect output = false + - check value = 0x3B + @return CRC-6 CDMA2000-B parameters +*/ +inline const CRC::Parameters &CRC::CRC_6_CDMA2000B() { + static const Parameters parameters = { + 0x07, 0x3F, 0x00, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-6 ITU. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-6 ITU has the following parameters and check value: + - polynomial = 0x03 + - initial value = 0x00 + - final XOR = 0x00 + - reflect input = true + - reflect output = true + - check value = 0x06 + @return CRC-6 ITU parameters +*/ +inline const CRC::Parameters &CRC::CRC_6_ITU() { + static const Parameters parameters = { + 0x03, 0x00, 0x00, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-7 JEDEC. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-7 JEDEC has the following parameters and check value: + - polynomial = 0x09 + - initial value = 0x00 + - final XOR = 0x00 + - reflect input = false + - reflect output = false + - check value = 0x75 + @return CRC-7 JEDEC parameters +*/ +inline const CRC::Parameters &CRC::CRC_7() { + static const Parameters parameters = { + 0x09, 0x00, 0x00, false, false}; + return parameters; +} +#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + +/** + @brief Returns a set of parameters for CRC-8 SMBus. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-8 SMBus has the following parameters and check value: + - polynomial = 0x07 + - initial value = 0x00 + - final XOR = 0x00 + - reflect input = false + - reflect output = false + - check value = 0xF4 + @return CRC-8 SMBus parameters +*/ +inline const CRC::Parameters &CRC::CRC_8() { + static const Parameters parameters = { + 0x07, 0x00, 0x00, false, false}; + return parameters; +} + +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS +/** + @brief Returns a set of parameters for CRC-8 EBU (aka CRC-8 AES). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-8 EBU has the following parameters and check value: + - polynomial = 0x1D + - initial value = 0xFF + - final XOR = 0x00 + - reflect input = true + - reflect output = true + - check value = 0x97 + @return CRC-8 EBU parameters +*/ +inline const CRC::Parameters &CRC::CRC_8_EBU() { + static const Parameters parameters = { + 0x1D, 0xFF, 0x00, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-8 MAXIM (aka CRC-8 DOW-CRC). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-8 MAXIM has the following parameters and check value: + - polynomial = 0x31 + - initial value = 0x00 + - final XOR = 0x00 + - reflect input = true + - reflect output = true + - check value = 0xA1 + @return CRC-8 MAXIM parameters +*/ +inline const CRC::Parameters &CRC::CRC_8_MAXIM() { + static const Parameters parameters = { + 0x31, 0x00, 0x00, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-8 WCDMA. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-8 WCDMA has the following parameters and check value: + - polynomial = 0x9B + - initial value = 0x00 + - final XOR = 0x00 + - reflect input = true + - reflect output = true + - check value = 0x25 + @return CRC-8 WCDMA parameters +*/ +inline const CRC::Parameters &CRC::CRC_8_WCDMA() { + static const Parameters parameters = { + 0x9B, 0x00, 0x00, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-10 ITU. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-10 ITU has the following parameters and check value: + - polynomial = 0x233 + - initial value = 0x000 + - final XOR = 0x000 + - reflect input = false + - reflect output = false + - check value = 0x199 + @return CRC-10 ITU parameters +*/ +inline const CRC::Parameters &CRC::CRC_10() { + static const Parameters parameters = { + 0x233, 0x000, 0x000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-10 CDMA2000. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-10 CDMA2000 has the following parameters and check value: + - polynomial = 0x3D9 + - initial value = 0x3FF + - final XOR = 0x000 + - reflect input = false + - reflect output = false + - check value = 0x233 + @return CRC-10 CDMA2000 parameters +*/ +inline const CRC::Parameters &CRC::CRC_10_CDMA2000() { + static const Parameters parameters = { + 0x3D9, 0x3FF, 0x000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-11 FlexRay. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-11 FlexRay has the following parameters and check value: + - polynomial = 0x385 + - initial value = 0x01A + - final XOR = 0x000 + - reflect input = false + - reflect output = false + - check value = 0x5A3 + @return CRC-11 FlexRay parameters +*/ +inline const CRC::Parameters &CRC::CRC_11() { + static const Parameters parameters = { + 0x385, 0x01A, 0x000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-12 CDMA2000. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-12 CDMA2000 has the following parameters and check value: + - polynomial = 0xF13 + - initial value = 0xFFF + - final XOR = 0x000 + - reflect input = false + - reflect output = false + - check value = 0xD4D + @return CRC-12 CDMA2000 parameters +*/ +inline const CRC::Parameters &CRC::CRC_12_CDMA2000() { + static const Parameters parameters = { + 0xF13, 0xFFF, 0x000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-12 DECT (aka CRC-12 X-CRC). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-12 DECT has the following parameters and check value: + - polynomial = 0x80F + - initial value = 0x000 + - final XOR = 0x000 + - reflect input = false + - reflect output = false + - check value = 0xF5B + @return CRC-12 DECT parameters +*/ +inline const CRC::Parameters &CRC::CRC_12_DECT() { + static const Parameters parameters = { + 0x80F, 0x000, 0x000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-12 UMTS (aka CRC-12 3GPP). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-12 UMTS has the following parameters and check value: + - polynomial = 0x80F + - initial value = 0x000 + - final XOR = 0x000 + - reflect input = false + - reflect output = true + - check value = 0xDAF + @return CRC-12 UMTS parameters +*/ +inline const CRC::Parameters &CRC::CRC_12_UMTS() { + static const Parameters parameters = { + 0x80F, 0x000, 0x000, false, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-13 BBC. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-13 BBC has the following parameters and check value: + - polynomial = 0x1CF5 + - initial value = 0x0000 + - final XOR = 0x0000 + - reflect input = false + - reflect output = false + - check value = 0x04FA + @return CRC-13 BBC parameters +*/ +inline const CRC::Parameters &CRC::CRC_13_BBC() { + static const Parameters parameters = { + 0x1CF5, 0x0000, 0x0000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-15 CAN. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-15 CAN has the following parameters and check value: + - polynomial = 0x4599 + - initial value = 0x0000 + - final XOR = 0x0000 + - reflect input = false + - reflect output = false + - check value = 0x059E + @return CRC-15 CAN parameters +*/ +inline const CRC::Parameters &CRC::CRC_15() { + static const Parameters parameters = { + 0x4599, 0x0000, 0x0000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-15 MPT1327. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-15 MPT1327 has the following parameters and check value: + - polynomial = 0x6815 + - initial value = 0x0000 + - final XOR = 0x0001 + - reflect input = false + - reflect output = false + - check value = 0x2566 + @return CRC-15 MPT1327 parameters +*/ +inline const CRC::Parameters &CRC::CRC_15_MPT1327() { + static const Parameters parameters = { + 0x6815, 0x0000, 0x0001, false, false}; + return parameters; +} +#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + +/** + @brief Returns a set of parameters for CRC-16 ARC (aka CRC-16 IBM, CRC-16 + LHA). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 ARC has the following parameters and check value: + - polynomial = 0x8005 + - initial value = 0x0000 + - final XOR = 0x0000 + - reflect input = true + - reflect output = true + - check value = 0xBB3D + @return CRC-16 ARC parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_ARC() { + static const Parameters parameters = { + 0x8005, 0x0000, 0x0000, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-16 BUYPASS (aka CRC-16 VERIFONE, + CRC-16 UMTS). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 BUYPASS has the following parameters and check value: + - polynomial = 0x8005 + - initial value = 0x0000 + - final XOR = 0x0000 + - reflect input = false + - reflect output = false + - check value = 0xFEE8 + @return CRC-16 BUYPASS parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_BUYPASS() { + static const Parameters parameters = { + 0x8005, 0x0000, 0x0000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-16 CCITT FALSE. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 CCITT FALSE has the following parameters and check value: + - polynomial = 0x1021 + - initial value = 0xFFFF + - final XOR = 0x0000 + - reflect input = false + - reflect output = false + - check value = 0x29B1 + @return CRC-16 CCITT FALSE parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_CCITTFALSE() { + static const Parameters parameters = { + 0x1021, 0xFFFF, 0x0000, false, false}; + return parameters; +} + +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS +/** + @brief Returns a set of parameters for CRC-16 CDMA2000. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 CDMA2000 has the following parameters and check value: + - polynomial = 0xC867 + - initial value = 0xFFFF + - final XOR = 0x0000 + - reflect input = false + - reflect output = false + - check value = 0x4C06 + @return CRC-16 CDMA2000 parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_CDMA2000() { + static const Parameters parameters = { + 0xC867, 0xFFFF, 0x0000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-16 DECT-R (aka CRC-16 R-CRC). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 DECT-R has the following parameters and check value: + - polynomial = 0x0589 + - initial value = 0x0000 + - final XOR = 0x0001 + - reflect input = false + - reflect output = false + - check value = 0x007E + @return CRC-16 DECT-R parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_DECTR() { + static const Parameters parameters = { + 0x0589, 0x0000, 0x0001, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-16 DECT-X (aka CRC-16 X-CRC). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 DECT-X has the following parameters and check value: + - polynomial = 0x0589 + - initial value = 0x0000 + - final XOR = 0x0000 + - reflect input = false + - reflect output = false + - check value = 0x007F + @return CRC-16 DECT-X parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_DECTX() { + static const Parameters parameters = { + 0x0589, 0x0000, 0x0000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-16 DNP. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 DNP has the following parameters and check value: + - polynomial = 0x3D65 + - initial value = 0x0000 + - final XOR = 0xFFFF + - reflect input = true + - reflect output = true + - check value = 0xEA82 + @return CRC-16 DNP parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_DNP() { + static const Parameters parameters = { + 0x3D65, 0x0000, 0xFFFF, true, true}; + return parameters; +} +#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + +/** + @brief Returns a set of parameters for CRC-16 GENIBUS (aka CRC-16 EPC, + CRC-16 I-CODE, CRC-16 DARC). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 GENIBUS has the following parameters and check value: + - polynomial = 0x1021 + - initial value = 0xFFFF + - final XOR = 0xFFFF + - reflect input = false + - reflect output = false + - check value = 0xD64E + @return CRC-16 GENIBUS parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_GENIBUS() { + static const Parameters parameters = { + 0x1021, 0xFFFF, 0xFFFF, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-16 KERMIT (aka CRC-16 CCITT, + CRC-16 CCITT-TRUE). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 KERMIT has the following parameters and check value: + - polynomial = 0x1021 + - initial value = 0x0000 + - final XOR = 0x0000 + - reflect input = true + - reflect output = true + - check value = 0x2189 + @return CRC-16 KERMIT parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_KERMIT() { + static const Parameters parameters = { + 0x1021, 0x0000, 0x0000, true, true}; + return parameters; +} + +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS +/** + @brief Returns a set of parameters for CRC-16 MAXIM. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 MAXIM has the following parameters and check value: + - polynomial = 0x8005 + - initial value = 0x0000 + - final XOR = 0xFFFF + - reflect input = true + - reflect output = true + - check value = 0x44C2 + @return CRC-16 MAXIM parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_MAXIM() { + static const Parameters parameters = { + 0x8005, 0x0000, 0xFFFF, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-16 MODBUS. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 MODBUS has the following parameters and check value: + - polynomial = 0x8005 + - initial value = 0xFFFF + - final XOR = 0x0000 + - reflect input = true + - reflect output = true + - check value = 0x4B37 + @return CRC-16 MODBUS parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_MODBUS() { + static const Parameters parameters = { + 0x8005, 0xFFFF, 0x0000, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-16 T10-DIF. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 T10-DIF has the following parameters and check value: + - polynomial = 0x8BB7 + - initial value = 0x0000 + - final XOR = 0x0000 + - reflect input = false + - reflect output = false + - check value = 0xD0DB + @return CRC-16 T10-DIF parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_T10DIF() { + static const Parameters parameters = { + 0x8BB7, 0x0000, 0x0000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-16 USB. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 USB has the following parameters and check value: + - polynomial = 0x8005 + - initial value = 0xFFFF + - final XOR = 0xFFFF + - reflect input = true + - reflect output = true + - check value = 0xB4C8 + @return CRC-16 USB parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_USB() { + static const Parameters parameters = { + 0x8005, 0xFFFF, 0xFFFF, true, true}; + return parameters; +} +#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + +/** + @brief Returns a set of parameters for CRC-16 X-25 (aka CRC-16 IBM-SDLC, + CRC-16 ISO-HDLC, CRC-16 B). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 X-25 has the following parameters and check value: + - polynomial = 0x1021 + - initial value = 0xFFFF + - final XOR = 0xFFFF + - reflect input = true + - reflect output = true + - check value = 0x906E + @return CRC-16 X-25 parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_X25() { + static const Parameters parameters = { + 0x1021, 0xFFFF, 0xFFFF, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-16 XMODEM (aka CRC-16 ZMODEM, + CRC-16 ACORN, CRC-16 LTE). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-16 XMODEM has the following parameters and check value: + - polynomial = 0x1021 + - initial value = 0x0000 + - final XOR = 0x0000 + - reflect input = false + - reflect output = false + - check value = 0x31C3 + @return CRC-16 XMODEM parameters +*/ +inline const CRC::Parameters &CRC::CRC_16_XMODEM() { + static const Parameters parameters = { + 0x1021, 0x0000, 0x0000, false, false}; + return parameters; +} + +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS +/** + @brief Returns a set of parameters for CRC-17 CAN. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-17 CAN has the following parameters and check value: + - polynomial = 0x1685B + - initial value = 0x00000 + - final XOR = 0x00000 + - reflect input = false + - reflect output = false + - check value = 0x04F03 + @return CRC-17 CAN parameters +*/ +inline const CRC::Parameters &CRC::CRC_17_CAN() { + static const Parameters parameters = { + 0x1685B, 0x00000, 0x00000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-21 CAN. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-21 CAN has the following parameters and check value: + - polynomial = 0x102899 + - initial value = 0x000000 + - final XOR = 0x000000 + - reflect input = false + - reflect output = false + - check value = 0x0ED841 + @return CRC-21 CAN parameters +*/ +inline const CRC::Parameters &CRC::CRC_21_CAN() { + static const Parameters parameters = { + 0x102899, 0x000000, 0x000000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-24 OPENPGP. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-24 OPENPGP has the following parameters and check value: + - polynomial = 0x864CFB + - initial value = 0xB704CE + - final XOR = 0x000000 + - reflect input = false + - reflect output = false + - check value = 0x21CF02 + @return CRC-24 OPENPGP parameters +*/ +inline const CRC::Parameters &CRC::CRC_24() { + static const Parameters parameters = { + 0x864CFB, 0xB704CE, 0x000000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-24 FlexRay-A. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-24 FlexRay-A has the following parameters and check value: + - polynomial = 0x5D6DCB + - initial value = 0xFEDCBA + - final XOR = 0x000000 + - reflect input = false + - reflect output = false + - check value = 0x7979BD + @return CRC-24 FlexRay-A parameters +*/ +inline const CRC::Parameters &CRC::CRC_24_FLEXRAYA() { + static const Parameters parameters = { + 0x5D6DCB, 0xFEDCBA, 0x000000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-24 FlexRay-B. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-24 FlexRay-B has the following parameters and check value: + - polynomial = 0x5D6DCB + - initial value = 0xABCDEF + - final XOR = 0x000000 + - reflect input = false + - reflect output = false + - check value = 0x1F23B8 + @return CRC-24 FlexRay-B parameters +*/ +inline const CRC::Parameters &CRC::CRC_24_FLEXRAYB() { + static const Parameters parameters = { + 0x5D6DCB, 0xABCDEF, 0x000000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-30 CDMA. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-30 CDMA has the following parameters and check value: + - polynomial = 0x2030B9C7 + - initial value = 0x3FFFFFFF + - final XOR = 0x00000000 + - reflect input = false + - reflect output = false + - check value = 0x3B3CB540 + @return CRC-30 CDMA parameters +*/ +inline const CRC::Parameters &CRC::CRC_30() { + static const Parameters parameters = { + 0x2030B9C7, 0x3FFFFFFF, 0x00000000, false, false}; + return parameters; +} +#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + +/** + @brief Returns a set of parameters for CRC-32 (aka CRC-32 ADCCP, CRC-32 + PKZip). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-32 has the following parameters and check value: + - polynomial = 0x04C11DB7 + - initial value = 0xFFFFFFFF + - final XOR = 0xFFFFFFFF + - reflect input = true + - reflect output = true + - check value = 0xCBF43926 + @return CRC-32 parameters +*/ +inline const CRC::Parameters &CRC::CRC_32() { + static const Parameters parameters = { + 0x04C11DB7, 0xFFFFFFFF, 0xFFFFFFFF, true, true}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-32 BZIP2 (aka CRC-32 AAL5, CRC-32 + DECT-B, CRC-32 B-CRC). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-32 BZIP2 has the following parameters and check value: + - polynomial = 0x04C11DB7 + - initial value = 0xFFFFFFFF + - final XOR = 0xFFFFFFFF + - reflect input = false + - reflect output = false + - check value = 0xFC891918 + @return CRC-32 BZIP2 parameters +*/ +inline const CRC::Parameters &CRC::CRC_32_BZIP2() { + static const Parameters parameters = { + 0x04C11DB7, 0xFFFFFFFF, 0xFFFFFFFF, false, false}; + return parameters; +} + +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS +/** + @brief Returns a set of parameters for CRC-32 C (aka CRC-32 ISCSI, CRC-32 + Castagnoli, CRC-32 Interlaken). + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-32 C has the following parameters and check value: + - polynomial = 0x1EDC6F41 + - initial value = 0xFFFFFFFF + - final XOR = 0xFFFFFFFF + - reflect input = true + - reflect output = true + - check value = 0xE3069283 + @return CRC-32 C parameters +*/ +inline const CRC::Parameters &CRC::CRC_32_C() { + static const Parameters parameters = { + 0x1EDC6F41, 0xFFFFFFFF, 0xFFFFFFFF, true, true}; + return parameters; +} +#endif + +/** + @brief Returns a set of parameters for CRC-32 MPEG-2. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-32 MPEG-2 has the following parameters and check value: + - polynomial = 0x04C11DB7 + - initial value = 0xFFFFFFFF + - final XOR = 0x00000000 + - reflect input = false + - reflect output = false + - check value = 0x0376E6E7 + @return CRC-32 MPEG-2 parameters +*/ +inline const CRC::Parameters &CRC::CRC_32_MPEG2() { + static const Parameters parameters = { + 0x04C11DB7, 0xFFFFFFFF, 0x00000000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-32 POSIX. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-32 POSIX has the following parameters and check value: + - polynomial = 0x04C11DB7 + - initial value = 0x00000000 + - final XOR = 0xFFFFFFFF + - reflect input = false + - reflect output = false + - check value = 0x765E7680 + @return CRC-32 POSIX parameters +*/ +inline const CRC::Parameters &CRC::CRC_32_POSIX() { + static const Parameters parameters = { + 0x04C11DB7, 0x00000000, 0xFFFFFFFF, false, false}; + return parameters; +} + +#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS +/** + @brief Returns a set of parameters for CRC-32 Q. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-32 Q has the following parameters and check value: + - polynomial = 0x814141AB + - initial value = 0x00000000 + - final XOR = 0x00000000 + - reflect input = false + - reflect output = false + - check value = 0x3010BF7F + @return CRC-32 Q parameters +*/ +inline const CRC::Parameters &CRC::CRC_32_Q() { + static const Parameters parameters = { + 0x814141AB, 0x00000000, 0x00000000, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-40 GSM. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-40 GSM has the following parameters and check value: + - polynomial = 0x0004820009 + - initial value = 0x0000000000 + - final XOR = 0xFFFFFFFFFF + - reflect input = false + - reflect output = false + - check value = 0xD4164FC646 + @return CRC-40 GSM parameters +*/ +inline const CRC::Parameters &CRC::CRC_40_GSM() { + static const Parameters parameters = { + 0x0004820009, 0x0000000000, 0xFFFFFFFFFF, false, false}; + return parameters; +} + +/** + @brief Returns a set of parameters for CRC-64 ECMA. + @note The parameters are static and are delayed-constructed to reduce memory + footprint. + @note CRC-64 ECMA has the following parameters and check value: + - polynomial = 0x42F0E1EBA9EA3693 + - initial value = 0x0000000000000000 + - final XOR = 0x0000000000000000 + - reflect input = false + - reflect output = false + - check value = 0x6C40DF5F0B497347 + @return CRC-64 ECMA parameters +*/ +inline const CRC::Parameters &CRC::CRC_64() { + static const Parameters parameters = { + 0x42F0E1EBA9EA3693, 0x0000000000000000, 0x0000000000000000, false, false}; + return parameters; +} +#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS + +#ifdef CRCPP_USE_NAMESPACE +} +#endif + +#endif // CRCPP_CRC_H_ diff --git a/paddle/fluid/recordio/header.cc b/paddle/fluid/recordio/header.cc index c82d05c3a2..4e35e62d0a 100644 --- a/paddle/fluid/recordio/header.cc +++ b/paddle/fluid/recordio/header.cc @@ -26,18 +26,18 @@ Header::Header() Header::Header(uint32_t num, uint32_t sum, Compressor c, uint32_t cs) : num_records_(num), checksum_(sum), compressor_(c), compress_size_(cs) {} -void Header::Parse(std::istream& iss) { - iss.read(reinterpret_cast(&num_records_), sizeof(uint32_t)); - iss.read(reinterpret_cast(&checksum_), sizeof(uint32_t)); - iss.read(reinterpret_cast(&compressor_), sizeof(uint32_t)); - iss.read(reinterpret_cast(&compress_size_), sizeof(uint32_t)); +void Header::Parse(Stream* iss) { + iss.Read(reinterpret_cast(&num_records_), sizeof(uint32_t)); + iss.Read(reinterpret_cast(&checksum_), sizeof(uint32_t)); + iss.Read(reinterpret_cast(&compressor_), sizeof(uint32_t)); + iss.Read(reinterpret_cast(&compress_size_), sizeof(uint32_t)); } -void Header::Write(std::ostream& os) { - os.write(reinterpret_cast(&num_records_), sizeof(uint32_t)); - os.write(reinterpret_cast(&checksum_), sizeof(uint32_t)); - os.write(reinterpret_cast(&compressor_), sizeof(uint32_t)); - os.write(reinterpret_cast(&compress_size_), sizeof(uint32_t)); +void Header::Write(Stream* os) { + os.Write(reinterpret_cast(&num_records_), sizeof(uint32_t)); + os.Write(reinterpret_cast(&checksum_), sizeof(uint32_t)); + os.Write(reinterpret_cast(&compressor_), sizeof(uint32_t)); + os.Write(reinterpret_cast(&compress_size_), sizeof(uint32_t)); } // std::ostream& operator << (std::ostream& os, Header h) { @@ -54,28 +54,8 @@ std::ostream& operator<<(std::ostream& os, Header h) { return os; } -// bool operator==(Header l, Header r) { -// return num_records_ == rhs.NumRecords() && -// checksum_ == rhs.Checksum() && -// compressor_ == rhs.CompressType() && -// compress_size_ == rhs.CompressSize(); -// } - bool operator==(Header l, Header r) { return l.NumRecords() == r.NumRecords() && l.Checksum() == r.Checksum() && l.CompressType() == r.CompressType() && l.CompressSize() == r.CompressSize(); } - -// size_t CompressData(const std::string& os, Compressor ct, char* buffer) { -// size_t compress_size = 0; - -// // std::unique_ptr buffer(new char[kDefaultMaxChunkSize]); -// // std::string compressed; -// compress_size =os.size(); -// memcpy(buffer, os.c_str(), compress_size); -// return compress_size; -// } - -} // namespace recordio -} // namespace paddle diff --git a/paddle/fluid/recordio/header.h b/paddle/fluid/recordio/header.h index 92c040617d..21e23f0a25 100644 --- a/paddle/fluid/recordio/header.h +++ b/paddle/fluid/recordio/header.h @@ -16,6 +16,8 @@ #include +#include "paddle/fluid/recordio/io.h" + namespace paddle { namespace recordio { @@ -43,8 +45,8 @@ public: Header(); Header(uint32_t num, uint32_t sum, Compressor ct, uint32_t cs); - void Write(std::ostream& os); - void Parse(std::istream& iss); + void Write(Stream* os); + void Parse(Stream* iss); uint32_t NumRecords() const { return num_records_; } uint32_t Checksum() const { return checksum_; } diff --git a/paddle/fluid/recordio/header_test.cc b/paddle/fluid/recordio/header_test.cc index 322f63190a..df52d7feef 100644 --- a/paddle/fluid/recordio/header_test.cc +++ b/paddle/fluid/recordio/header_test.cc @@ -22,15 +22,12 @@ using namespace paddle::recordio; TEST(Recordio, ChunkHead) { Header hdr(0, 1, Compressor::kGzip, 3); - std::ostringstream oss; + Stream* oss = Stream::Open("/tmp/record_1", "w"); hdr.Write(oss); - std::istringstream iss(oss.str()); + Stream* iss = Stream::Open("/tmp/record_1", "r"); Header hdr2; hdr2.Parse(iss); - std::ostringstream oss2; - hdr2.Write(oss2); - EXPECT_STREQ(oss2.str().c_str(), oss.str().c_str()); EXPECT_TRUE(hdr == hdr2); } diff --git a/paddle/fluid/recordio/io.cc b/paddle/fluid/recordio/io.cc index 2c82d1d42d..e5571ddf5d 100644 --- a/paddle/fluid/recordio/io.cc +++ b/paddle/fluid/recordio/io.cc @@ -15,6 +15,8 @@ #include "paddle/fluid/recordio/io.h" #include "paddle/fluid/string/piece.h" +#include + namespace paddle { namespace recordio { Stream* Stream::Open(const char* filename, const char* mode) { @@ -38,7 +40,7 @@ void FileStream::Write(const void* ptr, size_t size) { } size_t FileStream::Tell() { return ftell(fp_); } -void FileStream::Seek(size_t p) { fseek(fp_, static_cast(p), SEEK_SET); } +void FileStream::Seek(size_t p) { fseek(fp_, p, SEEK_SET); } bool FileStream::Eof() { return feof(fp_); } diff --git a/paddle/fluid/recordio/io.h b/paddle/fluid/recordio/io.h index ff647b95d8..dedfed787d 100644 --- a/paddle/fluid/recordio/io.h +++ b/paddle/fluid/recordio/io.h @@ -16,19 +16,21 @@ #include #include + #include "paddle/fluid/platform/enforce.h" +#include "paddle/fluid/platform/macros.h" // DISABLE_COPY_ASSIGN namespace paddle { namespace recordio { -// Stream abstract object for read and write +// Seekable Stream Interface for read and write class Stream { public: virtual ~Stream() {} - virtual size_t Read(void* ptr, size_t size); - virtual void Write(const void* ptr, size_t size); - virtual size_t Tell(); - virtual void Seek(); + virtual size_t Read(void* ptr, size_t size) = 0; + virtual void Write(const void* ptr, size_t size) = 0; + virtual size_t Tell() = 0; + virtual void Seek(size_t p) = 0; // Create Stream Instance static Stream* Open(const char* filename, const char* mode); }; @@ -47,6 +49,7 @@ public: private: FILE* fp_; + DISABLE_COPY_AND_ASSIGN(FileStream); }; } // namespace recordio diff --git a/paddle/fluid/recordio/io_test.cc b/paddle/fluid/recordio/io_test.cc index b2e5733ffe..8311494787 100644 --- a/paddle/fluid/recordio/io_test.cc +++ b/paddle/fluid/recordio/io_test.cc @@ -21,7 +21,7 @@ using namespace paddle::recordio; TEST(FileStream, IO) { { // Write - Stream* fs = Stream::Open("/tmp/record_0", "rw"); + Stream* fs = Stream::Open("/tmp/record_0", "w"); fs->Write("hello", 6); delete fs; } diff --git a/paddle/fluid/recordio/writer.cc b/paddle/fluid/recordio/writer.cc index acb84fb8e8..b2b0dd1017 100644 --- a/paddle/fluid/recordio/writer.cc +++ b/paddle/fluid/recordio/writer.cc @@ -26,16 +26,16 @@ Writer::Writer(Stream* fo, int maxChunkSize, int compressor) chunk_.reset(new Chunk); } -size_t Writer::Write(const std::string& record) { +size_t Writer::Write(const char* buf, size_t length) { if (stream_ == nullptr) { LOG(WARNING) << "Cannot write since writer had been closed."; return 0; } - if ((record.size() + chunk_->NumBytes()) > max_chunk_size_) { + if ((length + chunk_->NumBytes()) > max_chunk_size_) { chunk_->Dump(stream_, compressor_); } - chunk_->Add(record); - return record.size(); + chunk_->Add(buf, length); + return length; } // size_t Writer::Write(const char* buf, size_t length) { diff --git a/paddle/fluid/recordio/writer.h b/paddle/fluid/recordio/writer.h index 250d59813c..d610450c53 100644 --- a/paddle/fluid/recordio/writer.h +++ b/paddle/fluid/recordio/writer.h @@ -16,7 +16,6 @@ #include #include -#include "paddle/fluid/platform/macros.h" // DISABLE_COPY_ASSIGN #include "paddle/fluid/recordio/header.h" #include "paddle/fluid/recordio/io.h" @@ -44,7 +43,6 @@ private: int max_chunk_size_; // Compressor used for chuck Compressor compressor_; - DISABLE_COPY_AND_ASSIGN(Writer); }; diff --git a/paddle/fluid/recordio/writer_test.cc b/paddle/fluid/recordio/writer_test.cc index 1ba32bf2df..7c7f823c8d 100644 --- a/paddle/fluid/recordio/writer_test.cc +++ b/paddle/fluid/recordio/writer_test.cc @@ -18,4 +18,4 @@ using namespace paddle::recordio; -TEST(Writer, Normal) { Stream } +TEST(Writer, Normal) {} -- GitLab