Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
BaiXuePrincess
Paddle
提交
7016979c
P
Paddle
项目概览
BaiXuePrincess
/
Paddle
与 Fork 源项目一致
Fork自
PaddlePaddle / Paddle
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Paddle
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
7016979c
编写于
3月 05, 2018
作者:
D
dongzhihong
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
"add crc32 encoder"
上级
69c79911
变更
15
显示空白变更内容
内联
并排
Showing
15 changed file
with
2114 addition
and
144 deletion
+2114
-144
paddle/fluid/recordio/CMakeLists.txt
paddle/fluid/recordio/CMakeLists.txt
+1
-0
paddle/fluid/recordio/chunk.cc
paddle/fluid/recordio/chunk.cc
+105
-0
paddle/fluid/recordio/chunk.h
paddle/fluid/recordio/chunk.h
+20
-93
paddle/fluid/recordio/chunk_test.cc
paddle/fluid/recordio/chunk_test.cc
+23
-0
paddle/fluid/recordio/crc32.h
paddle/fluid/recordio/crc32.h
+33
-0
paddle/fluid/recordio/detail/crc.h
paddle/fluid/recordio/detail/crc.h
+1899
-0
paddle/fluid/recordio/header.cc
paddle/fluid/recordio/header.cc
+10
-30
paddle/fluid/recordio/header.h
paddle/fluid/recordio/header.h
+4
-2
paddle/fluid/recordio/header_test.cc
paddle/fluid/recordio/header_test.cc
+2
-5
paddle/fluid/recordio/io.cc
paddle/fluid/recordio/io.cc
+3
-1
paddle/fluid/recordio/io.h
paddle/fluid/recordio/io.h
+8
-5
paddle/fluid/recordio/io_test.cc
paddle/fluid/recordio/io_test.cc
+1
-1
paddle/fluid/recordio/writer.cc
paddle/fluid/recordio/writer.cc
+4
-4
paddle/fluid/recordio/writer.h
paddle/fluid/recordio/writer.h
+0
-2
paddle/fluid/recordio/writer_test.cc
paddle/fluid/recordio/writer_test.cc
+1
-1
未找到文件。
paddle/fluid/recordio/CMakeLists.txt
浏览文件 @
7016979c
...
@@ -2,3 +2,4 @@ cc_library(header SRCS header.cc)
...
@@ -2,3 +2,4 @@ cc_library(header SRCS header.cc)
cc_test
(
header_test SRCS header_test.cc DEPS header
)
cc_test
(
header_test SRCS header_test.cc DEPS header
)
cc_library
(
io SRCS io.cc DEPS stringpiece
)
cc_library
(
io SRCS io.cc DEPS stringpiece
)
cc_test
(
io_test SRCS io_test.cc DEPS io
)
cc_test
(
io_test SRCS io_test.cc DEPS io
)
cc_library
(
chunk SRCS chunk.cc DEPS snappy
)
paddle/fluid/recordio/chunk.cc
0 → 100644
浏览文件 @
7016979c
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/recordio/chunk.h"
#include <cstring>
#include <sstream>
#include <utility>
#include "snappy.h"
#include "paddle/fluid/recordio/crc32.h"
namespace
paddle
{
namespace
recordio
{
void
Chunk
::
Add
(
const
char
*
record
,
size_t
length
)
{
records_
.
emplace_after
(
std
::
move
(
s
));
num_bytes_
+=
s
.
size
()
*
sizeof
(
char
);
}
bool
Chunk
::
Dump
(
Stream
*
fo
,
Compressor
ct
)
{
// NOTE(dzhwinter): don't check records.numBytes instead, because
// empty records are allowed.
if
(
records_
.
size
()
==
0
)
return
false
;
// pack the record into consecutive memory for compress
std
::
ostringstream
os
;
for
(
auto
&
record
:
records_
)
{
os
.
write
(
record
.
size
(),
sizeof
(
size_t
));
os
.
write
(
record
.
data
(),
static_cast
<
std
::
streamsize
>
(
record
.
size
()));
}
std
::
unique_ptr
<
char
[]
>
buffer
(
new
char
[
kDefaultMaxChunkSize
]);
size_t
compressed
=
CompressData
(
os
.
str
().
c_str
(),
num_bytes_
,
ct
,
buffer
.
get
());
uint32_t
checksum
=
Crc32
(
buffer
.
get
(),
compressed
);
Header
hdr
(
records_
.
size
(),
checksum
,
ct
,
static_cast
<
uint32_t
>
(
compressed
));
hdr
.
Write
(
fo
);
fo
.
Write
(
buffer
.
get
(),
compressed
);
return
true
;
}
void
Chunk
::
Parse
(
Stream
*
fi
,
size_t
offset
)
{
fi
->
Seek
(
offset
);
Header
hdr
;
hdr
.
Parse
(
fi
);
std
::
unique_ptr
<
char
[]
>
buffer
(
new
char
[
kDefaultMaxChunkSize
]);
fi
->
Read
(
buffer
.
get
(),
static_cast
<
size_t
>
(
hdr
.
CompressSize
()));
uint32_t
deflated_size
=
DeflateData
(
buffer
.
get
(),
hdr
.
CompressSize
(),
hdr
.
CompressType
());
std
::
istringstream
deflated
(
std
::
string
(
buffer
.
get
(),
deflated_size
));
for
(
size_t
i
=
0
;
i
<
hdr
.
NumRecords
();
++
i
)
{
uint32_t
rs
;
deflated
>>
rs
;
std
::
string
record
(
rs
,
'\0'
);
deflated
.
read
(
&
record
[
0
],
rs
);
records_
.
emplace_back
(
record
);
num_bytes_
+=
record
.
size
();
}
}
size_t
CompressData
(
const
char
*
in
,
size_t
in_length
,
Compressor
ct
,
char
*
out
)
{
size_t
compressd_size
=
0
;
switch
(
ct
)
{
case
Compressor
::
kNoCompress
:
// do nothing
memcpy
(
out
,
in
,
in_length
);
compressd_size
=
in_length
;
break
;
case
Compressor
::
kSnappy
:
snappy
::
RawCompress
(
in
,
in_length
,
out
,
&
compressd_size
);
break
;
}
return
compressd_size
;
}
void
DeflateData
(
const
char
*
in
,
size_t
in_length
,
Compressor
ct
,
char
*
out
)
{
switch
(
c
)
{
case
Compressor
::
kNoCompress
:
memcpy
(
out
,
in
,
in_length
);
break
;
case
Compressor
::
kSnappy
:
snappy
::
RawUncompress
(
in
,
in_length
,
out
);
break
;
}
}
}
// namespace recordio
}
// namespace paddle
paddle/fluid/recordio/chunk.h
浏览文件 @
7016979c
...
@@ -13,109 +13,36 @@
...
@@ -13,109 +13,36 @@
// limitations under the License.
// limitations under the License.
#pragma once
#pragma once
#include <forward_list>
#include <fstream>
#include <memory>
#include <sstream>
#include <string>
#include <string>
#include <utility>
#include <vector>
// Chunk
#include "paddle/fluid/recordio/header.h"
// a chunk contains the Header and optionally compressed records.
#include "paddle/fluid/recordio/io.h"
namespace
paddle
{
namespace
recordio
{
// A Chunk contains the Header and optionally compressed records.
class
Chunk
{
class
Chunk
{
public:
public:
Chunk
()
=
default
;
Chunk
()
{}
void
Add
(
const
char
*
record
,
size_t
length
);
void
Add
(
const
char
*
record
,
size_t
size
);
void
Add
(
const
std
::
string
&
);
// dump the chunk into w, and clears the chunk and makes it ready for
// the next add invocation.
bool
Dump
(
std
::
ostream
&
os
,
Compressor
ct
);
bool
Dump
(
Stream
*
fo
,
Compressor
ct
);
void
Parse
(
std
::
istream
&
iss
,
int64_t
offset
);
void
Parse
(
Stream
*
fi
,
size_t
offset
);
const
std
::
string
Record
(
int
i
)
{
return
records_
[
i
];
}
size_t
NumBytes
()
{
return
num_bytes_
;
}
size_t
NumBytes
()
{
return
num_bytes_
;
}
private:
private:
std
::
vector
<
std
::
string
>
records_
;
std
::
forward_list
<
std
::
string
>
records_
;
// sum of record lengths in bytes.
// sum of record lengths in bytes.
size_t
num_bytes_
;
size_t
num_bytes_
;
DISABLE_COPY_AND_ASSIGN
(
Chunk
);
};
};
size_t
CompressData
(
const
std
::
stringstream
&
ss
,
Compressor
ct
,
char
*
buffer
);
size_t
CompressData
(
const
char
*
in
,
size_t
in_length
,
Compressor
ct
,
char
*
out
);
uint32_t
DeflateData
(
char
*
buffer
,
uint32_t
size
,
Compressor
c
);
// implementation
void
Chunk
::
Add
(
const
std
::
string
&
s
)
{
num_bytes_
+=
s
.
size
()
*
sizeof
(
char
);
records_
.
emplace_back
(
std
::
move
(
s
));
// records_.resize(records_.size()+1);
// records_[records_.size()-1] = s;
}
void
Chunk
::
Add
(
const
char
*
record
,
size_t
length
)
{
Add
(
std
::
string
(
record
,
length
));
}
bool
Chunk
::
Dump
(
std
::
ostream
&
os
,
Compressor
ct
)
{
if
(
records_
.
size
()
==
0
)
return
false
;
// TODO(dzhwinter):
// we pack the string with same size buffer,
// then compress with another buffer.
// Here can be optimized if it is the bottle-neck.
std
::
ostringstream
oss
;
for
(
auto
&
record
:
records_
)
{
unsigned
len
=
record
.
size
();
oss
<<
len
;
oss
<<
record
;
// os.write(std::to_string(len).c_str(), sizeof(unsigned));
// os.write(record.c_str(), record.size());
}
std
::
unique_ptr
<
char
[]
>
buffer
(
new
char
[
kDefaultMaxChunkSize
]);
size_t
compressed
=
CompressData
(
oss
.
str
(),
ct
,
buffer
.
get
());
// TODO(dzhwinter): crc32 checksum
size_t
checksum
=
compressed
;
Header
hdr
(
records_
.
size
(),
checksum
,
ct
,
compressed
);
return
true
;
}
void
Chunk
::
Parse
(
std
::
istream
&
iss
,
int64_t
offset
)
{
iss
.
seekg
(
offset
,
iss
.
beg
);
Header
hdr
;
hdr
.
Parse
(
iss
);
std
::
unique_ptr
<
char
[]
>
buffer
(
new
char
[
kDefaultMaxChunkSize
]);
void
DeflateData
(
const
char
*
in
,
size_t
in_length
,
Compressor
ct
,
char
*
out
);
iss
.
read
(
buffer
.
get
(),
static_cast
<
size_t
>
(
hdr
.
CompressSize
()));
// TODO(dzhwinter): checksum
uint32_t
deflated_size
=
DeflateData
(
buffer
.
get
(),
hdr
.
CompressSize
(),
hdr
.
CompressType
());
std
::
istringstream
deflated
(
std
::
string
(
buffer
.
get
(),
deflated_size
));
for
(
size_t
i
=
0
;
i
<
hdr
.
NumRecords
();
++
i
)
{
uint32_t
rs
;
deflated
>>
rs
;
std
::
string
record
(
rs
,
'\0'
);
deflated
.
read
(
&
record
[
0
],
rs
);
records_
.
emplace_back
(
record
);
num_bytes_
+=
record
.
size
();
}
}
uint32_t
DeflateData
(
char
*
buffer
,
uint32_t
size
,
Compressor
c
)
{
}
// namespace recordio
uint32_t
deflated_size
=
0
;
}
// namespace paddle
std
::
string
uncompressed
;
switch
(
c
)
{
case
Compressor
::
kNoCompress
:
deflated_size
=
size
;
break
;
case
Compressor
::
kSnappy
:
// snappy::Uncompress(buffer, size, &uncompressed);
// deflated_size = uncompressed.size();
// memcpy(buffer, uncompressed.data(), uncompressed.size() *
// sizeof(char));
break
;
}
return
deflated_size
;
}
paddle/fluid/recordio/chunk_test.cc
0 → 100644
浏览文件 @
7016979c
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
#include "paddle/fluid/recordio/chunk.h"
#include <sstream>
#include "gtest/gtest.h"
using
namespace
paddle
::
recordio
;
TEST
(
Chunk
,
SaveLoad
)
{}
paddle/fluid/recordio/crc32.h
0 → 100644
浏览文件 @
7016979c
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// A wrapper on crc library https://github.com/d-bahr/CRCpp
#include <cstdint>
#include "paddle/fluid/recordio/detail/crc.h"
namespace
paddle
{
namespace
recordio
{
// usage
// char data[] = "hello,world";
// crc = Crc32(data, 12);
// Assert_EQ(crc, 68a85159);
uint32_t
Crc32
(
const
char
*
data
,
size_t
size
)
{
return
CRC
::
Calculate
(
data
,
size
,
CRC
::
CRC_32
())
}
}
// namespace recordio
}
// namespace paddle
paddle/fluid/recordio/detail/crc.h
0 → 100644
浏览文件 @
7016979c
// Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/**
@file CRC.h
@author Daniel Bahr
@version 0.2.0.6
@copyright
@parblock
CRC++
Copyright (c) 2016, Daniel Bahr
All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright notice,
this
list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above copyright
notice,
this list of conditions and the following disclaimer in the
documentation
and/or other materials provided with the distribution.
* Neither the name of CRC++ nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
@endparblock
*/
/*
CRC++ can be configured by setting various #defines before #including this
header file:
#define crcpp_uint8 - Specifies the type
used to store CRCs that have a width of 8 bits or less.
This type is not used
in CRC calculations. Defaults to ::std::uint8_t.
#define crcpp_uint16 - Specifies the type
used to store CRCs that have a width between 9 and 16 bits (inclusive).
This type is not used
in CRC calculations. Defaults to ::std::uint16_t.
#define crcpp_uint32 - Specifies the type
used to store CRCs that have a width between 17 and 32 bits (inclusive).
This type is not used
in CRC calculations. Defaults to ::std::uint32_t.
#define crcpp_uint64 - Specifies the type
used to store CRCs that have a width between 33 and 64 bits (inclusive).
This type is not used
in CRC calculations. Defaults to ::std::uint64_t.
#define crcpp_size - This type is used for
loop iteration and function signatures only. Defaults to ::std::size_t.
#define CRCPP_USE_NAMESPACE - Define to place all
CRC++ code within the ::CRCPP namespace.
#define CRCPP_BRANCHLESS - Define to enable a
branchless CRC implementation. The branchless implementation uses a single
integer
multiplication in the
bit-by-bit calculation instead of a small conditional. The branchless
implementation
may be faster on
processor architectures which support single-instruction integer
multiplication.
#define CRCPP_USE_CPP11 - Define to enables
C++11 features (move semantics, constexpr, static_assert, etc.).
#define CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS - Define to include
definitions for little-used CRCs.
*/
#ifndef CRCPP_CRC_H_
#define CRCPP_CRC_H_
#include <climits> // Includes CHAR_BIT
#ifdef CRCPP_USE_CPP11
#include <cstddef> // Includes ::std::size_t
#include <cstdint> // Includes ::std::uint8_t, ::std::uint16_t, ::std::uint32_t, ::std::uint64_t
#else
#include <stddef.h> // Includes size_t
#include <stdint.h> // Includes uint8_t, uint16_t, uint32_t, uint64_t
#endif
#include <limits> // Includes ::std::numeric_limits
#include <utility> // Includes ::std::move
#ifndef crcpp_uint8
#ifdef CRCPP_USE_CPP11
/// @brief Unsigned 8-bit integer definition, used primarily for parameter
/// definitions.
#define crcpp_uint8 ::std::uint8_t
#else
/// @brief Unsigned 8-bit integer definition, used primarily for parameter
/// definitions.
#define crcpp_uint8 uint8_t
#endif
#endif
#ifndef crcpp_uint16
#ifdef CRCPP_USE_CPP11
/// @brief Unsigned 16-bit integer definition, used primarily for parameter
/// definitions.
#define crcpp_uint16 ::std::uint16_t
#else
/// @brief Unsigned 16-bit integer definition, used primarily for parameter
/// definitions.
#define crcpp_uint16 uint16_t
#endif
#endif
#ifndef crcpp_uint32
#ifdef CRCPP_USE_CPP11
/// @brief Unsigned 32-bit integer definition, used primarily for parameter
/// definitions.
#define crcpp_uint32 ::std::uint32_t
#else
/// @brief Unsigned 32-bit integer definition, used primarily for parameter
/// definitions.
#define crcpp_uint32 uint32_t
#endif
#endif
#ifndef crcpp_uint64
#ifdef CRCPP_USE_CPP11
/// @brief Unsigned 64-bit integer definition, used primarily for parameter
/// definitions.
#define crcpp_uint64 ::std::uint64_t
#else
/// @brief Unsigned 64-bit integer definition, used primarily for parameter
/// definitions.
#define crcpp_uint64 uint64_t
#endif
#endif
#ifndef crcpp_size
#ifdef CRCPP_USE_CPP11
/// @brief Unsigned size definition, used for specifying data sizes.
#define crcpp_size ::std::size_t
#else
/// @brief Unsigned size definition, used for specifying data sizes.
#define crcpp_size size_t
#endif
#endif
#ifdef CRCPP_USE_CPP11
/// @brief Compile-time expression definition.
#define crcpp_constexpr constexpr
#else
/// @brief Compile-time expression definition.
#define crcpp_constexpr const
#endif
#ifdef CRCPP_USE_NAMESPACE
namespace
CRCPP
{
#endif
/**
@brief Static class for computing CRCs.
@note This class supports computation of full and multi-part CRCs, using a
bit-by-bit algorithm or a
byte-by-byte lookup table. The CRCs are calculated using as many
optimizations as is reasonable.
If compiling with C++11, the constexpr keyword is used liberally so that
many calculations are
performed at compile-time instead of at runtime.
*/
class
CRC
{
public:
// Forward declaration
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
struct
Table
;
/**
@brief CRC parameters.
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
struct
Parameters
{
CRCType
polynomial
;
///< CRC polynomial
CRCType
initialValue
;
///< Initial CRC value
CRCType
finalXOR
;
///< Value to XOR with the final CRC
bool
reflectInput
;
///< true to reflect all input bytes
bool
reflectOutput
;
///< true to reflect the output CRC (reflection occurs
/// before the final XOR)
Table
<
CRCType
,
CRCWidth
>
MakeTable
()
const
;
};
/**
@brief CRC lookup table. After construction, the CRC parameters are fixed.
@note A CRC table can be used for multiple CRC calculations.
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
struct
Table
{
// Constructors are intentionally NOT marked explicit.
Table
(
const
Parameters
<
CRCType
,
CRCWidth
>
&
parameters
);
#ifdef CRCPP_USE_CPP11
Table
(
Parameters
<
CRCType
,
CRCWidth
>
&&
parameters
);
#endif
const
Parameters
<
CRCType
,
CRCWidth
>
&
GetParameters
()
const
;
const
CRCType
*
GetTable
()
const
;
CRCType
operator
[](
unsigned
char
index
)
const
;
private:
void
InitTable
();
Parameters
<
CRCType
,
CRCWidth
>
parameters
;
///< CRC parameters used to construct the table
CRCType
table
[
1
<<
CHAR_BIT
];
///< CRC lookup table
};
// The number of bits in CRCType must be at least as large as CRCWidth.
// CRCType must be an unsigned integer type or a custom type with operator
// overloads.
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
static
CRCType
Calculate
(
const
void
*
data
,
crcpp_size
size
,
const
Parameters
<
CRCType
,
CRCWidth
>
&
parameters
);
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
static
CRCType
Calculate
(
const
void
*
data
,
crcpp_size
size
,
const
Parameters
<
CRCType
,
CRCWidth
>
&
parameters
,
CRCType
crc
);
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
static
CRCType
Calculate
(
const
void
*
data
,
crcpp_size
size
,
const
Table
<
CRCType
,
CRCWidth
>
&
lookupTable
);
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
static
CRCType
Calculate
(
const
void
*
data
,
crcpp_size
size
,
const
Table
<
CRCType
,
CRCWidth
>
&
lookupTable
,
CRCType
crc
);
// Common CRCs up to 64 bits.
// Note: Check values are the computed CRCs when given an ASCII input of
// "123456789" (without null terminator)
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
static
const
Parameters
<
crcpp_uint8
,
4
>
&
CRC_4_ITU
();
static
const
Parameters
<
crcpp_uint8
,
5
>
&
CRC_5_EPC
();
static
const
Parameters
<
crcpp_uint8
,
5
>
&
CRC_5_ITU
();
static
const
Parameters
<
crcpp_uint8
,
5
>
&
CRC_5_USB
();
static
const
Parameters
<
crcpp_uint8
,
6
>
&
CRC_6_CDMA2000A
();
static
const
Parameters
<
crcpp_uint8
,
6
>
&
CRC_6_CDMA2000B
();
static
const
Parameters
<
crcpp_uint8
,
6
>
&
CRC_6_ITU
();
static
const
Parameters
<
crcpp_uint8
,
7
>
&
CRC_7
();
#endif
static
const
Parameters
<
crcpp_uint8
,
8
>
&
CRC_8
();
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
static
const
Parameters
<
crcpp_uint8
,
8
>
&
CRC_8_EBU
();
static
const
Parameters
<
crcpp_uint8
,
8
>
&
CRC_8_MAXIM
();
static
const
Parameters
<
crcpp_uint8
,
8
>
&
CRC_8_WCDMA
();
static
const
Parameters
<
crcpp_uint16
,
10
>
&
CRC_10
();
static
const
Parameters
<
crcpp_uint16
,
10
>
&
CRC_10_CDMA2000
();
static
const
Parameters
<
crcpp_uint16
,
11
>
&
CRC_11
();
static
const
Parameters
<
crcpp_uint16
,
12
>
&
CRC_12_CDMA2000
();
static
const
Parameters
<
crcpp_uint16
,
12
>
&
CRC_12_DECT
();
static
const
Parameters
<
crcpp_uint16
,
12
>
&
CRC_12_UMTS
();
static
const
Parameters
<
crcpp_uint16
,
13
>
&
CRC_13_BBC
();
static
const
Parameters
<
crcpp_uint16
,
15
>
&
CRC_15
();
static
const
Parameters
<
crcpp_uint16
,
15
>
&
CRC_15_MPT1327
();
#endif
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_ARC
();
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_BUYPASS
();
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_CCITTFALSE
();
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_CDMA2000
();
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_DECTR
();
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_DECTX
();
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_DNP
();
#endif
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_GENIBUS
();
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_KERMIT
();
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_MAXIM
();
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_MODBUS
();
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_T10DIF
();
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_USB
();
#endif
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_X25
();
static
const
Parameters
<
crcpp_uint16
,
16
>
&
CRC_16_XMODEM
();
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
static
const
Parameters
<
crcpp_uint32
,
17
>
&
CRC_17_CAN
();
static
const
Parameters
<
crcpp_uint32
,
21
>
&
CRC_21_CAN
();
static
const
Parameters
<
crcpp_uint32
,
24
>
&
CRC_24
();
static
const
Parameters
<
crcpp_uint32
,
24
>
&
CRC_24_FLEXRAYA
();
static
const
Parameters
<
crcpp_uint32
,
24
>
&
CRC_24_FLEXRAYB
();
static
const
Parameters
<
crcpp_uint32
,
30
>
&
CRC_30
();
#endif
static
const
Parameters
<
crcpp_uint32
,
32
>
&
CRC_32
();
static
const
Parameters
<
crcpp_uint32
,
32
>
&
CRC_32_BZIP2
();
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
static
const
Parameters
<
crcpp_uint32
,
32
>
&
CRC_32_C
();
#endif
static
const
Parameters
<
crcpp_uint32
,
32
>
&
CRC_32_MPEG2
();
static
const
Parameters
<
crcpp_uint32
,
32
>
&
CRC_32_POSIX
();
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
static
const
Parameters
<
crcpp_uint32
,
32
>
&
CRC_32_Q
();
static
const
Parameters
<
crcpp_uint64
,
40
>
&
CRC_40_GSM
();
static
const
Parameters
<
crcpp_uint64
,
64
>
&
CRC_64
();
#endif
#ifdef CRCPP_USE_CPP11
CRC
()
=
delete
;
CRC
(
const
CRC
&
other
)
=
delete
;
CRC
&
operator
=
(
const
CRC
&
other
)
=
delete
;
CRC
(
CRC
&&
other
)
=
delete
;
CRC
&
operator
=
(
CRC
&&
other
)
=
delete
;
#endif
private:
#ifndef CRCPP_USE_CPP11
CRC
();
CRC
(
const
CRC
&
other
);
CRC
&
operator
=
(
const
CRC
&
other
);
#endif
template
<
typename
IntegerType
>
static
IntegerType
Reflect
(
IntegerType
value
,
crcpp_uint16
numBits
);
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
static
CRCType
Finalize
(
CRCType
remainder
,
CRCType
finalXOR
,
bool
reflectOutput
);
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
static
CRCType
UndoFinalize
(
CRCType
remainder
,
CRCType
finalXOR
,
bool
reflectOutput
);
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
static
CRCType
CalculateRemainder
(
const
void
*
data
,
crcpp_size
size
,
const
Parameters
<
CRCType
,
CRCWidth
>
&
parameters
,
CRCType
remainder
);
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
static
CRCType
CalculateRemainder
(
const
void
*
data
,
crcpp_size
size
,
const
Table
<
CRCType
,
CRCWidth
>
&
lookupTable
,
CRCType
remainder
);
template
<
typename
IntegerType
>
static
crcpp_constexpr
IntegerType
BoundedConstexprValue
(
IntegerType
x
);
};
/**
@brief Returns a CRC lookup table construct using these CRC parameters.
@note This function primarily exists to allow use of the auto keyword
instead of instantiating
a table directly, since template parameters are not inferred in
constructors.
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return CRC lookup table
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRC
::
Table
<
CRCType
,
CRCWidth
>
CRC
::
Parameters
<
CRCType
,
CRCWidth
>::
MakeTable
()
const
{
// This should take advantage of RVO and optimize out the copy.
return
CRC
::
Table
<
CRCType
,
CRCWidth
>
(
*
this
);
}
/**
@brief Constructs a CRC table from a set of CRC parameters
@param[in] parameters CRC parameters
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRC
::
Table
<
CRCType
,
CRCWidth
>::
Table
(
const
Parameters
<
CRCType
,
CRCWidth
>
&
parameters
)
:
parameters
(
parameters
)
{
InitTable
();
}
#ifdef CRCPP_USE_CPP11
/**
@brief Constructs a CRC table from a set of CRC parameters
@param[in] parameters CRC parameters
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRC
::
Table
<
CRCType
,
CRCWidth
>::
Table
(
Parameters
<
CRCType
,
CRCWidth
>
&&
parameters
)
:
parameters
(
::
std
::
move
(
parameters
))
{
InitTable
();
}
#endif
/**
@brief Gets the CRC parameters used to construct the CRC table
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return CRC parameters
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
const
CRC
::
Parameters
<
CRCType
,
CRCWidth
>
&
CRC
::
Table
<
CRCType
,
CRCWidth
>::
GetParameters
()
const
{
return
parameters
;
}
/**
@brief Gets the CRC table
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return CRC table
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
const
CRCType
*
CRC
::
Table
<
CRCType
,
CRCWidth
>::
GetTable
()
const
{
return
table
;
}
/**
@brief Gets an entry in the CRC table
@param[in] index Index into the CRC table
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return CRC table entry
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRCType
CRC
::
Table
<
CRCType
,
CRCWidth
>::
operator
[](
unsigned
char
index
)
const
{
return
table
[
index
];
}
/**
@brief Initializes a CRC table.
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
void
CRC
::
Table
<
CRCType
,
CRCWidth
>::
InitTable
()
{
// For masking off the bits for the CRC (in the event that the number of bits
// in CRCType is larger than CRCWidth)
static
crcpp_constexpr
CRCType
BIT_MASK
(
(
CRCType
(
1
)
<<
(
CRCWidth
-
CRCType
(
1
)))
|
((
CRCType
(
1
)
<<
(
CRCWidth
-
CRCType
(
1
)))
-
CRCType
(
1
)));
static
crcpp_constexpr
CRCType
SHIFT
(
CRC
::
BoundedConstexprValue
(
CHAR_BIT
-
CRCWidth
));
CRCType
crc
;
unsigned
char
byte
=
0
;
// Loop over each dividend (each possible number storable in an unsigned char)
do
{
crc
=
CRC
::
CalculateRemainder
<
CRCType
,
CRCWidth
>
(
&
byte
,
sizeof
(
byte
),
parameters
,
CRCType
(
0
));
// This mask might not be necessary; all unit tests pass with this line
// commented out,
// but that might just be a coincidence based on the CRC parameters used for
// testing.
// In any case, this is harmless to leave in and only adds a single machine
// instruction per loop iteration.
crc
&=
BIT_MASK
;
if
(
!
parameters
.
reflectInput
&&
CRCWidth
<
CHAR_BIT
)
{
// Undo the special operation at the end of the CalculateRemainder()
// function for non-reflected CRCs < CHAR_BIT.
crc
<<=
SHIFT
;
}
table
[
byte
]
=
crc
;
}
while
(
++
byte
);
}
/**
@brief Computes a CRC.
@param[in] data Data over which CRC will be computed
@param[in] size Size of the data
@param[in] parameters CRC parameters
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return CRC
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRCType
CRC
::
Calculate
(
const
void
*
data
,
crcpp_size
size
,
const
Parameters
<
CRCType
,
CRCWidth
>
&
parameters
)
{
CRCType
remainder
=
CalculateRemainder
(
data
,
size
,
parameters
,
parameters
.
initialValue
);
// No need to mask the remainder here; the mask will be applied in the
// Finalize() function.
return
Finalize
<
CRCType
,
CRCWidth
>
(
remainder
,
parameters
.
finalXOR
,
parameters
.
reflectInput
!=
parameters
.
reflectOutput
);
}
/**
@brief Appends additional data to a previous CRC calculation.
@note This function can be used to compute multi-part CRCs.
@param[in] data Data over which CRC will be computed
@param[in] size Size of the data
@param[in] parameters CRC parameters
@param[in] crc CRC from a previous calculation
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return CRC
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRCType
CRC
::
Calculate
(
const
void
*
data
,
crcpp_size
size
,
const
Parameters
<
CRCType
,
CRCWidth
>
&
parameters
,
CRCType
crc
)
{
CRCType
remainder
=
UndoFinalize
<
CRCType
,
CRCWidth
>
(
crc
,
parameters
.
finalXOR
,
parameters
.
reflectInput
!=
parameters
.
reflectOutput
);
remainder
=
CalculateRemainder
(
data
,
size
,
parameters
,
remainder
);
// No need to mask the remainder here; the mask will be applied in the
// Finalize() function.
return
Finalize
<
CRCType
,
CRCWidth
>
(
remainder
,
parameters
.
finalXOR
,
parameters
.
reflectInput
!=
parameters
.
reflectOutput
);
}
/**
@brief Computes a CRC via a lookup table.
@param[in] data Data over which CRC will be computed
@param[in] size Size of the data
@param[in] lookupTable CRC lookup table
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return CRC
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRCType
CRC
::
Calculate
(
const
void
*
data
,
crcpp_size
size
,
const
Table
<
CRCType
,
CRCWidth
>
&
lookupTable
)
{
const
Parameters
<
CRCType
,
CRCWidth
>
&
parameters
=
lookupTable
.
GetParameters
();
CRCType
remainder
=
CalculateRemainder
(
data
,
size
,
lookupTable
,
parameters
.
initialValue
);
// No need to mask the remainder here; the mask will be applied in the
// Finalize() function.
return
Finalize
<
CRCType
,
CRCWidth
>
(
remainder
,
parameters
.
finalXOR
,
parameters
.
reflectInput
!=
parameters
.
reflectOutput
);
}
/**
@brief Appends additional data to a previous CRC calculation using a lookup
table.
@note This function can be used to compute multi-part CRCs.
@param[in] data Data over which CRC will be computed
@param[in] size Size of the data
@param[in] lookupTable CRC lookup table
@param[in] crc CRC from a previous calculation
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return CRC
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRCType
CRC
::
Calculate
(
const
void
*
data
,
crcpp_size
size
,
const
Table
<
CRCType
,
CRCWidth
>
&
lookupTable
,
CRCType
crc
)
{
const
Parameters
<
CRCType
,
CRCWidth
>
&
parameters
=
lookupTable
.
GetParameters
();
CRCType
remainder
=
UndoFinalize
<
CRCType
,
CRCWidth
>
(
crc
,
parameters
.
finalXOR
,
parameters
.
reflectInput
!=
parameters
.
reflectOutput
);
remainder
=
CalculateRemainder
(
data
,
size
,
lookupTable
,
remainder
);
// No need to mask the remainder here; the mask will be applied in the
// Finalize() function.
return
Finalize
<
CRCType
,
CRCWidth
>
(
remainder
,
parameters
.
finalXOR
,
parameters
.
reflectInput
!=
parameters
.
reflectOutput
);
}
/**
@brief Reflects (i.e. reverses the bits within) an integer value.
@param[in] value Value to reflect
@param[in] numBits Number of bits in the integer which will be reflected
@tparam IntegerType Integer type of the value being reflected
@return Reflected value
*/
template
<
typename
IntegerType
>
inline
IntegerType
CRC
::
Reflect
(
IntegerType
value
,
crcpp_uint16
numBits
)
{
IntegerType
reversedValue
(
0
);
for
(
crcpp_uint16
i
=
0
;
i
<
numBits
;
++
i
)
{
reversedValue
=
(
reversedValue
<<
1
)
|
(
value
&
1
);
value
>>=
1
;
}
return
reversedValue
;
}
/**
@brief Computes the final reflection and XOR of a CRC remainder.
@param[in] remainder CRC remainder to reflect and XOR
@param[in] finalXOR Final value to XOR with the remainder
@param[in] reflectOutput true to reflect each byte of the remainder before
the XOR
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return Final CRC
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRCType
CRC
::
Finalize
(
CRCType
remainder
,
CRCType
finalXOR
,
bool
reflectOutput
)
{
// For masking off the bits for the CRC (in the event that the number of bits
// in CRCType is larger than CRCWidth)
static
crcpp_constexpr
CRCType
BIT_MASK
=
(
CRCType
(
1
)
<<
(
CRCWidth
-
CRCType
(
1
)))
|
((
CRCType
(
1
)
<<
(
CRCWidth
-
CRCType
(
1
)))
-
CRCType
(
1
));
if
(
reflectOutput
)
{
remainder
=
Reflect
(
remainder
,
CRCWidth
);
}
return
(
remainder
^
finalXOR
)
&
BIT_MASK
;
}
/**
@brief Undoes the process of computing the final reflection and XOR of a CRC
remainder.
@note This function allows for computation of multi-part CRCs
@note Calling UndoFinalize() followed by Finalize() (or vice versa) will
always return the original remainder value:
CRCType x = ...;
CRCType y = Finalize(x, finalXOR, reflectOutput);
CRCType z = UndoFinalize(y, finalXOR, reflectOutput);
assert(x == z);
@param[in] crc Reflected and XORed CRC
@param[in] finalXOR Final value XORed with the remainder
@param[in] reflectOutput true if the remainder is to be reflected
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return Un-finalized CRC remainder
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRCType
CRC
::
UndoFinalize
(
CRCType
crc
,
CRCType
finalXOR
,
bool
reflectOutput
)
{
// For masking off the bits for the CRC (in the event that the number of bits
// in CRCType is larger than CRCWidth)
static
crcpp_constexpr
CRCType
BIT_MASK
=
(
CRCType
(
1
)
<<
(
CRCWidth
-
CRCType
(
1
)))
|
((
CRCType
(
1
)
<<
(
CRCWidth
-
CRCType
(
1
)))
-
CRCType
(
1
));
crc
=
(
crc
&
BIT_MASK
)
^
finalXOR
;
if
(
reflectOutput
)
{
crc
=
Reflect
(
crc
,
CRCWidth
);
}
return
crc
;
}
/**
@brief Computes a CRC remainder.
@param[in] data Data over which the remainder will be computed
@param[in] size Size of the data
@param[in] parameters CRC parameters
@param[in] remainder Running CRC remainder. Can be an initial value or the
result of a previous CRC remainder calculation.
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return CRC remainder
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRCType
CRC
::
CalculateRemainder
(
const
void
*
data
,
crcpp_size
size
,
const
Parameters
<
CRCType
,
CRCWidth
>
&
parameters
,
CRCType
remainder
)
{
#ifdef CRCPP_USE_CPP11
// This static_assert is put here because this function will always be
// compiled in no matter what
// the template parameters are and whether or not a table lookup or bit-by-bit
// algorithm is used.
static_assert
(
::
std
::
numeric_limits
<
CRCType
>::
digits
>=
CRCWidth
,
"CRCType is too small to contain a CRC of width CRCWidth."
);
#else
// Catching this compile-time error is very important. Sadly, the compiler
// error will be very cryptic, but it's
// better than nothing.
enum
{
static_assert_failed_CRCType_is_too_small_to_contain_a_CRC_of_width_CRCWidth
=
1
/
(
::
std
::
numeric_limits
<
CRCType
>::
digits
>=
CRCWidth
?
1
:
0
)
};
#endif
const
unsigned
char
*
current
=
reinterpret_cast
<
const
unsigned
char
*>
(
data
);
// Slightly different implementations based on the parameters. The current
// implementations try to eliminate as much
// computation from the inner loop (looping over each bit) as possible.
if
(
parameters
.
reflectInput
)
{
CRCType
polynomial
=
CRC
::
Reflect
(
parameters
.
polynomial
,
CRCWidth
);
while
(
size
--
)
{
remainder
^=
*
current
++
;
// An optimizing compiler might choose to unroll this loop.
for
(
crcpp_size
i
=
0
;
i
<
CHAR_BIT
;
++
i
)
{
#ifdef CRCPP_BRANCHLESS
// Clever way to avoid a branch at the expense of a multiplication. This
// code is equivalent to the following:
// if (remainder & 1)
// remainder = (remainder >> 1) ^ polynomial;
// else
// remainder >>= 1;
remainder
=
(
remainder
>>
1
)
^
((
remainder
&
1
)
*
polynomial
);
#else
remainder
=
(
remainder
&
1
)
?
((
remainder
>>
1
)
^
polynomial
)
:
(
remainder
>>
1
);
#endif
}
}
}
else
if
(
CRCWidth
>=
CHAR_BIT
)
{
static
crcpp_constexpr
CRCType
CRC_WIDTH_MINUS_ONE
(
CRCWidth
-
CRCType
(
1
));
#ifndef CRCPP_BRANCHLESS
static
crcpp_constexpr
CRCType
CRC_HIGHEST_BIT_MASK
(
CRCType
(
1
)
<<
CRC_WIDTH_MINUS_ONE
);
#endif
static
crcpp_constexpr
CRCType
SHIFT
(
BoundedConstexprValue
(
CRCWidth
-
CHAR_BIT
));
while
(
size
--
)
{
remainder
^=
(
static_cast
<
CRCType
>
(
*
current
++
)
<<
SHIFT
);
// An optimizing compiler might choose to unroll this loop.
for
(
crcpp_size
i
=
0
;
i
<
CHAR_BIT
;
++
i
)
{
#ifdef CRCPP_BRANCHLESS
// Clever way to avoid a branch at the expense of a multiplication. This
// code is equivalent to the following:
// if (remainder & CRC_HIGHEST_BIT_MASK)
// remainder = (remainder << 1) ^ parameters.polynomial;
// else
// remainder <<= 1;
remainder
=
(
remainder
<<
1
)
^
(((
remainder
>>
CRC_WIDTH_MINUS_ONE
)
&
1
)
*
parameters
.
polynomial
);
#else
remainder
=
(
remainder
&
CRC_HIGHEST_BIT_MASK
)
?
((
remainder
<<
1
)
^
parameters
.
polynomial
)
:
(
remainder
<<
1
);
#endif
}
}
}
else
{
static
crcpp_constexpr
CRCType
CHAR_BIT_MINUS_ONE
(
CHAR_BIT
-
1
);
#ifndef CRCPP_BRANCHLESS
static
crcpp_constexpr
CRCType
CHAR_BIT_HIGHEST_BIT_MASK
(
CRCType
(
1
)
<<
CHAR_BIT_MINUS_ONE
);
#endif
static
crcpp_constexpr
CRCType
SHIFT
(
BoundedConstexprValue
(
CHAR_BIT
-
CRCWidth
));
CRCType
polynomial
=
parameters
.
polynomial
<<
SHIFT
;
remainder
<<=
SHIFT
;
while
(
size
--
)
{
remainder
^=
*
current
++
;
// An optimizing compiler might choose to unroll this loop.
for
(
crcpp_size
i
=
0
;
i
<
CHAR_BIT
;
++
i
)
{
#ifdef CRCPP_BRANCHLESS
// Clever way to avoid a branch at the expense of a multiplication. This
// code is equivalent to the following:
// if (remainder & CHAR_BIT_HIGHEST_BIT_MASK)
// remainder = (remainder << 1) ^ polynomial;
// else
// remainder <<= 1;
remainder
=
(
remainder
<<
1
)
^
(((
remainder
>>
CHAR_BIT_MINUS_ONE
)
&
1
)
*
polynomial
);
#else
remainder
=
(
remainder
&
CHAR_BIT_HIGHEST_BIT_MASK
)
?
((
remainder
<<
1
)
^
polynomial
)
:
(
remainder
<<
1
);
#endif
}
}
remainder
>>=
SHIFT
;
}
return
remainder
;
}
/**
@brief Computes a CRC remainder using lookup table.
@param[in] data Data over which the remainder will be computed
@param[in] size Size of the data
@param[in] lookupTable CRC lookup table
@param[in] remainder Running CRC remainder. Can be an initial value or the
result of a previous CRC remainder calculation.
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return CRC remainder
*/
template
<
typename
CRCType
,
crcpp_uint16
CRCWidth
>
inline
CRCType
CRC
::
CalculateRemainder
(
const
void
*
data
,
crcpp_size
size
,
const
Table
<
CRCType
,
CRCWidth
>
&
lookupTable
,
CRCType
remainder
)
{
const
unsigned
char
*
current
=
reinterpret_cast
<
const
unsigned
char
*>
(
data
);
if
(
lookupTable
.
GetParameters
().
reflectInput
)
{
while
(
size
--
)
{
#if defined(WIN32) || defined(_WIN32) || defined(WINCE)
// Disable warning about data loss when doing (remainder >> CHAR_BIT) when
// remainder is one byte long. The algorithm is still correct in this case,
// though it's possible that one additional machine instruction will be
// executed.
#pragma warning(push)
#pragma warning(disable : 4333)
#endif
remainder
=
(
remainder
>>
CHAR_BIT
)
^
lookupTable
[
static_cast
<
unsigned
char
>
(
remainder
^
*
current
++
)];
#if defined(WIN32) || defined(_WIN32) || defined(WINCE)
#pragma warning(pop)
#endif
}
}
else
if
(
CRCWidth
>=
CHAR_BIT
)
{
static
crcpp_constexpr
CRCType
SHIFT
(
BoundedConstexprValue
(
CRCWidth
-
CHAR_BIT
));
while
(
size
--
)
{
remainder
=
(
remainder
<<
CHAR_BIT
)
^
lookupTable
[
static_cast
<
unsigned
char
>
((
remainder
>>
SHIFT
)
^
*
current
++
)];
}
}
else
{
static
crcpp_constexpr
CRCType
SHIFT
(
BoundedConstexprValue
(
CHAR_BIT
-
CRCWidth
));
remainder
<<=
SHIFT
;
while
(
size
--
)
{
// Note: no need to mask here since remainder is guaranteed to fit in a
// single byte.
remainder
=
lookupTable
[
static_cast
<
unsigned
char
>
(
remainder
^
*
current
++
)];
}
remainder
>>=
SHIFT
;
}
return
remainder
;
}
/**
@brief Function to force a compile-time expression to be >= 0.
@note This function is used to avoid compiler warnings because all constexpr
values are evaluated
in a function even in a branch will never be executed. This also means
we don't need pragmas
to get rid of warnings, but it still can be computed at compile-time.
Win-win!
@param[in] x Compile-time expression to bound
@tparam CRCType Integer type for storing the CRC result
@tparam CRCWidth Number of bits in the CRC
@return Non-negative compile-time expression
*/
template
<
typename
IntegerType
>
inline
crcpp_constexpr
IntegerType
CRC
::
BoundedConstexprValue
(
IntegerType
x
)
{
return
(
x
<
IntegerType
(
0
))
?
IntegerType
(
0
)
:
x
;
}
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-4 ITU.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-4 ITU has the following parameters and check value:
- polynomial = 0x3
- initial value = 0x0
- final XOR = 0x0
- reflect input = true
- reflect output = true
- check value = 0x7
@return CRC-4 ITU parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
4
>
&
CRC
::
CRC_4_ITU
()
{
static
const
Parameters
<
crcpp_uint8
,
4
>
parameters
=
{
0x3
,
0x0
,
0x0
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-5 EPC.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-5 EPC has the following parameters and check value:
- polynomial = 0x09
- initial value = 0x09
- final XOR = 0x00
- reflect input = false
- reflect output = false
- check value = 0x00
@return CRC-5 EPC parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
5
>
&
CRC
::
CRC_5_EPC
()
{
static
const
Parameters
<
crcpp_uint8
,
5
>
parameters
=
{
0x09
,
0x09
,
0x00
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-5 ITU.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-5 ITU has the following parameters and check value:
- polynomial = 0x15
- initial value = 0x00
- final XOR = 0x00
- reflect input = true
- reflect output = true
- check value = 0x07
@return CRC-5 ITU parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
5
>
&
CRC
::
CRC_5_ITU
()
{
static
const
Parameters
<
crcpp_uint8
,
5
>
parameters
=
{
0x15
,
0x00
,
0x00
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-5 USB.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-5 USB has the following parameters and check value:
- polynomial = 0x05
- initial value = 0x1F
- final XOR = 0x1F
- reflect input = true
- reflect output = true
- check value = 0x19
@return CRC-5 USB parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
5
>
&
CRC
::
CRC_5_USB
()
{
static
const
Parameters
<
crcpp_uint8
,
5
>
parameters
=
{
0x05
,
0x1F
,
0x1F
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-6 CDMA2000-A.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-6 CDMA2000-A has the following parameters and check value:
- polynomial = 0x27
- initial value = 0x3F
- final XOR = 0x00
- reflect input = false
- reflect output = false
- check value = 0x0D
@return CRC-6 CDMA2000-A parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
6
>
&
CRC
::
CRC_6_CDMA2000A
()
{
static
const
Parameters
<
crcpp_uint8
,
6
>
parameters
=
{
0x27
,
0x3F
,
0x00
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-6 CDMA2000-B.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-6 CDMA2000-A has the following parameters and check value:
- polynomial = 0x07
- initial value = 0x3F
- final XOR = 0x00
- reflect input = false
- reflect output = false
- check value = 0x3B
@return CRC-6 CDMA2000-B parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
6
>
&
CRC
::
CRC_6_CDMA2000B
()
{
static
const
Parameters
<
crcpp_uint8
,
6
>
parameters
=
{
0x07
,
0x3F
,
0x00
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-6 ITU.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-6 ITU has the following parameters and check value:
- polynomial = 0x03
- initial value = 0x00
- final XOR = 0x00
- reflect input = true
- reflect output = true
- check value = 0x06
@return CRC-6 ITU parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
6
>
&
CRC
::
CRC_6_ITU
()
{
static
const
Parameters
<
crcpp_uint8
,
6
>
parameters
=
{
0x03
,
0x00
,
0x00
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-7 JEDEC.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-7 JEDEC has the following parameters and check value:
- polynomial = 0x09
- initial value = 0x00
- final XOR = 0x00
- reflect input = false
- reflect output = false
- check value = 0x75
@return CRC-7 JEDEC parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
7
>
&
CRC
::
CRC_7
()
{
static
const
Parameters
<
crcpp_uint8
,
7
>
parameters
=
{
0x09
,
0x00
,
0x00
,
false
,
false
};
return
parameters
;
}
#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-8 SMBus.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-8 SMBus has the following parameters and check value:
- polynomial = 0x07
- initial value = 0x00
- final XOR = 0x00
- reflect input = false
- reflect output = false
- check value = 0xF4
@return CRC-8 SMBus parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
8
>
&
CRC
::
CRC_8
()
{
static
const
Parameters
<
crcpp_uint8
,
8
>
parameters
=
{
0x07
,
0x00
,
0x00
,
false
,
false
};
return
parameters
;
}
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-8 EBU (aka CRC-8 AES).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-8 EBU has the following parameters and check value:
- polynomial = 0x1D
- initial value = 0xFF
- final XOR = 0x00
- reflect input = true
- reflect output = true
- check value = 0x97
@return CRC-8 EBU parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
8
>
&
CRC
::
CRC_8_EBU
()
{
static
const
Parameters
<
crcpp_uint8
,
8
>
parameters
=
{
0x1D
,
0xFF
,
0x00
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-8 MAXIM (aka CRC-8 DOW-CRC).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-8 MAXIM has the following parameters and check value:
- polynomial = 0x31
- initial value = 0x00
- final XOR = 0x00
- reflect input = true
- reflect output = true
- check value = 0xA1
@return CRC-8 MAXIM parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
8
>
&
CRC
::
CRC_8_MAXIM
()
{
static
const
Parameters
<
crcpp_uint8
,
8
>
parameters
=
{
0x31
,
0x00
,
0x00
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-8 WCDMA.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-8 WCDMA has the following parameters and check value:
- polynomial = 0x9B
- initial value = 0x00
- final XOR = 0x00
- reflect input = true
- reflect output = true
- check value = 0x25
@return CRC-8 WCDMA parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint8
,
8
>
&
CRC
::
CRC_8_WCDMA
()
{
static
const
Parameters
<
crcpp_uint8
,
8
>
parameters
=
{
0x9B
,
0x00
,
0x00
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-10 ITU.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-10 ITU has the following parameters and check value:
- polynomial = 0x233
- initial value = 0x000
- final XOR = 0x000
- reflect input = false
- reflect output = false
- check value = 0x199
@return CRC-10 ITU parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
10
>
&
CRC
::
CRC_10
()
{
static
const
Parameters
<
crcpp_uint16
,
10
>
parameters
=
{
0x233
,
0x000
,
0x000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-10 CDMA2000.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-10 CDMA2000 has the following parameters and check value:
- polynomial = 0x3D9
- initial value = 0x3FF
- final XOR = 0x000
- reflect input = false
- reflect output = false
- check value = 0x233
@return CRC-10 CDMA2000 parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
10
>
&
CRC
::
CRC_10_CDMA2000
()
{
static
const
Parameters
<
crcpp_uint16
,
10
>
parameters
=
{
0x3D9
,
0x3FF
,
0x000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-11 FlexRay.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-11 FlexRay has the following parameters and check value:
- polynomial = 0x385
- initial value = 0x01A
- final XOR = 0x000
- reflect input = false
- reflect output = false
- check value = 0x5A3
@return CRC-11 FlexRay parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
11
>
&
CRC
::
CRC_11
()
{
static
const
Parameters
<
crcpp_uint16
,
11
>
parameters
=
{
0x385
,
0x01A
,
0x000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-12 CDMA2000.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-12 CDMA2000 has the following parameters and check value:
- polynomial = 0xF13
- initial value = 0xFFF
- final XOR = 0x000
- reflect input = false
- reflect output = false
- check value = 0xD4D
@return CRC-12 CDMA2000 parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
12
>
&
CRC
::
CRC_12_CDMA2000
()
{
static
const
Parameters
<
crcpp_uint16
,
12
>
parameters
=
{
0xF13
,
0xFFF
,
0x000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-12 DECT (aka CRC-12 X-CRC).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-12 DECT has the following parameters and check value:
- polynomial = 0x80F
- initial value = 0x000
- final XOR = 0x000
- reflect input = false
- reflect output = false
- check value = 0xF5B
@return CRC-12 DECT parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
12
>
&
CRC
::
CRC_12_DECT
()
{
static
const
Parameters
<
crcpp_uint16
,
12
>
parameters
=
{
0x80F
,
0x000
,
0x000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-12 UMTS (aka CRC-12 3GPP).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-12 UMTS has the following parameters and check value:
- polynomial = 0x80F
- initial value = 0x000
- final XOR = 0x000
- reflect input = false
- reflect output = true
- check value = 0xDAF
@return CRC-12 UMTS parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
12
>
&
CRC
::
CRC_12_UMTS
()
{
static
const
Parameters
<
crcpp_uint16
,
12
>
parameters
=
{
0x80F
,
0x000
,
0x000
,
false
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-13 BBC.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-13 BBC has the following parameters and check value:
- polynomial = 0x1CF5
- initial value = 0x0000
- final XOR = 0x0000
- reflect input = false
- reflect output = false
- check value = 0x04FA
@return CRC-13 BBC parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
13
>
&
CRC
::
CRC_13_BBC
()
{
static
const
Parameters
<
crcpp_uint16
,
13
>
parameters
=
{
0x1CF5
,
0x0000
,
0x0000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-15 CAN.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-15 CAN has the following parameters and check value:
- polynomial = 0x4599
- initial value = 0x0000
- final XOR = 0x0000
- reflect input = false
- reflect output = false
- check value = 0x059E
@return CRC-15 CAN parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
15
>
&
CRC
::
CRC_15
()
{
static
const
Parameters
<
crcpp_uint16
,
15
>
parameters
=
{
0x4599
,
0x0000
,
0x0000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-15 MPT1327.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-15 MPT1327 has the following parameters and check value:
- polynomial = 0x6815
- initial value = 0x0000
- final XOR = 0x0001
- reflect input = false
- reflect output = false
- check value = 0x2566
@return CRC-15 MPT1327 parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
15
>
&
CRC
::
CRC_15_MPT1327
()
{
static
const
Parameters
<
crcpp_uint16
,
15
>
parameters
=
{
0x6815
,
0x0000
,
0x0001
,
false
,
false
};
return
parameters
;
}
#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-16 ARC (aka CRC-16 IBM, CRC-16
LHA).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 ARC has the following parameters and check value:
- polynomial = 0x8005
- initial value = 0x0000
- final XOR = 0x0000
- reflect input = true
- reflect output = true
- check value = 0xBB3D
@return CRC-16 ARC parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_ARC
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x8005
,
0x0000
,
0x0000
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-16 BUYPASS (aka CRC-16 VERIFONE,
CRC-16 UMTS).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 BUYPASS has the following parameters and check value:
- polynomial = 0x8005
- initial value = 0x0000
- final XOR = 0x0000
- reflect input = false
- reflect output = false
- check value = 0xFEE8
@return CRC-16 BUYPASS parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_BUYPASS
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x8005
,
0x0000
,
0x0000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-16 CCITT FALSE.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 CCITT FALSE has the following parameters and check value:
- polynomial = 0x1021
- initial value = 0xFFFF
- final XOR = 0x0000
- reflect input = false
- reflect output = false
- check value = 0x29B1
@return CRC-16 CCITT FALSE parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_CCITTFALSE
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x1021
,
0xFFFF
,
0x0000
,
false
,
false
};
return
parameters
;
}
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-16 CDMA2000.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 CDMA2000 has the following parameters and check value:
- polynomial = 0xC867
- initial value = 0xFFFF
- final XOR = 0x0000
- reflect input = false
- reflect output = false
- check value = 0x4C06
@return CRC-16 CDMA2000 parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_CDMA2000
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0xC867
,
0xFFFF
,
0x0000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-16 DECT-R (aka CRC-16 R-CRC).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 DECT-R has the following parameters and check value:
- polynomial = 0x0589
- initial value = 0x0000
- final XOR = 0x0001
- reflect input = false
- reflect output = false
- check value = 0x007E
@return CRC-16 DECT-R parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_DECTR
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x0589
,
0x0000
,
0x0001
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-16 DECT-X (aka CRC-16 X-CRC).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 DECT-X has the following parameters and check value:
- polynomial = 0x0589
- initial value = 0x0000
- final XOR = 0x0000
- reflect input = false
- reflect output = false
- check value = 0x007F
@return CRC-16 DECT-X parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_DECTX
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x0589
,
0x0000
,
0x0000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-16 DNP.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 DNP has the following parameters and check value:
- polynomial = 0x3D65
- initial value = 0x0000
- final XOR = 0xFFFF
- reflect input = true
- reflect output = true
- check value = 0xEA82
@return CRC-16 DNP parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_DNP
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x3D65
,
0x0000
,
0xFFFF
,
true
,
true
};
return
parameters
;
}
#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-16 GENIBUS (aka CRC-16 EPC,
CRC-16 I-CODE, CRC-16 DARC).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 GENIBUS has the following parameters and check value:
- polynomial = 0x1021
- initial value = 0xFFFF
- final XOR = 0xFFFF
- reflect input = false
- reflect output = false
- check value = 0xD64E
@return CRC-16 GENIBUS parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_GENIBUS
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x1021
,
0xFFFF
,
0xFFFF
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-16 KERMIT (aka CRC-16 CCITT,
CRC-16 CCITT-TRUE).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 KERMIT has the following parameters and check value:
- polynomial = 0x1021
- initial value = 0x0000
- final XOR = 0x0000
- reflect input = true
- reflect output = true
- check value = 0x2189
@return CRC-16 KERMIT parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_KERMIT
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x1021
,
0x0000
,
0x0000
,
true
,
true
};
return
parameters
;
}
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-16 MAXIM.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 MAXIM has the following parameters and check value:
- polynomial = 0x8005
- initial value = 0x0000
- final XOR = 0xFFFF
- reflect input = true
- reflect output = true
- check value = 0x44C2
@return CRC-16 MAXIM parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_MAXIM
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x8005
,
0x0000
,
0xFFFF
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-16 MODBUS.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 MODBUS has the following parameters and check value:
- polynomial = 0x8005
- initial value = 0xFFFF
- final XOR = 0x0000
- reflect input = true
- reflect output = true
- check value = 0x4B37
@return CRC-16 MODBUS parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_MODBUS
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x8005
,
0xFFFF
,
0x0000
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-16 T10-DIF.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 T10-DIF has the following parameters and check value:
- polynomial = 0x8BB7
- initial value = 0x0000
- final XOR = 0x0000
- reflect input = false
- reflect output = false
- check value = 0xD0DB
@return CRC-16 T10-DIF parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_T10DIF
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x8BB7
,
0x0000
,
0x0000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-16 USB.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 USB has the following parameters and check value:
- polynomial = 0x8005
- initial value = 0xFFFF
- final XOR = 0xFFFF
- reflect input = true
- reflect output = true
- check value = 0xB4C8
@return CRC-16 USB parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_USB
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x8005
,
0xFFFF
,
0xFFFF
,
true
,
true
};
return
parameters
;
}
#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-16 X-25 (aka CRC-16 IBM-SDLC,
CRC-16 ISO-HDLC, CRC-16 B).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 X-25 has the following parameters and check value:
- polynomial = 0x1021
- initial value = 0xFFFF
- final XOR = 0xFFFF
- reflect input = true
- reflect output = true
- check value = 0x906E
@return CRC-16 X-25 parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_X25
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x1021
,
0xFFFF
,
0xFFFF
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-16 XMODEM (aka CRC-16 ZMODEM,
CRC-16 ACORN, CRC-16 LTE).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-16 XMODEM has the following parameters and check value:
- polynomial = 0x1021
- initial value = 0x0000
- final XOR = 0x0000
- reflect input = false
- reflect output = false
- check value = 0x31C3
@return CRC-16 XMODEM parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint16
,
16
>
&
CRC
::
CRC_16_XMODEM
()
{
static
const
Parameters
<
crcpp_uint16
,
16
>
parameters
=
{
0x1021
,
0x0000
,
0x0000
,
false
,
false
};
return
parameters
;
}
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-17 CAN.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-17 CAN has the following parameters and check value:
- polynomial = 0x1685B
- initial value = 0x00000
- final XOR = 0x00000
- reflect input = false
- reflect output = false
- check value = 0x04F03
@return CRC-17 CAN parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
17
>
&
CRC
::
CRC_17_CAN
()
{
static
const
Parameters
<
crcpp_uint32
,
17
>
parameters
=
{
0x1685B
,
0x00000
,
0x00000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-21 CAN.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-21 CAN has the following parameters and check value:
- polynomial = 0x102899
- initial value = 0x000000
- final XOR = 0x000000
- reflect input = false
- reflect output = false
- check value = 0x0ED841
@return CRC-21 CAN parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
21
>
&
CRC
::
CRC_21_CAN
()
{
static
const
Parameters
<
crcpp_uint32
,
21
>
parameters
=
{
0x102899
,
0x000000
,
0x000000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-24 OPENPGP.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-24 OPENPGP has the following parameters and check value:
- polynomial = 0x864CFB
- initial value = 0xB704CE
- final XOR = 0x000000
- reflect input = false
- reflect output = false
- check value = 0x21CF02
@return CRC-24 OPENPGP parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
24
>
&
CRC
::
CRC_24
()
{
static
const
Parameters
<
crcpp_uint32
,
24
>
parameters
=
{
0x864CFB
,
0xB704CE
,
0x000000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-24 FlexRay-A.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-24 FlexRay-A has the following parameters and check value:
- polynomial = 0x5D6DCB
- initial value = 0xFEDCBA
- final XOR = 0x000000
- reflect input = false
- reflect output = false
- check value = 0x7979BD
@return CRC-24 FlexRay-A parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
24
>
&
CRC
::
CRC_24_FLEXRAYA
()
{
static
const
Parameters
<
crcpp_uint32
,
24
>
parameters
=
{
0x5D6DCB
,
0xFEDCBA
,
0x000000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-24 FlexRay-B.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-24 FlexRay-B has the following parameters and check value:
- polynomial = 0x5D6DCB
- initial value = 0xABCDEF
- final XOR = 0x000000
- reflect input = false
- reflect output = false
- check value = 0x1F23B8
@return CRC-24 FlexRay-B parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
24
>
&
CRC
::
CRC_24_FLEXRAYB
()
{
static
const
Parameters
<
crcpp_uint32
,
24
>
parameters
=
{
0x5D6DCB
,
0xABCDEF
,
0x000000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-30 CDMA.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-30 CDMA has the following parameters and check value:
- polynomial = 0x2030B9C7
- initial value = 0x3FFFFFFF
- final XOR = 0x00000000
- reflect input = false
- reflect output = false
- check value = 0x3B3CB540
@return CRC-30 CDMA parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
30
>
&
CRC
::
CRC_30
()
{
static
const
Parameters
<
crcpp_uint32
,
30
>
parameters
=
{
0x2030B9C7
,
0x3FFFFFFF
,
0x00000000
,
false
,
false
};
return
parameters
;
}
#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-32 (aka CRC-32 ADCCP, CRC-32
PKZip).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-32 has the following parameters and check value:
- polynomial = 0x04C11DB7
- initial value = 0xFFFFFFFF
- final XOR = 0xFFFFFFFF
- reflect input = true
- reflect output = true
- check value = 0xCBF43926
@return CRC-32 parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
32
>
&
CRC
::
CRC_32
()
{
static
const
Parameters
<
crcpp_uint32
,
32
>
parameters
=
{
0x04C11DB7
,
0xFFFFFFFF
,
0xFFFFFFFF
,
true
,
true
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-32 BZIP2 (aka CRC-32 AAL5, CRC-32
DECT-B, CRC-32 B-CRC).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-32 BZIP2 has the following parameters and check value:
- polynomial = 0x04C11DB7
- initial value = 0xFFFFFFFF
- final XOR = 0xFFFFFFFF
- reflect input = false
- reflect output = false
- check value = 0xFC891918
@return CRC-32 BZIP2 parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
32
>
&
CRC
::
CRC_32_BZIP2
()
{
static
const
Parameters
<
crcpp_uint32
,
32
>
parameters
=
{
0x04C11DB7
,
0xFFFFFFFF
,
0xFFFFFFFF
,
false
,
false
};
return
parameters
;
}
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-32 C (aka CRC-32 ISCSI, CRC-32
Castagnoli, CRC-32 Interlaken).
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-32 C has the following parameters and check value:
- polynomial = 0x1EDC6F41
- initial value = 0xFFFFFFFF
- final XOR = 0xFFFFFFFF
- reflect input = true
- reflect output = true
- check value = 0xE3069283
@return CRC-32 C parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
32
>
&
CRC
::
CRC_32_C
()
{
static
const
Parameters
<
crcpp_uint32
,
32
>
parameters
=
{
0x1EDC6F41
,
0xFFFFFFFF
,
0xFFFFFFFF
,
true
,
true
};
return
parameters
;
}
#endif
/**
@brief Returns a set of parameters for CRC-32 MPEG-2.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-32 MPEG-2 has the following parameters and check value:
- polynomial = 0x04C11DB7
- initial value = 0xFFFFFFFF
- final XOR = 0x00000000
- reflect input = false
- reflect output = false
- check value = 0x0376E6E7
@return CRC-32 MPEG-2 parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
32
>
&
CRC
::
CRC_32_MPEG2
()
{
static
const
Parameters
<
crcpp_uint32
,
32
>
parameters
=
{
0x04C11DB7
,
0xFFFFFFFF
,
0x00000000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-32 POSIX.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-32 POSIX has the following parameters and check value:
- polynomial = 0x04C11DB7
- initial value = 0x00000000
- final XOR = 0xFFFFFFFF
- reflect input = false
- reflect output = false
- check value = 0x765E7680
@return CRC-32 POSIX parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
32
>
&
CRC
::
CRC_32_POSIX
()
{
static
const
Parameters
<
crcpp_uint32
,
32
>
parameters
=
{
0x04C11DB7
,
0x00000000
,
0xFFFFFFFF
,
false
,
false
};
return
parameters
;
}
#ifdef CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
/**
@brief Returns a set of parameters for CRC-32 Q.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-32 Q has the following parameters and check value:
- polynomial = 0x814141AB
- initial value = 0x00000000
- final XOR = 0x00000000
- reflect input = false
- reflect output = false
- check value = 0x3010BF7F
@return CRC-32 Q parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint32
,
32
>
&
CRC
::
CRC_32_Q
()
{
static
const
Parameters
<
crcpp_uint32
,
32
>
parameters
=
{
0x814141AB
,
0x00000000
,
0x00000000
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-40 GSM.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-40 GSM has the following parameters and check value:
- polynomial = 0x0004820009
- initial value = 0x0000000000
- final XOR = 0xFFFFFFFFFF
- reflect input = false
- reflect output = false
- check value = 0xD4164FC646
@return CRC-40 GSM parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint64
,
40
>
&
CRC
::
CRC_40_GSM
()
{
static
const
Parameters
<
crcpp_uint64
,
40
>
parameters
=
{
0x0004820009
,
0x0000000000
,
0xFFFFFFFFFF
,
false
,
false
};
return
parameters
;
}
/**
@brief Returns a set of parameters for CRC-64 ECMA.
@note The parameters are static and are delayed-constructed to reduce memory
footprint.
@note CRC-64 ECMA has the following parameters and check value:
- polynomial = 0x42F0E1EBA9EA3693
- initial value = 0x0000000000000000
- final XOR = 0x0000000000000000
- reflect input = false
- reflect output = false
- check value = 0x6C40DF5F0B497347
@return CRC-64 ECMA parameters
*/
inline
const
CRC
::
Parameters
<
crcpp_uint64
,
64
>
&
CRC
::
CRC_64
()
{
static
const
Parameters
<
crcpp_uint64
,
64
>
parameters
=
{
0x42F0E1EBA9EA3693
,
0x0000000000000000
,
0x0000000000000000
,
false
,
false
};
return
parameters
;
}
#endif // CRCPP_INCLUDE_ESOTERIC_CRC_DEFINITIONS
#ifdef CRCPP_USE_NAMESPACE
}
#endif
#endif // CRCPP_CRC_H_
paddle/fluid/recordio/header.cc
浏览文件 @
7016979c
...
@@ -26,18 +26,18 @@ Header::Header()
...
@@ -26,18 +26,18 @@ Header::Header()
Header
::
Header
(
uint32_t
num
,
uint32_t
sum
,
Compressor
c
,
uint32_t
cs
)
Header
::
Header
(
uint32_t
num
,
uint32_t
sum
,
Compressor
c
,
uint32_t
cs
)
:
num_records_
(
num
),
checksum_
(
sum
),
compressor_
(
c
),
compress_size_
(
cs
)
{}
:
num_records_
(
num
),
checksum_
(
sum
),
compressor_
(
c
),
compress_size_
(
cs
)
{}
void
Header
::
Parse
(
std
::
istream
&
iss
)
{
void
Header
::
Parse
(
Stream
*
iss
)
{
iss
.
r
ead
(
reinterpret_cast
<
char
*>
(
&
num_records_
),
sizeof
(
uint32_t
));
iss
.
R
ead
(
reinterpret_cast
<
char
*>
(
&
num_records_
),
sizeof
(
uint32_t
));
iss
.
r
ead
(
reinterpret_cast
<
char
*>
(
&
checksum_
),
sizeof
(
uint32_t
));
iss
.
R
ead
(
reinterpret_cast
<
char
*>
(
&
checksum_
),
sizeof
(
uint32_t
));
iss
.
r
ead
(
reinterpret_cast
<
char
*>
(
&
compressor_
),
sizeof
(
uint32_t
));
iss
.
R
ead
(
reinterpret_cast
<
char
*>
(
&
compressor_
),
sizeof
(
uint32_t
));
iss
.
r
ead
(
reinterpret_cast
<
char
*>
(
&
compress_size_
),
sizeof
(
uint32_t
));
iss
.
R
ead
(
reinterpret_cast
<
char
*>
(
&
compress_size_
),
sizeof
(
uint32_t
));
}
}
void
Header
::
Write
(
std
::
ostream
&
os
)
{
void
Header
::
Write
(
Stream
*
os
)
{
os
.
w
rite
(
reinterpret_cast
<
char
*>
(
&
num_records_
),
sizeof
(
uint32_t
));
os
.
W
rite
(
reinterpret_cast
<
char
*>
(
&
num_records_
),
sizeof
(
uint32_t
));
os
.
w
rite
(
reinterpret_cast
<
char
*>
(
&
checksum_
),
sizeof
(
uint32_t
));
os
.
W
rite
(
reinterpret_cast
<
char
*>
(
&
checksum_
),
sizeof
(
uint32_t
));
os
.
w
rite
(
reinterpret_cast
<
char
*>
(
&
compressor_
),
sizeof
(
uint32_t
));
os
.
W
rite
(
reinterpret_cast
<
char
*>
(
&
compressor_
),
sizeof
(
uint32_t
));
os
.
w
rite
(
reinterpret_cast
<
char
*>
(
&
compress_size_
),
sizeof
(
uint32_t
));
os
.
W
rite
(
reinterpret_cast
<
char
*>
(
&
compress_size_
),
sizeof
(
uint32_t
));
}
}
// std::ostream& operator << (std::ostream& os, Header h) {
// std::ostream& operator << (std::ostream& os, Header h) {
...
@@ -54,28 +54,8 @@ std::ostream& operator<<(std::ostream& os, Header h) {
...
@@ -54,28 +54,8 @@ std::ostream& operator<<(std::ostream& os, Header h) {
return
os
;
return
os
;
}
}
// bool operator==(Header l, Header r) {
// return num_records_ == rhs.NumRecords() &&
// checksum_ == rhs.Checksum() &&
// compressor_ == rhs.CompressType() &&
// compress_size_ == rhs.CompressSize();
// }
bool
operator
==
(
Header
l
,
Header
r
)
{
bool
operator
==
(
Header
l
,
Header
r
)
{
return
l
.
NumRecords
()
==
r
.
NumRecords
()
&&
l
.
Checksum
()
==
r
.
Checksum
()
&&
return
l
.
NumRecords
()
==
r
.
NumRecords
()
&&
l
.
Checksum
()
==
r
.
Checksum
()
&&
l
.
CompressType
()
==
r
.
CompressType
()
&&
l
.
CompressType
()
==
r
.
CompressType
()
&&
l
.
CompressSize
()
==
r
.
CompressSize
();
l
.
CompressSize
()
==
r
.
CompressSize
();
}
}
// size_t CompressData(const std::string& os, Compressor ct, char* buffer) {
// size_t compress_size = 0;
// // std::unique_ptr<char[]> buffer(new char[kDefaultMaxChunkSize]);
// // std::string compressed;
// compress_size =os.size();
// memcpy(buffer, os.c_str(), compress_size);
// return compress_size;
// }
}
// namespace recordio
}
// namespace paddle
paddle/fluid/recordio/header.h
浏览文件 @
7016979c
...
@@ -16,6 +16,8 @@
...
@@ -16,6 +16,8 @@
#include <sstream>
#include <sstream>
#include "paddle/fluid/recordio/io.h"
namespace
paddle
{
namespace
paddle
{
namespace
recordio
{
namespace
recordio
{
...
@@ -43,8 +45,8 @@ public:
...
@@ -43,8 +45,8 @@ public:
Header
();
Header
();
Header
(
uint32_t
num
,
uint32_t
sum
,
Compressor
ct
,
uint32_t
cs
);
Header
(
uint32_t
num
,
uint32_t
sum
,
Compressor
ct
,
uint32_t
cs
);
void
Write
(
std
::
ostream
&
os
);
void
Write
(
Stream
*
os
);
void
Parse
(
std
::
istream
&
iss
);
void
Parse
(
Stream
*
iss
);
uint32_t
NumRecords
()
const
{
return
num_records_
;
}
uint32_t
NumRecords
()
const
{
return
num_records_
;
}
uint32_t
Checksum
()
const
{
return
checksum_
;
}
uint32_t
Checksum
()
const
{
return
checksum_
;
}
...
...
paddle/fluid/recordio/header_test.cc
浏览文件 @
7016979c
...
@@ -22,15 +22,12 @@ using namespace paddle::recordio;
...
@@ -22,15 +22,12 @@ using namespace paddle::recordio;
TEST
(
Recordio
,
ChunkHead
)
{
TEST
(
Recordio
,
ChunkHead
)
{
Header
hdr
(
0
,
1
,
Compressor
::
kGzip
,
3
);
Header
hdr
(
0
,
1
,
Compressor
::
kGzip
,
3
);
std
::
ostringstream
oss
;
Stream
*
oss
=
Stream
::
Open
(
"/tmp/record_1"
,
"w"
)
;
hdr
.
Write
(
oss
);
hdr
.
Write
(
oss
);
std
::
istringstream
iss
(
oss
.
str
()
);
Stream
*
iss
=
Stream
::
Open
(
"/tmp/record_1"
,
"r"
);
Header
hdr2
;
Header
hdr2
;
hdr2
.
Parse
(
iss
);
hdr2
.
Parse
(
iss
);
std
::
ostringstream
oss2
;
hdr2
.
Write
(
oss2
);
EXPECT_STREQ
(
oss2
.
str
().
c_str
(),
oss
.
str
().
c_str
());
EXPECT_TRUE
(
hdr
==
hdr2
);
EXPECT_TRUE
(
hdr
==
hdr2
);
}
}
paddle/fluid/recordio/io.cc
浏览文件 @
7016979c
...
@@ -15,6 +15,8 @@
...
@@ -15,6 +15,8 @@
#include "paddle/fluid/recordio/io.h"
#include "paddle/fluid/recordio/io.h"
#include "paddle/fluid/string/piece.h"
#include "paddle/fluid/string/piece.h"
#include <iostream>
namespace
paddle
{
namespace
paddle
{
namespace
recordio
{
namespace
recordio
{
Stream
*
Stream
::
Open
(
const
char
*
filename
,
const
char
*
mode
)
{
Stream
*
Stream
::
Open
(
const
char
*
filename
,
const
char
*
mode
)
{
...
@@ -38,7 +40,7 @@ void FileStream::Write(const void* ptr, size_t size) {
...
@@ -38,7 +40,7 @@ void FileStream::Write(const void* ptr, size_t size) {
}
}
size_t
FileStream
::
Tell
()
{
return
ftell
(
fp_
);
}
size_t
FileStream
::
Tell
()
{
return
ftell
(
fp_
);
}
void
FileStream
::
Seek
(
size_t
p
)
{
fseek
(
fp_
,
static_cast
<
long
>
(
p
)
,
SEEK_SET
);
}
void
FileStream
::
Seek
(
size_t
p
)
{
fseek
(
fp_
,
p
,
SEEK_SET
);
}
bool
FileStream
::
Eof
()
{
return
feof
(
fp_
);
}
bool
FileStream
::
Eof
()
{
return
feof
(
fp_
);
}
...
...
paddle/fluid/recordio/io.h
浏览文件 @
7016979c
...
@@ -16,19 +16,21 @@
...
@@ -16,19 +16,21 @@
#include <stdio.h>
#include <stdio.h>
#include <string>
#include <string>
#include "paddle/fluid/platform/enforce.h"
#include "paddle/fluid/platform/enforce.h"
#include "paddle/fluid/platform/macros.h" // DISABLE_COPY_ASSIGN
namespace
paddle
{
namespace
paddle
{
namespace
recordio
{
namespace
recordio
{
// S
tream abstract object
for read and write
// S
eekable Stream Interface
for read and write
class
Stream
{
class
Stream
{
public:
public:
virtual
~
Stream
()
{}
virtual
~
Stream
()
{}
virtual
size_t
Read
(
void
*
ptr
,
size_t
size
);
virtual
size_t
Read
(
void
*
ptr
,
size_t
size
)
=
0
;
virtual
void
Write
(
const
void
*
ptr
,
size_t
size
);
virtual
void
Write
(
const
void
*
ptr
,
size_t
size
)
=
0
;
virtual
size_t
Tell
();
virtual
size_t
Tell
()
=
0
;
virtual
void
Seek
(
)
;
virtual
void
Seek
(
size_t
p
)
=
0
;
// Create Stream Instance
// Create Stream Instance
static
Stream
*
Open
(
const
char
*
filename
,
const
char
*
mode
);
static
Stream
*
Open
(
const
char
*
filename
,
const
char
*
mode
);
};
};
...
@@ -47,6 +49,7 @@ public:
...
@@ -47,6 +49,7 @@ public:
private:
private:
FILE
*
fp_
;
FILE
*
fp_
;
DISABLE_COPY_AND_ASSIGN
(
FileStream
);
};
};
}
// namespace recordio
}
// namespace recordio
...
...
paddle/fluid/recordio/io_test.cc
浏览文件 @
7016979c
...
@@ -21,7 +21,7 @@ using namespace paddle::recordio;
...
@@ -21,7 +21,7 @@ using namespace paddle::recordio;
TEST
(
FileStream
,
IO
)
{
TEST
(
FileStream
,
IO
)
{
{
{
// Write
// Write
Stream
*
fs
=
Stream
::
Open
(
"/tmp/record_0"
,
"
r
w"
);
Stream
*
fs
=
Stream
::
Open
(
"/tmp/record_0"
,
"w"
);
fs
->
Write
(
"hello"
,
6
);
fs
->
Write
(
"hello"
,
6
);
delete
fs
;
delete
fs
;
}
}
...
...
paddle/fluid/recordio/writer.cc
浏览文件 @
7016979c
...
@@ -26,16 +26,16 @@ Writer::Writer(Stream* fo, int maxChunkSize, int compressor)
...
@@ -26,16 +26,16 @@ Writer::Writer(Stream* fo, int maxChunkSize, int compressor)
chunk_
.
reset
(
new
Chunk
);
chunk_
.
reset
(
new
Chunk
);
}
}
size_t
Writer
::
Write
(
const
std
::
string
&
record
)
{
size_t
Writer
::
Write
(
const
char
*
buf
,
size_t
length
)
{
if
(
stream_
==
nullptr
)
{
if
(
stream_
==
nullptr
)
{
LOG
(
WARNING
)
<<
"Cannot write since writer had been closed."
;
LOG
(
WARNING
)
<<
"Cannot write since writer had been closed."
;
return
0
;
return
0
;
}
}
if
((
record
.
size
()
+
chunk_
->
NumBytes
())
>
max_chunk_size_
)
{
if
((
length
+
chunk_
->
NumBytes
())
>
max_chunk_size_
)
{
chunk_
->
Dump
(
stream_
,
compressor_
);
chunk_
->
Dump
(
stream_
,
compressor_
);
}
}
chunk_
->
Add
(
record
);
chunk_
->
Add
(
buf
,
length
);
return
record
.
size
()
;
return
length
;
}
}
// size_t Writer::Write(const char* buf, size_t length) {
// size_t Writer::Write(const char* buf, size_t length) {
...
...
paddle/fluid/recordio/writer.h
浏览文件 @
7016979c
...
@@ -16,7 +16,6 @@
...
@@ -16,7 +16,6 @@
#include <memory>
#include <memory>
#include <string>
#include <string>
#include "paddle/fluid/platform/macros.h" // DISABLE_COPY_ASSIGN
#include "paddle/fluid/recordio/header.h"
#include "paddle/fluid/recordio/header.h"
#include "paddle/fluid/recordio/io.h"
#include "paddle/fluid/recordio/io.h"
...
@@ -44,7 +43,6 @@ private:
...
@@ -44,7 +43,6 @@ private:
int
max_chunk_size_
;
int
max_chunk_size_
;
// Compressor used for chuck
// Compressor used for chuck
Compressor
compressor_
;
Compressor
compressor_
;
DISABLE_COPY_AND_ASSIGN
(
Writer
);
DISABLE_COPY_AND_ASSIGN
(
Writer
);
};
};
...
...
paddle/fluid/recordio/writer_test.cc
浏览文件 @
7016979c
...
@@ -18,4 +18,4 @@
...
@@ -18,4 +18,4 @@
using
namespace
paddle
::
recordio
;
using
namespace
paddle
::
recordio
;
TEST
(
Writer
,
Normal
)
{
Stream
}
TEST
(
Writer
,
Normal
)
{}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录