Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
b3bb6527
T
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1180
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
b3bb6527
编写于
4月 03, 2022
作者:
C
Cary Xu
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
data and bitmap compress/store/decompress separatedly
上级
ca3206d4
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
120 addition
and
59 deletion
+120
-59
include/common/tdataformat.h
include/common/tdataformat.h
+3
-2
include/common/trow.h
include/common/trow.h
+14
-14
source/common/src/tdataformat.c
source/common/src/tdataformat.c
+17
-9
source/common/src/trow.c
source/common/src/trow.c
+1
-1
source/dnode/vnode/src/inc/tsdbReadImpl.h
source/dnode/vnode/src/inc/tsdbReadImpl.h
+4
-4
source/dnode/vnode/src/tsdb/tsdbCommit.c
source/dnode/vnode/src/tsdb/tsdbCommit.c
+29
-4
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
+50
-23
source/util/src/tcompression.c
source/util/src/tcompression.c
+2
-2
未找到文件。
include/common/tdataformat.h
浏览文件 @
b3bb6527
...
...
@@ -135,8 +135,8 @@ typedef struct {
#define TD_VTYPE_PARTS 4 // 8 bits / TD_VTYPE_BITS = 4
#define TD_VTYPE_OPTR 3 // TD_VTYPE_PARTS - 1, utilize to get remainder
#define TD_BITMAP_BYTES(cnt) (ceil((double)
cnt
/ TD_VTYPE_PARTS))
#define TD_BIT_TO_BYTES(cnt) (ceil((double)
cnt
/ 8))
#define TD_BITMAP_BYTES(cnt) (ceil((double)
(cnt)
/ TD_VTYPE_PARTS))
#define TD_BIT_TO_BYTES(cnt) (ceil((double)
(cnt)
/ 8))
int32_t
tdInitTSchemaBuilder
(
STSchemaBuilder
*
pBuilder
,
schema_ver_t
version
);
void
tdDestroyTSchemaBuilder
(
STSchemaBuilder
*
pBuilder
);
...
...
@@ -365,6 +365,7 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche
}
#endif
// ----------------- Data column structure
// SDataCol arrangement: data => bitmap => dataOffset
typedef
struct
SDataCol
{
int8_t
type
;
// column type
uint8_t
bitmap
:
1
;
// 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
...
...
include/common/trow.h
浏览文件 @
b3bb6527
...
...
@@ -147,20 +147,20 @@ typedef struct {
typedef
struct
{
// basic info
int8_t
rowType
;
int16
_t
sver
;
STSRow
*
pBuf
;
int8_t
rowType
;
schema_ver
_t
sver
;
STSRow
*
pBuf
;
// extended info
int32_t
flen
;
int16
_t
nBoundCols
;
int16
_t
nCols
;
int16
_t
nBitmaps
;
int16
_t
nBoundBitmaps
;
int32_t
offset
;
void
*
pBitmap
;
void
*
pOffset
;
int32_t
extendedRowSize
;
int32_t
flen
;
col_id
_t
nBoundCols
;
col_id
_t
nCols
;
col_id
_t
nBitmaps
;
col_id
_t
nBoundBitmaps
;
int32_t
offset
;
void
*
pBitmap
;
void
*
pOffset
;
int32_t
extendedRowSize
;
}
SRowBuilder
;
#define TD_ROW_HEAD_LEN (sizeof(STSRow))
...
...
@@ -448,9 +448,9 @@ static FORCE_INLINE int32_t tdSRowSetExtendedInfo(SRowBuilder *pBuilder, int32_t
}
#ifdef TD_SUPPORT_BITMAP
// the primary TS key is stored separatedly
pBuilder
->
nBitmaps
=
(
int16
_t
)
TD_BITMAP_BYTES
(
pBuilder
->
nCols
-
1
);
pBuilder
->
nBitmaps
=
(
col_id
_t
)
TD_BITMAP_BYTES
(
pBuilder
->
nCols
-
1
);
if
(
nBoundCols
>
0
)
{
pBuilder
->
nBoundBitmaps
=
(
int16
_t
)
TD_BITMAP_BYTES
(
pBuilder
->
nBoundCols
-
1
);
pBuilder
->
nBoundBitmaps
=
(
col_id
_t
)
TD_BITMAP_BYTES
(
pBuilder
->
nBoundCols
-
1
);
}
else
{
pBuilder
->
nBoundBitmaps
=
0
;
}
...
...
source/common/src/tdataformat.c
浏览文件 @
b3bb6527
...
...
@@ -33,7 +33,8 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
spaceNeeded
+=
(
int
)
nBitmapBytes
;
// TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more
// TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered.
spaceNeeded
+=
TYPE_BYTES
[
pCol
->
type
];
// spaceNeeded += TYPE_BYTES[pCol->type]; // the bitmap part is append as a single part since 2022.04.03, thus remove
// the additional space
#endif
if
(
pCol
->
spaceSize
<
spaceNeeded
)
{
...
...
@@ -47,6 +48,7 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
}
}
#ifdef TD_SUPPORT_BITMAP
if
(
IS_VAR_DATA_TYPE
(
pCol
->
type
))
{
pCol
->
pBitmap
=
POINTER_SHIFT
(
pCol
->
pData
,
pCol
->
bytes
*
maxPoints
);
pCol
->
dataOff
=
POINTER_SHIFT
(
pCol
->
pBitmap
,
nBitmapBytes
);
...
...
@@ -306,7 +308,7 @@ static FORCE_INLINE const void *tdGetColDataOfRowUnsafe(SDataCol *pCol, int row)
bool
isNEleNull
(
SDataCol
*
pCol
,
int
nEle
)
{
if
(
isAllRowsNull
(
pCol
))
return
true
;
for
(
int
i
=
0
;
i
<
nEle
;
i
++
)
{
for
(
int
i
=
0
;
i
<
nEle
;
++
i
)
{
if
(
!
isNull
(
tdGetColDataOfRowUnsafe
(
pCol
,
i
),
pCol
->
type
))
return
false
;
}
return
true
;
...
...
@@ -327,7 +329,7 @@ static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index) {
static
void
dataColSetNEleNull
(
SDataCol
*
pCol
,
int
nEle
)
{
if
(
IS_VAR_DATA_TYPE
(
pCol
->
type
))
{
pCol
->
len
=
0
;
for
(
int
i
=
0
;
i
<
nEle
;
i
++
)
{
for
(
int
i
=
0
;
i
<
nEle
;
++
i
)
{
dataColSetNullAt
(
pCol
,
i
);
}
}
else
{
...
...
@@ -343,7 +345,7 @@ void *dataColSetOffset(SDataCol *pCol, int nEle) {
// char *tptr = (char *)(pCol->pData);
VarDataOffsetT
offset
=
0
;
for
(
int
i
=
0
;
i
<
nEle
;
i
++
)
{
for
(
int
i
=
0
;
i
<
nEle
;
++
i
)
{
pCol
->
dataOff
[
i
]
=
offset
;
offset
+=
varDataTLen
(
tptr
);
tptr
=
POINTER_SHIFT
(
tptr
,
varDataTLen
(
tptr
));
...
...
@@ -371,6 +373,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
tdFreeDataCols
(
pCols
);
return
NULL
;
}
#if 0 // no need as calloc used
int i;
for (i = 0; i < maxCols; i++) {
pCols->cols[i].spaceSize = 0;
...
...
@@ -378,6 +381,7 @@ SDataCols *tdNewDataCols(int maxCols, int maxRows) {
pCols->cols[i].pData = NULL;
pCols->cols[i].dataOff = NULL;
}
#endif
}
return
pCols
;
...
...
@@ -391,17 +395,21 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
void
*
ptr
=
(
SDataCol
*
)
taosMemoryRealloc
(
pCols
->
cols
,
sizeof
(
SDataCol
)
*
pCols
->
maxCols
);
if
(
ptr
==
NULL
)
return
-
1
;
pCols
->
cols
=
ptr
;
for
(
i
=
oldMaxCols
;
i
<
pCols
->
maxCols
;
i
++
)
{
for
(
i
=
oldMaxCols
;
i
<
pCols
->
maxCols
;
++
i
)
{
pCols
->
cols
[
i
].
pData
=
NULL
;
pCols
->
cols
[
i
].
dataOff
=
NULL
;
pCols
->
cols
[
i
].
pBitmap
=
NULL
;
pCols
->
cols
[
i
].
spaceSize
=
0
;
}
}
#if 0
tdResetDataCols(pCols); // redundant loop to reset len/blen to 0, already reset in following dataColInit(...)
#endif
tdResetDataCols
(
pCols
)
;
pCols
->
numOfRows
=
0
;
pCols
->
numOfCols
=
schemaNCols
(
pSchema
);
for
(
i
=
0
;
i
<
schemaNCols
(
pSchema
);
i
++
)
{
for
(
i
=
0
;
i
<
schemaNCols
(
pSchema
);
++
i
)
{
dataColInit
(
pCols
->
cols
+
i
,
schemaColAt
(
pSchema
,
i
),
pCols
->
maxPoints
);
}
...
...
@@ -413,7 +421,7 @@ SDataCols *tdFreeDataCols(SDataCols *pCols) {
if
(
pCols
)
{
if
(
pCols
->
cols
)
{
int
maxCols
=
pCols
->
maxCols
;
for
(
i
=
0
;
i
<
maxCols
;
i
++
)
{
for
(
i
=
0
;
i
<
maxCols
;
++
i
)
{
SDataCol
*
pCol
=
&
pCols
->
cols
[
i
];
taosMemoryFreeClear
(
pCol
->
pData
);
}
...
...
@@ -464,7 +472,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
void
tdResetDataCols
(
SDataCols
*
pCols
)
{
if
(
pCols
!=
NULL
)
{
pCols
->
numOfRows
=
0
;
for
(
int
i
=
0
;
i
<
pCols
->
maxCols
;
i
++
)
{
for
(
int
i
=
0
;
i
<
pCols
->
maxCols
;
++
i
)
{
dataColReset
(
pCols
->
cols
+
i
);
}
}
...
...
source/common/src/trow.c
浏览文件 @
b3bb6527
...
...
@@ -503,7 +503,7 @@ SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {
memcpy
(
pRet
->
cols
[
i
].
dataOff
,
pDataCols
->
cols
[
i
].
dataOff
,
dataOffSize
);
}
if
(
!
TD_COL_ROWS_NORM
(
pRet
->
cols
+
i
))
{
int32_t
nBitmapBytes
=
(
int32_t
)
TD_BITMAP_BYTES
(
pDataCols
->
maxPoint
s
);
int32_t
nBitmapBytes
=
(
int32_t
)
TD_BITMAP_BYTES
(
pDataCols
->
numOfRow
s
);
memcpy
(
pRet
->
cols
[
i
].
pBitmap
,
pDataCols
->
cols
[
i
].
pBitmap
,
nBitmapBytes
);
}
}
...
...
source/dnode/vnode/src/inc/tsdbReadImpl.h
浏览文件 @
b3bb6527
...
...
@@ -112,10 +112,10 @@ typedef struct {
#else
typedef
struct
{
int16_t
colId
;
uint
8_t
bitmap
:
1
;
// 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
uint
8_t
reserve
:
7
;
uint
8_t
type
;
int32_t
len
;
uint
16_t
type
:
6
;
uint
16_t
blen
:
10
;
// bitmap length(TODO: full UT for the bitmap compress of various data input)
uint
32_t
bitmap
:
1
;
// 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
uint32_t
len
:
31
;
// data length + bitmap length
uint32_t
offset
;
}
SBlockColV0
;
...
...
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @
b3bb6527
...
...
@@ -1281,7 +1281,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
uint32_t
tsizeAggr
=
(
uint32_t
)
tsdbBlockAggrSize
(
nColsNotAllNull
,
SBlockVerLatest
);
int32_t
keyLen
=
0
;
int32_t
nBitmaps
=
(
int32_t
)
TD_BITMAP_BYTES
(
rowsToWrite
);
int32_t
tBitmaps
=
0
;
//
int32_t tBitmaps = 0;
for
(
int
ncol
=
0
;
ncol
<
pDataCols
->
numOfCols
;
++
ncol
)
{
// All not NULL columns finish
...
...
@@ -1297,7 +1297,10 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
#ifdef TD_SUPPORT_BITMAP
int32_t
tBitmaps
=
0
;
int32_t
tBitmapsLen
=
0
;
if
((
ncol
!=
0
)
&&
!
TD_COL_ROWS_NORM
(
pBlockCol
))
{
tBitmaps
=
nBitmaps
;
#if 0
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
tBitmaps = nBitmaps;
tlen += tBitmaps;
...
...
@@ -1305,16 +1308,17 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
tlen += tBitmaps * TYPE_BYTES[pDataCol->type];
}
#endif
// move bitmap parts ahead
// TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove
memcpy
(
POINTER_SHIFT
(
pDataCol
->
pData
,
pDataCol
->
len
),
pDataCol
->
pBitmap
,
nBitmaps
);
//
memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps);
}
#endif
void
*
tptr
;
// Make room
if
(
tsdbMakeRoom
(
ppBuf
,
lsize
+
tlen
+
COMP_OVERFLOW_BYTES
+
sizeof
(
TSCKSUM
))
<
0
)
{
if
(
tsdbMakeRoom
(
ppBuf
,
lsize
+
tlen
+
tBitmaps
+
2
*
COMP_OVERFLOW_BYTES
+
sizeof
(
TSCKSUM
))
<
0
)
{
return
-
1
;
}
pBlockData
=
(
SBlockData
*
)(
*
ppBuf
);
...
...
@@ -1327,23 +1331,44 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
// Compress or just copy
if
(
pCfg
->
compression
)
{
#if 0
flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite + tBitmaps, tptr,
tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf,
tlen + COMP_OVERFLOW_BYTES);
#endif
flen
=
(
*
(
tDataTypes
[
pDataCol
->
type
].
compFunc
))((
char
*
)
pDataCol
->
pData
,
tlen
,
rowsToWrite
,
tptr
,
tlen
+
COMP_OVERFLOW_BYTES
,
pCfg
->
compression
,
*
ppCBuf
,
tlen
+
COMP_OVERFLOW_BYTES
);
if
(
tBitmaps
>
0
)
{
tptr
=
POINTER_SHIFT
(
pBlockData
,
lsize
+
flen
);
tBitmapsLen
=
tsCompressTinyint
((
char
*
)
pDataCol
->
pBitmap
,
tBitmaps
,
rowsToWrite
,
tptr
,
tBitmaps
+
COMP_OVERFLOW_BYTES
,
pCfg
->
compression
,
*
ppCBuf
,
tBitmaps
+
COMP_OVERFLOW_BYTES
);
TASSERT
((
tBitmapsLen
>
0
)
&&
(
tBitmapsLen
<=
(
tBitmaps
+
COMP_OVERFLOW_BYTES
)));
flen
+=
tBitmapsLen
;
}
}
else
{
flen
=
tlen
;
memcpy
(
tptr
,
pDataCol
->
pData
,
flen
);
if
(
tBitmaps
>
0
)
{
tptr
=
POINTER_SHIFT
(
pBlockData
,
lsize
+
flen
);
memcpy
(
tptr
,
pDataCol
->
pBitmap
,
tBitmaps
);
tBitmapsLen
=
tBitmaps
;
flen
+=
tBitmapsLen
;
}
}
// Add checksum
ASSERT
(
flen
>
0
);
ASSERT
(
tBitmapsLen
<=
1024
);
flen
+=
sizeof
(
TSCKSUM
);
taosCalcChecksumAppend
(
0
,
(
uint8_t
*
)
tptr
,
flen
);
tsdbUpdateDFileMagic
(
pDFile
,
POINTER_SHIFT
(
tptr
,
flen
-
sizeof
(
TSCKSUM
)));
if
(
ncol
!=
0
)
{
tsdbSetBlockColOffset
(
pBlockCol
,
toffset
);
pBlockCol
->
len
=
flen
;
pBlockCol
->
len
=
flen
;
// data + bitmaps
pBlockCol
->
blen
=
tBitmapsLen
;
++
tcol
;
}
else
{
keyLen
=
flen
;
...
...
source/dnode/vnode/src/tsdb/tsdbReadImpl.c
浏览文件 @
b3bb6527
...
...
@@ -21,9 +21,8 @@ static void tsdbResetReadTable(SReadH *pReadh);
static
void
tsdbResetReadFile
(
SReadH
*
pReadh
);
static
int
tsdbLoadBlockOffset
(
SReadH
*
pReadh
,
SBlock
*
pBlock
);
static
int
tsdbLoadBlockDataImpl
(
SReadH
*
pReadh
,
SBlock
*
pBlock
,
SDataCols
*
pDataCols
);
static
int
tsdbCheckAndDecodeColumnData
(
SDataCol
*
pDataCol
,
void
*
content
,
int32_t
len
,
int8_t
comp
,
int
numOfRows
,
int
numOfBitmaps
,
int
lenOfBitmaps
,
int
maxPoints
,
char
*
buffer
,
int
bufferSize
);
static
int
tsdbCheckAndDecodeColumnData
(
SDataCol
*
pDataCol
,
void
*
content
,
int32_t
len
,
int32_t
bitmapLen
,
int8_t
comp
,
int
numOfRows
,
int
numOfBitmaps
,
int
maxPoints
,
char
*
buffer
,
int
bufferSize
);
static
int
tsdbLoadBlockDataColsImpl
(
SReadH
*
pReadh
,
SBlock
*
pBlock
,
SDataCols
*
pDataCols
,
const
int16_t
*
colIds
,
int
numOfColIds
);
static
int
tsdbLoadColData
(
SReadH
*
pReadh
,
SDFile
*
pDFile
,
SBlock
*
pBlock
,
SBlockCol
*
pBlockCol
,
SDataCol
*
pDataCol
);
...
...
@@ -548,7 +547,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
if
(
dcol
!=
0
&&
ccol
>=
pBlockData
->
numOfCols
)
{
// Set current column as NULL and forward
dataColReset
(
pDataCol
);
dcol
++
;
++
dcol
;
continue
;
}
...
...
@@ -567,9 +566,11 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
TD_SET_COL_ROWS_NORM
(
pDataCol
);
}
int32_t
tBitmaps
=
0
;
//
int32_t tBitmaps = 0;
int32_t
tLenBitmap
=
0
;
if
((
dcol
!=
0
)
&&
!
TD_COL_ROWS_NORM
(
pBlockCol
))
{
tLenBitmap
=
nBitmaps
;
#if 0
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
tBitmaps = nBitmaps;
tLenBitmap = tBitmaps;
...
...
@@ -577,17 +578,18 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type];
}
#endif
}
if
(
tcolId
==
pDataCol
->
colId
)
{
if
(
pBlock
->
algorithm
==
TWO_STAGE_COMP
)
{
int
zsize
=
pDataCol
->
bytes
*
pBlock
->
numOfRows
+
COMP_OVERFLOW_BYTES
;
int
zsize
=
pDataCol
->
bytes
*
pBlock
->
numOfRows
+
tLenBitmap
+
2
*
COMP_OVERFLOW_BYTES
;
if
(
tsdbMakeRoom
((
void
**
)(
&
TSDB_READ_COMP_BUF
(
pReadh
)),
zsize
)
<
0
)
return
-
1
;
}
if
(
tsdbCheckAndDecodeColumnData
(
pDataCol
,
POINTER_SHIFT
(
pBlockData
,
tsize
+
toffset
),
tlen
,
pBlock
->
algorithm
,
pBlock
->
numOfRows
,
tBitmaps
,
tLenBitmap
,
pDataCols
->
maxPoints
,
TSDB_READ_COMP_BUF
(
pReadh
)
,
(
int
)
taosTSizeof
(
TSDB_READ_COMP_BUF
(
pReadh
)))
<
0
)
{
if
(
tsdbCheckAndDecodeColumnData
(
pDataCol
,
POINTER_SHIFT
(
pBlockData
,
tsize
+
toffset
),
tlen
,
pBlock
Col
->
blen
,
pBlock
->
algorithm
,
pBlock
->
numOfRows
,
tLenBitmap
,
pDataCols
->
maxPoints
,
TSDB_READ_COMP_BUF
(
pReadh
),
(
int
)
taosTSizeof
(
TSDB_READ_COMP_BUF
(
pReadh
)))
<
0
)
{
tsdbError
(
"vgId:%d file %s is broken at column %d block offset %"
PRId64
" column offset %u"
,
TSDB_READ_REPO_ID
(
pReadh
),
TSDB_FILE_FULL_NAME
(
pDFile
),
tcolId
,
(
int64_t
)
pBlock
->
offset
,
toffset
);
return
-
1
;
...
...
@@ -609,9 +611,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
return
0
;
}
static
int
tsdbCheckAndDecodeColumnData
(
SDataCol
*
pDataCol
,
void
*
content
,
int32_t
len
,
int8_t
comp
,
int
numOfRows
,
int
numOfBitmaps
,
int
lenOfBitmaps
,
int
maxPoints
,
char
*
buffer
,
int
bufferSize
)
{
static
int
tsdbCheckAndDecodeColumnData
(
SDataCol
*
pDataCol
,
void
*
content
,
int32_t
len
,
int32_t
bitmapLen
,
int8_t
comp
,
int
numOfRows
,
int
numOfBitmaps
,
int
maxPoints
,
char
*
buffer
,
int
bufferSize
)
{
if
(
!
taosCheckChecksumWhole
((
uint8_t
*
)
content
,
len
))
{
terrno
=
TSDB_CODE_TDB_FILE_CORRUPTED
;
return
-
1
;
...
...
@@ -623,21 +624,41 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
if
(
comp
)
{
// Need to decompress
int
tlen
=
(
*
(
tDataTypes
[
pDataCol
->
type
].
decompFunc
))(
content
,
len
-
sizeof
(
TSCKSUM
),
numOfRows
+
numOfBitmap
s
,
(
*
(
tDataTypes
[
pDataCol
->
type
].
decompFunc
))(
content
,
len
-
bitmapLen
-
sizeof
(
TSCKSUM
),
numOfRow
s
,
pDataCol
->
pData
,
pDataCol
->
spaceSize
,
comp
,
buffer
,
bufferSize
);
if
(
tlen
<=
0
)
{
tsdbError
(
"Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d"
,
len
,
comp
,
numOfRows
,
maxPoints
,
bufferSize
);
tsdbError
(
"Failed to decompress column data, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d"
,
(
int32_t
)(
len
-
bitmapLen
-
sizeof
(
TSCKSUM
)),
comp
,
numOfRows
,
maxPoints
,
bufferSize
);
terrno
=
TSDB_CODE_TDB_FILE_CORRUPTED
;
return
-
1
;
}
pDataCol
->
len
=
tlen
;
if
(
numOfBitmaps
>
0
)
{
tlen
=
tsDecompressTinyint
(
POINTER_SHIFT
(
content
,
len
-
bitmapLen
-
sizeof
(
TSCKSUM
)),
bitmapLen
,
numOfBitmaps
,
pDataCol
->
pBitmap
,
pDataCol
->
spaceSize
,
comp
,
buffer
,
bufferSize
);
if
(
tlen
<=
0
)
{
tsdbError
(
"Failed to decompress column bitmap, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d "
"bufferSize:%d"
,
bitmapLen
,
comp
,
numOfBitmaps
,
maxPoints
,
bufferSize
);
terrno
=
TSDB_CODE_TDB_FILE_CORRUPTED
;
return
-
1
;
}
// pDataCol->blen = tlen;
}
}
else
{
// No need to decompress, just memcpy it
pDataCol
->
len
=
len
-
sizeof
(
TSCKSUM
);
pDataCol
->
len
=
len
-
bitmapLen
-
sizeof
(
TSCKSUM
);
memcpy
(
pDataCol
->
pData
,
content
,
pDataCol
->
len
);
if
(
numOfBitmaps
>
0
)
{
// pDataCol->blen = bitmapLen;
memcpy
(
pDataCol
->
pBitmap
,
POINTER_SHIFT
(
content
,
len
-
bitmapLen
-
sizeof
(
TSCKSUM
)),
bitmapLen
);
}
}
#if 0
if (lenOfBitmaps > 0) {
pDataCol->len -= lenOfBitmaps;
...
...
@@ -653,7 +674,10 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
} else if (IS_VAR_DATA_TYPE(pDataCol->type)) {
dataColSetOffset(pDataCol, numOfRows);
}
#endif
if
(
IS_VAR_DATA_TYPE
(
pDataCol
->
type
))
{
dataColSetOffset
(
pDataCol
,
numOfRows
);
}
return
0
;
}
...
...
@@ -740,14 +764,16 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
static
int
tsdbLoadColData
(
SReadH
*
pReadh
,
SDFile
*
pDFile
,
SBlock
*
pBlock
,
SBlockCol
*
pBlockCol
,
SDataCol
*
pDataCol
)
{
ASSERT
(
pDataCol
->
colId
==
pBlockCol
->
colId
);
STsdb
*
pRepo
=
TSDB_READ_REPO
(
pReadh
);
STsdb
*
pRepo
=
TSDB_READ_REPO
(
pReadh
);
STsdbCfg
*
pCfg
=
REPO_CFG
(
pRepo
);
int
nBitmaps
=
(
int
)
TD_BITMAP_BYTES
(
pBlock
->
numOfRows
);
int32_t
tBitmaps
=
0
;
int
nBitmaps
=
(
int
)
TD_BITMAP_BYTES
(
pBlock
->
numOfRows
);
//
int32_t tBitmaps = 0;
int32_t
tLenBitmap
=
0
;
if
(
!
TD_COL_ROWS_NORM
(
pBlockCol
))
{
tLenBitmap
=
nBitmaps
;
#if 0
if (IS_VAR_DATA_TYPE(pDataCol->type)) {
tBitmaps = nBitmaps;
tLenBitmap = tBitmaps;
...
...
@@ -755,9 +781,10 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type];
}
#endif
}
int
tsize
=
pDataCol
->
bytes
*
pBlock
->
numOfRows
+
tLenBitmap
+
COMP_OVERFLOW_BYTES
;
int
tsize
=
pDataCol
->
bytes
*
pBlock
->
numOfRows
+
tLenBitmap
+
2
*
COMP_OVERFLOW_BYTES
;
if
(
tsdbMakeRoom
((
void
**
)(
&
TSDB_READ_BUF
(
pReadh
)),
pBlockCol
->
len
)
<
0
)
return
-
1
;
if
(
tsdbMakeRoom
((
void
**
)(
&
TSDB_READ_COMP_BUF
(
pReadh
)),
tsize
)
<
0
)
return
-
1
;
...
...
@@ -785,8 +812,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
return
-
1
;
}
if
(
tsdbCheckAndDecodeColumnData
(
pDataCol
,
pReadh
->
pBuf
,
pBlockCol
->
len
,
pBlock
->
algorithm
,
pBlock
->
numOfRows
,
tBitmap
s
,
tLenBitmap
,
pCfg
->
maxRowsPerFileBlock
,
pReadh
->
pCBuf
,
if
(
tsdbCheckAndDecodeColumnData
(
pDataCol
,
pReadh
->
pBuf
,
pBlockCol
->
len
,
pBlock
Col
->
blen
,
pBlock
->
algorithm
,
pBlock
->
numOfRow
s
,
tLenBitmap
,
pCfg
->
maxRowsPerFileBlock
,
pReadh
->
pCBuf
,
(
int32_t
)
taosTSizeof
(
pReadh
->
pCBuf
))
<
0
)
{
tsdbError
(
"vgId:%d file %s is broken at column %d offset %"
PRId64
,
REPO_ID
(
pRepo
),
TSDB_FILE_FULL_NAME
(
pDFile
),
pBlockCol
->
colId
,
offset
);
...
...
source/util/src/tcompression.c
浏览文件 @
b3bb6527
...
...
@@ -28,11 +28,11 @@
*
* BOOLEAN Compression Algorithm:
* We provide two methods for compress boolean types. Because boolean types in C
* code are char bytes with 0 and 1 values only, only one bit can used to discrim
e
nate
* code are char bytes with 0 and 1 values only, only one bit can used to discrim
i
nate
* the values.
* 1. The first method is using only 1 bit to represent the boolean value with 1 for
* true and 0 for false. Then the compression rate is 1/8.
* 2. The second method is using run length encoding (RLE) methods. This metho
s
works
* 2. The second method is using run length encoding (RLE) methods. This metho
d
works
* better when there are a lot of consecutive true values or false values.
*
* STRING Compression Algorithm:
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录