Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
a4d16f1c
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1184
Star
22015
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
a4d16f1c
编写于
9月 27, 2022
作者:
H
Hongze Cheng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
more code
上级
fe95393b
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
78 addition
and
49 deletion
+78
-49
include/util/tcompression.h
include/util/tcompression.h
+1
-1
source/dnode/vnode/src/inc/tsdb.h
source/dnode/vnode/src/inc/tsdb.h
+3
-3
source/dnode/vnode/src/tsdb/tsdbCommit.c
source/dnode/vnode/src/tsdb/tsdbCommit.c
+3
-4
source/dnode/vnode/src/tsdb/tsdbDiskData.c
source/dnode/vnode/src/tsdb/tsdbDiskData.c
+60
-37
source/util/src/tcompression.c
source/util/src/tcompression.c
+11
-4
未找到文件。
include/util/tcompression.h
浏览文件 @
a4d16f1c
...
...
@@ -131,7 +131,7 @@ typedef struct SCompressor SCompressor;
int32_t
tCompressorCreate
(
SCompressor
**
ppCmprsor
);
int32_t
tCompressorDestroy
(
SCompressor
*
pCmprsor
);
int32_t
tCompressStart
(
SCompressor
*
pCmprsor
,
int8_t
type
,
int8_t
cmprAlg
);
int32_t
tCompressEnd
(
SCompressor
*
pCmprsor
,
const
uint8_t
**
pp
Data
,
int32_t
*
nData
);
int32_t
tCompressEnd
(
SCompressor
*
pCmprsor
,
const
uint8_t
**
pp
Out
,
int32_t
*
nOut
,
int32_t
*
nOrigin
);
int32_t
tCompress
(
SCompressor
*
pCmprsor
,
const
void
*
pData
,
int64_t
nData
);
#ifdef __cplusplus
...
...
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @
a4d16f1c
...
...
@@ -326,9 +326,9 @@ int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder);
void
*
tDiskDataBuilderDestroy
(
SDiskDataBuilder
*
pBuilder
);
int32_t
tDiskDataBuilderInit
(
SDiskDataBuilder
*
pBuilder
,
STSchema
*
pTSchema
,
TABLEID
*
pId
,
uint8_t
cmprAlg
,
uint8_t
calcSma
);
int32_t
tDiskDataBuilder
AddRow
(
SDiskDataBuilder
*
pBuilder
,
TSDBROW
*
pRow
,
STSchema
*
pTSchema
,
TABLEID
*
pId
);
int32_t
t
GnrtDiskData
(
SDiskDataBuilder
*
pBuilder
,
SDiskData
*
pDiskData
);
int32_t
t
DiskDataDestroy
(
SDiskData
*
pDiskData
);
int32_t
tDiskDataBuilder
Clear
(
SDiskDataBuilder
*
pBuilder
);
int32_t
t
DiskDataAddRow
(
SDiskDataBuilder
*
pBuilder
,
TSDBROW
*
pRow
,
STSchema
*
pTSchema
,
TABLEID
*
pId
);
int32_t
t
GnrtDiskData
(
SDiskDataBuilder
*
pBuilder
,
const
SDiskData
**
p
pDiskData
);
// structs =======================
struct
STsdbFS
{
...
...
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @
a4d16f1c
...
...
@@ -665,7 +665,7 @@ static int32_t tsdbCommitSttBlk(SDataFWriter *pWriter, SDiskDataBuilder *pBuilde
.
nRow
=
pBuilder
->
nRow
};
// gnrt
code
=
tGnrtDiskData
(
pBuilder
,
&
pBuilder
->
dd
);
//
code = tGnrtDiskData(pBuilder, &pBuilder->dd);
TSDB_CHECK_CODE
(
code
,
lino
,
_exit
);
// write
...
...
@@ -1369,7 +1369,6 @@ static int32_t tsdbInitSttBlockBuilderIfNeed(SCommitter *pCommitter, TABLEID id)
if
(
!
pBuilder
->
suid
&&
!
pBuilder
->
uid
)
{
ASSERT
(
pCommitter
->
skmTable
.
suid
==
id
.
suid
);
ASSERT
(
pCommitter
->
skmTable
.
uid
==
id
.
uid
);
TABLEID
tid
=
{.
suid
=
id
.
suid
,
.
uid
=
id
.
suid
?
0
:
id
.
uid
};
code
=
tDiskDataBuilderInit
(
pCommitter
->
dWriter
.
pBuilder
,
pCommitter
->
skmTable
.
pTSchema
,
&
id
,
pCommitter
->
cmprAlg
,
0
);
TSDB_CHECK_CODE
(
code
,
lino
,
_exit
);
...
...
@@ -1396,7 +1395,7 @@ static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) {
for
(
int32_t
iRow
=
0
;
iRow
<
pBData
->
nRow
;
iRow
++
)
{
TSDBROW
row
=
tsdbRowFromBlockData
(
pBData
,
iRow
);
code
=
tDiskData
Builder
AddRow
(
pCommitter
->
dWriter
.
pBuilder
,
&
row
,
NULL
,
&
id
);
code
=
tDiskDataAddRow
(
pCommitter
->
dWriter
.
pBuilder
,
&
row
,
NULL
,
&
id
);
TSDB_CHECK_CODE
(
code
,
lino
,
_exit
);
if
(
pCommitter
->
dWriter
.
pBuilder
->
nRow
>=
pCommitter
->
maxRow
)
{
...
...
@@ -1435,7 +1434,7 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) {
pTSchema
=
pCommitter
->
skmRow
.
pTSchema
;
}
code
=
tDiskData
Builder
AddRow
(
pCommitter
->
dWriter
.
pBuilder
,
&
pRowInfo
->
row
,
pTSchema
,
&
id
);
code
=
tDiskDataAddRow
(
pCommitter
->
dWriter
.
pBuilder
,
&
pRowInfo
->
row
,
pTSchema
,
&
id
);
TSDB_CHECK_CODE
(
code
,
lino
,
_exit
);
code
=
tsdbNextCommitRow
(
pCommitter
);
...
...
source/dnode/vnode/src/tsdb/tsdbDiskData.c
浏览文件 @
a4d16f1c
...
...
@@ -31,9 +31,16 @@ struct SDiskColBuilder {
SColumnDataAgg
sma
;
uint8_t
minSet
;
uint8_t
maxSet
;
uint8_t
*
aBuf
[
1
];
uint8_t
*
aBuf
[
2
];
};
// SDiskData ================================================
static
int32_t
tDiskDataDestroy
(
SDiskData
*
pDiskData
)
{
int32_t
code
=
0
;
pDiskData
->
aDiskCol
=
taosArrayDestroy
(
pDiskData
->
aDiskCol
);
return
code
;
}
// SDiskColBuilder ================================================
#define tDiskColBuilderCreate() \
(SDiskColBuilder) { 0 }
...
...
@@ -91,7 +98,7 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) {
.
type
=
pBuilder
->
type
,
.
smaOn
=
pBuilder
->
calcSma
,
.
flag
=
pBuilder
->
flag
,
.
szOrigin
=
0
,
// todo
.
szOrigin
=
0
,
.
szBitmap
=
0
,
.
szOffset
=
0
,
.
szValue
=
0
,
...
...
@@ -109,20 +116,27 @@ static int32_t tGnrtDiskCol(SDiskColBuilder *pBuilder, SDiskCol *pDiskCol) {
nBit
=
BIT1_SIZE
(
pBuilder
->
nVal
);
}
pDiskCol
->
bCol
.
szBitmap
=
tsCompressTinyint
(
pBuilder
->
pBitMap
,
nBit
,
nBit
,
pBuilder
->
aBuf
[
0
],
0
,
pBuilder
->
cmprAlg
,
NULL
,
0
);
// todo: alloc
code
=
tRealloc
(
&
pBuilder
->
aBuf
[
0
],
nBit
+
COMP_OVERFLOW_BYTES
);
if
(
code
)
return
code
;
code
=
tRealloc
(
&
pBuilder
->
aBuf
[
1
],
nBit
+
COMP_OVERFLOW_BYTES
);
if
(
code
)
return
code
;
pDiskCol
->
bCol
.
szBitmap
=
tsCompressTinyint
(
pBuilder
->
pBitMap
,
nBit
,
nBit
,
pBuilder
->
aBuf
[
0
],
nBit
+
COMP_OVERFLOW_BYTES
,
pBuilder
->
cmprAlg
,
pBuilder
->
aBuf
[
1
],
nBit
+
COMP_OVERFLOW_BYTES
);
pDiskCol
->
pBit
=
pBuilder
->
aBuf
[
0
];
}
// OFFSET
if
(
IS_VAR_DATA_TYPE
(
pBuilder
->
type
))
{
code
=
tCompressEnd
(
pBuilder
->
pOffC
,
&
pDiskCol
->
pOff
,
&
pDiskCol
->
bCol
.
szOffset
);
code
=
tCompressEnd
(
pBuilder
->
pOffC
,
&
pDiskCol
->
pOff
,
&
pDiskCol
->
bCol
.
szOffset
,
NULL
);
if
(
code
)
return
code
;
}
// VALUE
if
(
pBuilder
->
flag
!=
(
HAS_NULL
|
HAS_NONE
))
{
code
=
tCompressEnd
(
pBuilder
->
pValC
,
&
pDiskCol
->
pVal
,
&
pDiskCol
->
bCol
.
szValue
);
code
=
tCompressEnd
(
pBuilder
->
pValC
,
&
pDiskCol
->
pVal
,
&
pDiskCol
->
bCol
.
szValue
,
&
pDiskCol
->
bCol
.
szOrigin
);
if
(
code
)
return
code
;
}
...
...
@@ -425,8 +439,8 @@ static int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) {
}
}
if
(
tDiskColAddValImpl
[
pBuilder
->
type
][
pColVal
->
type
])
{
code
=
tDiskColAddValImpl
[
pBuilder
->
type
][
pColVal
->
type
](
pBuilder
,
pColVal
);
if
(
tDiskColAddValImpl
[
pBuilder
->
flag
][
pColVal
->
flag
])
{
code
=
tDiskColAddValImpl
[
pBuilder
->
flag
][
pColVal
->
flag
](
pBuilder
,
pColVal
);
if
(
code
)
return
code
;
}
...
...
@@ -465,6 +479,7 @@ void *tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) {
for
(
int32_t
iBuf
=
0
;
iBuf
<
sizeof
(
pBuilder
->
aBuf
)
/
sizeof
(
pBuilder
->
aBuf
[
0
]);
iBuf
++
)
{
tFree
(
pBuilder
->
aBuf
[
iBuf
]);
}
tDiskDataDestroy
(
&
pBuilder
->
dd
);
taosMemoryFree
(
pBuilder
);
return
NULL
;
...
...
@@ -474,6 +489,8 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB
uint8_t
calcSma
)
{
int32_t
code
=
0
;
ASSERT
(
pId
->
suid
||
pId
->
uid
);
pBuilder
->
suid
=
pId
->
suid
;
pBuilder
->
uid
=
pId
->
uid
;
pBuilder
->
nRow
=
0
;
...
...
@@ -512,9 +529,9 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB
}
}
SDiskColBuilder
*
pD
iskCol
Builder
=
(
SDiskColBuilder
*
)
taosArrayGet
(
pBuilder
->
aBuilder
,
pBuilder
->
nBuilder
);
SDiskColBuilder
*
pD
C
Builder
=
(
SDiskColBuilder
*
)
taosArrayGet
(
pBuilder
->
aBuilder
,
pBuilder
->
nBuilder
);
code
=
tDiskColBuilderInit
(
pD
iskCol
Builder
,
pTColumn
->
colId
,
pTColumn
->
type
,
cmprAlg
,
code
=
tDiskColBuilderInit
(
pD
C
Builder
,
pTColumn
->
colId
,
pTColumn
->
type
,
cmprAlg
,
(
calcSma
&&
(
pTColumn
->
flags
&
COL_SMA_ON
)));
if
(
code
)
return
code
;
...
...
@@ -524,14 +541,22 @@ int32_t tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, STSchema *pTSchema, TAB
return
code
;
}
int32_t
tDiskDataBuilderAddRow
(
SDiskDataBuilder
*
pBuilder
,
TSDBROW
*
pRow
,
STSchema
*
pTSchema
,
TABLEID
*
pId
)
{
int32_t
tDiskDataBuilderClear
(
SDiskDataBuilder
*
pBuilder
)
{
int32_t
code
=
0
;
pBuilder
->
suid
=
0
;
pBuilder
->
uid
=
0
;
return
code
;
}
int32_t
tDiskDataAddRow
(
SDiskDataBuilder
*
pBuilder
,
TSDBROW
*
pRow
,
STSchema
*
pTSchema
,
TABLEID
*
pId
)
{
int32_t
code
=
0
;
ASSERT
(
pBuilder
->
suid
||
pBuilder
->
uid
);
ASSERT
(
pId
->
suid
==
pBuilder
->
suid
);
// uid
if
(
pBuilder
->
uid
&&
pBuilder
->
uid
!=
pId
->
uid
)
{
ASSERT
(
!
pBuilder
->
calcSma
);
ASSERT
(
pBuilder
->
suid
);
for
(
int32_t
iRow
=
0
;
iRow
<
pBuilder
->
nRow
;
iRow
++
)
{
code
=
tCompress
(
pBuilder
->
pUidC
,
&
pBuilder
->
uid
,
sizeof
(
int64_t
));
if
(
code
)
return
code
;
...
...
@@ -564,14 +589,13 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche
pColVal
=
tRowIterNext
(
&
iter
);
}
if
(
pColVal
==
NULL
||
pColVal
->
cid
>
pDCBuilder
->
cid
)
{
SColVal
cv
=
COL_VAL_NONE
(
pDCBuilder
->
cid
,
pDCBuilder
->
type
);
code
=
tDiskColAddVal
(
pDCBuilder
,
&
cv
);
if
(
code
)
return
code
;
}
else
{
if
(
pColVal
&&
pColVal
->
cid
==
pDCBuilder
->
cid
)
{
code
=
tDiskColAddVal
(
pDCBuilder
,
pColVal
);
if
(
code
)
return
code
;
pColVal
=
tRowIterNext
(
&
iter
);
}
else
{
code
=
tDiskColAddVal
(
pDCBuilder
,
&
COL_VAL_NONE
(
pDCBuilder
->
cid
,
pDCBuilder
->
type
));
if
(
code
)
return
code
;
}
}
pBuilder
->
nRow
++
;
...
...
@@ -579,11 +603,14 @@ int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSche
return
code
;
}
int32_t
tGnrtDiskData
(
SDiskDataBuilder
*
pBuilder
,
SDiskData
*
pDiskData
)
{
int32_t
tGnrtDiskData
(
SDiskDataBuilder
*
pBuilder
,
const
SDiskData
**
p
pDiskData
)
{
int32_t
code
=
0
;
ASSERT
(
pBuilder
->
nRow
);
*
ppDiskData
=
NULL
;
SDiskData
*
pDiskData
=
&
pBuilder
->
dd
;
// reset SDiskData
pDiskData
->
hdr
=
(
SDiskDataHdr
){.
delimiter
=
TSDB_FILE_DLMT
,
.
fmtVer
=
0
,
...
...
@@ -598,30 +625,32 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) {
pDiskData
->
pUid
=
NULL
;
pDiskData
->
pVer
=
NULL
;
pDiskData
->
pKey
=
NULL
;
if
(
pDiskData
->
aDiskCol
)
{
taosArrayClear
(
pDiskData
->
aDiskCol
);
}
else
{
pDiskData
->
aDiskCol
=
taosArrayInit
(
pBuilder
->
nBuilder
,
sizeof
(
SDiskCol
));
if
(
pDiskData
->
aDiskCol
==
NULL
)
{
code
=
TSDB_CODE_OUT_OF_MEMORY
;
return
code
;
}
}
// UID
if
(
pBuilder
->
uid
==
0
)
{
code
=
tCompressEnd
(
pBuilder
->
pUidC
,
&
pDiskData
->
pUid
,
&
pDiskData
->
hdr
.
szUid
);
code
=
tCompressEnd
(
pBuilder
->
pUidC
,
&
pDiskData
->
pUid
,
&
pDiskData
->
hdr
.
szUid
,
NULL
);
if
(
code
)
return
code
;
}
// VERSION
code
=
tCompressEnd
(
pBuilder
->
pVerC
,
&
pDiskData
->
pVer
,
&
pDiskData
->
hdr
.
szVer
);
code
=
tCompressEnd
(
pBuilder
->
pVerC
,
&
pDiskData
->
pVer
,
&
pDiskData
->
hdr
.
szVer
,
NULL
);
if
(
code
)
return
code
;
// TSKEY
code
=
tCompressEnd
(
pBuilder
->
pKeyC
,
&
pDiskData
->
pKey
,
&
pDiskData
->
hdr
.
szKey
);
code
=
tCompressEnd
(
pBuilder
->
pKeyC
,
&
pDiskData
->
pKey
,
&
pDiskData
->
hdr
.
szKey
,
NULL
);
if
(
code
)
return
code
;
// aDiskCol
if
(
pDiskData
->
aDiskCol
)
{
taosArrayClear
(
pDiskData
->
aDiskCol
);
}
else
{
pDiskData
->
aDiskCol
=
taosArrayInit
(
pBuilder
->
nBuilder
,
sizeof
(
SDiskCol
));
if
(
pDiskData
->
aDiskCol
==
NULL
)
{
code
=
TSDB_CODE_OUT_OF_MEMORY
;
return
code
;
}
}
int32_t
offset
=
0
;
for
(
int32_t
iBuilder
=
0
;
iBuilder
<
pBuilder
->
nBuilder
;
iBuilder
++
)
{
SDiskColBuilder
*
pDCBuilder
=
(
SDiskColBuilder
*
)
taosArrayGet
(
pBuilder
->
aBuilder
,
iBuilder
);
...
...
@@ -644,12 +673,6 @@ int32_t tGnrtDiskData(SDiskDataBuilder *pBuilder, SDiskData *pDiskData) {
pDiskData
->
hdr
.
szBlkCol
+=
tPutBlockCol
(
NULL
,
&
dCol
.
bCol
);
}
return
code
;
}
// SDiskData ================================================
int32_t
tDiskDataDestroy
(
SDiskData
*
pDiskData
)
{
int32_t
code
=
0
;
pDiskData
->
aDiskCol
=
taosArrayDestroy
(
pDiskData
->
aDiskCol
);
*
ppDiskData
=
pDiskData
;
return
code
;
}
source/util/src/tcompression.c
浏览文件 @
a4d16f1c
...
...
@@ -2000,16 +2000,23 @@ int32_t tCompressStart(SCompressor *pCmprsor, int8_t type, int8_t cmprAlg) {
return
code
;
}
int32_t
tCompressEnd
(
SCompressor
*
pCmprsor
,
const
uint8_t
**
pp
Data
,
int32_t
*
nData
)
{
int32_t
tCompressEnd
(
SCompressor
*
pCmprsor
,
const
uint8_t
**
pp
Out
,
int32_t
*
nOut
,
int32_t
*
nOrigin
)
{
int32_t
code
=
0
;
*
ppData
=
NULL
;
*
nData
=
0
;
*
ppOut
=
NULL
;
*
nOut
=
0
;
if
(
nOrigin
)
{
if
(
DATA_TYPE_INFO
[
pCmprsor
->
type
].
isVarLen
)
{
*
nOrigin
=
pCmprsor
->
nBuf
-
1
;
}
else
{
*
nOrigin
=
pCmprsor
->
nVal
*
DATA_TYPE_INFO
[
pCmprsor
->
type
].
bytes
;
}
}
if
(
pCmprsor
->
nVal
==
0
)
return
code
;
if
(
DATA_TYPE_INFO
[
pCmprsor
->
type
].
endFn
)
{
return
DATA_TYPE_INFO
[
pCmprsor
->
type
].
endFn
(
pCmprsor
,
pp
Data
,
nData
);
return
DATA_TYPE_INFO
[
pCmprsor
->
type
].
endFn
(
pCmprsor
,
pp
Out
,
nOut
);
}
return
code
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录