Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
7e458785
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1181
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
7e458785
编写于
7月 15, 2022
作者:
H
Hongze Cheng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
more vnode snapshot
上级
74b31e8c
变更
1
隐藏空白更改
内联
并排
Showing
1 changed file
with
159 addition
and
137 deletion
+159
-137
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
+159
-137
未找到文件。
source/dnode/vnode/src/tsdb/tsdbSnapshot.c
浏览文件 @
7e458785
...
...
@@ -488,52 +488,42 @@ _err:
return
code
;
}
static
int32_t
tsdbSnap
WriteTableDataAhead
(
STsdbSnapWriter
*
pWriter
,
TABLEID
id
)
{
static
int32_t
tsdbSnap
MoveWriteTableData
(
STsdbSnapWriter
*
pWriter
,
SBlockIdx
*
pBlockIdx
)
{
int32_t
code
=
0
;
if
(
pWriter
->
pDataFReader
==
NULL
)
goto
_exit
;
while
(
true
)
{
if
(
pWriter
->
iBlockIdx
>=
taosArrayGetSize
(
pWriter
->
aBlockIdx
))
break
;
SBlockIdx
*
pBlockIdx
=
(
SBlockIdx
*
)
taosArrayGet
(
pWriter
->
aBlockIdx
,
pWriter
->
iBlockIdx
);
int32_t
c
=
tTABLEIDCmprFn
(
pBlockIdx
,
&
id
);
if
(
c
>=
0
)
break
;
pWriter
->
iBlockIdx
++
;
code
=
tsdbReadBlock
(
pWriter
->
pDataFReader
,
pBlockIdx
,
&
pWriter
->
mBlock
,
NULL
);
if
(
code
)
goto
_err
;
SBlock
block
;
tMapDataReset
(
&
pWriter
->
mBlockW
);
for
(
int32_t
iBlock
=
0
;
iBlock
<
pWriter
->
mBlock
.
nItem
;
iBlock
++
)
{
tMapDataGetItemByIdx
(
&
pWriter
->
mBlock
,
iBlock
,
&
block
,
tGetBlock
);
code
=
tsdbReadBlock
(
pWriter
->
pDataFReader
,
pBlockIdx
,
&
pWriter
->
mBlock
,
NULL
);
if
(
code
)
goto
_err
;
if
(
block
.
last
)
{
code
=
tsdbReadBlockData
(
pWriter
->
pDataFReader
,
pBlockIdx
,
&
block
,
&
pWriter
->
bDataR
,
NULL
,
NULL
);
if
(
code
)
goto
_err
;
// SBlockData
SBlock
block
;
tMapDataReset
(
&
pWriter
->
mBlockW
);
for
(
int32_t
iBlock
=
0
;
iBlock
<
pWriter
->
mBlock
.
nItem
;
iBlock
++
)
{
tMapDataGetItemByIdx
(
&
pWriter
->
mBlock
,
iBlock
,
&
block
,
tGetBlock
);
tBlockReset
(
&
block
);
block
.
last
=
1
;
code
=
tsdbWriteBlockData
(
pWriter
->
pDataFWriter
,
&
pWriter
->
bDataR
,
NULL
,
NULL
,
pBlockIdx
,
&
block
,
pWriter
->
cmprAlg
);
if
(
code
)
goto
_err
;
}
if
(
block
.
last
)
{
code
=
tsdbReadBlockData
(
pWriter
->
pDataFReader
,
pBlockIdx
,
&
block
,
&
pWriter
->
bDataR
,
NULL
,
NULL
);
if
(
code
)
goto
_err
;
code
=
tMapDataPutItem
(
&
pWriter
->
mBlockW
,
&
block
,
tPutBlock
);
tBlockReset
(
&
block
);
block
.
last
=
1
;
code
=
tsdbWriteBlockData
(
pWriter
->
pDataFWriter
,
&
pWriter
->
bDataR
,
NULL
,
NULL
,
pBlockIdx
,
&
block
,
pWriter
->
cmprAlg
);
if
(
code
)
goto
_err
;
}
SBlockIdx
blockIdx
=
{.
suid
=
pBlockIdx
->
suid
,
.
uid
=
pBlockIdx
->
uid
};
code
=
tsdbWriteBlock
(
pWriter
->
pDataFWriter
,
&
pWriter
->
mBlockW
,
NULL
,
&
blockIdx
);
code
=
tMapDataPutItem
(
&
pWriter
->
mBlockW
,
&
block
,
tPutBlock
);
if
(
code
)
goto
_err
;
}
if
(
taosArrayPush
(
pWriter
->
aBlockIdxW
,
&
blockIdx
)
==
NULL
)
{
code
=
TSDB_CODE_OUT_OF_MEMORY
;
goto
_err
;
}
// SBlock
SBlockIdx
blockIdx
=
{.
suid
=
pBlockIdx
->
suid
,
.
uid
=
pBlockIdx
->
uid
};
code
=
tsdbWriteBlock
(
pWriter
->
pDataFWriter
,
&
pWriter
->
mBlockW
,
NULL
,
&
blockIdx
);
if
(
code
)
goto
_err
;
// SBlockIdx
if
(
taosArrayPush
(
pWriter
->
aBlockIdxW
,
&
blockIdx
)
==
NULL
)
{
code
=
TSDB_CODE_OUT_OF_MEMORY
;
goto
_err
;
}
_exit:
...
...
@@ -543,77 +533,19 @@ _err:
return
code
;
}
static
int32_t
tsdbSnapWrite
DataImpl
(
STsdbSnapWriter
*
pWriter
,
TABLEID
id
)
{
static
int32_t
tsdbSnapWrite
TableDataImpl
(
STsdbSnapWriter
*
pWriter
)
{
int32_t
code
=
0
;
SBlockData
*
pBlockData
=
&
pWriter
->
bData
;
TSDBKEY
keyFirst
=
tBlockDataFirstKey
(
pBlockData
);
TSDBKEY
keyLast
=
tBlockDataLastKey
(
pBlockData
);
// TABLE ====================================
// end last table write if should
if
(
pWriter
->
pBlockIdxW
)
{
int32_t
c
=
tTABLEIDCmprFn
(
pWriter
->
pBlockIdxW
,
&
id
);
if
(
c
<
0
)
{
// end
code
=
tsdbSnapWriteTableDataEnd
(
pWriter
);
if
(
code
)
goto
_err
;
int32_t
iRow
=
0
;
TSDBROW
row
;
TSDBROW
*
pRow
=
&
row
;
// reset
pWriter
->
pBlockIdxW
=
NULL
;
}
else
if
(
c
>
0
)
{
ASSERT
(
0
);
}
}
// start new table data write if need
if
(
pWriter
->
pBlockIdxW
==
NULL
)
{
// write table data ahead
code
=
tsdbSnapWriteTableDataAhead
(
pWriter
,
id
);
if
(
code
)
goto
_err
;
// reader
if
(
pWriter
->
iBlockIdx
<
taosArrayGetSize
(
pWriter
->
aBlockIdx
))
{
ASSERT
(
pWriter
->
pDataFReader
);
pWriter
->
pBlockIdx
=
(
SBlockIdx
*
)
taosArrayGet
(
pWriter
->
aBlockIdx
,
pWriter
->
iBlock
);
int32_t
c
=
tTABLEIDCmprFn
(
pWriter
->
pBlockIdx
,
&
id
);
if
(
c
)
{
ASSERT
(
c
>
0
);
pWriter
->
pBlockIdx
=
NULL
;
}
else
{
pWriter
->
iBlockIdx
++
;
}
}
else
{
pWriter
->
pBlockIdx
=
NULL
;
}
if
(
pWriter
->
pBlockIdx
)
{
code
=
tsdbReadBlock
(
pWriter
->
pDataFReader
,
pWriter
->
pBlockIdx
,
&
pWriter
->
mBlock
,
NULL
);
if
(
code
)
goto
_err
;
}
else
{
tMapDataReset
(
&
pWriter
->
mBlock
);
}
pWriter
->
iBlock
=
0
;
pWriter
->
pBlockData
=
NULL
;
pWriter
->
iRow
=
0
;
// writer
pWriter
->
pBlockIdxW
=
&
pWriter
->
blockIdxW
;
pWriter
->
pBlockIdxW
->
suid
=
id
.
suid
;
pWriter
->
pBlockIdxW
->
uid
=
id
.
uid
;
tBlockReset
(
&
pWriter
->
blockW
);
tBlockDataReset
(
&
pWriter
->
bDataW
);
tMapDataReset
(
&
pWriter
->
mBlockW
);
}
ASSERT
(
pWriter
->
pBlockIdxW
&&
pWriter
->
pBlockIdxW
->
suid
==
id
.
suid
&&
pWriter
->
pBlockIdxW
->
uid
==
id
.
uid
);
ASSERT
(
pWriter
->
pBlockIdx
==
NULL
||
(
pWriter
->
pBlockIdx
->
suid
==
id
.
suid
&&
pWriter
->
pBlockIdx
->
uid
==
id
.
uid
));
// correct schema
code
=
tBlockDataCorrectSchema
(
&
pWriter
->
bDataW
,
pBlockData
);
if
(
code
)
goto
_err
;
// BLOCK ====================================
int32_t
iRow
=
0
;
TSDBROW
*
pRow
=
&
tsdbRowFromBlockData
(
pBlockData
,
iRow
);
// loop to merge
*
pRow
=
tsdbRowFromBlockData
(
pBlockData
,
iRow
);
while
(
true
)
{
if
(
pRow
==
NULL
)
break
;
...
...
@@ -630,7 +562,7 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
iRow
++
;
if
(
iRow
<
pWriter
->
pBlockData
->
nRow
)
{
pRow
=
&
tsdbRowFromBlockData
(
pBlockData
,
iRow
);
*
pRow
=
tsdbRowFromBlockData
(
pBlockData
,
iRow
);
}
else
{
pRow
=
NULL
;
}
...
...
@@ -644,55 +576,60 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
}
}
}
else
{
SBlock
block
;
TSDBKEY
key
=
TSDBROW_KEY
(
pRow
)
;
if
(
pWriter
->
iBlock
<
pWriter
->
mBlock
.
nItem
)
{
tMapDataGetItemByIdx
(
&
pWriter
->
mBlock
,
pWriter
->
iBlock
,
&
block
,
tGetBlock
);
while
(
true
)
{
if
(
pWriter
->
iBlock
>=
pWriter
->
mBlock
.
nItem
)
break
;
SBlock
block
;
int32_t
c
;
c
=
tsdbKeyCmprFn
(
&
block
.
maxKey
,
&
TSDBROW_KEY
(
pRow
)
);
tMapDataGetItemByIdx
(
&
pWriter
->
mBlock
,
pWriter
->
iBlock
,
&
block
,
tGetBlock
);
ASSERT
(
c
);
if
(
block
.
last
)
{
pWriter
->
pBlockData
=
&
pWriter
->
bDataR
;
if
(
c
<
0
)
{
code
=
tMapDataPutItem
(
&
pWriter
->
mBlockW
,
&
block
,
tPutBlock
);
code
=
tsdbReadBlockData
(
pWriter
->
pDataFReader
,
pWriter
->
pBlockIdx
,
&
block
,
pWriter
->
pBlockData
,
NULL
,
NULL
);
if
(
code
)
goto
_err
;
pWriter
->
iRow
=
0
;
pWriter
->
iBlock
++
;
continue
;
break
;
}
c
=
tsdbKeyCmprFn
(
&
block
.
m
inKey
,
&
TSDBROW_KEY
(
pRow
)
);
c
=
tsdbKeyCmprFn
(
&
block
.
m
axKey
,
&
key
);
ASSERT
(
c
);
if
(
c
>
0
)
{
code
=
tBlockDataAppendRow
(
&
pWriter
->
bDataW
,
pRow
,
NULL
);
if
(
c
>
0
)
break
;
if
(
pWriter
->
bDataW
.
nRow
)
{
pWriter
->
blockW
.
last
=
0
;
code
=
tsdbWriteBlockData
(
pWriter
->
pDataFWriter
,
&
pWriter
->
bDataW
,
NULL
,
NULL
,
pWriter
->
pBlockIdxW
,
&
pWriter
->
blockW
,
pWriter
->
cmprAlg
);
if
(
code
)
goto
_err
;
iRow
++
;
if
(
iRow
<
pWriter
->
pBlockData
->
nRow
)
{
pRow
=
&
tsdbRowFromBlockData
(
pBlockData
,
iRow
);
}
else
{
pRow
=
NULL
;
}
code
=
tMapDataPutItem
(
&
pWriter
->
mBlockW
,
&
pWriter
->
blockW
,
tPutBlock
);
if
(
code
)
goto
_err
;
goto
_check_write
;
tBlockReset
(
&
pWriter
->
blockW
);
tBlockDataClearData
(
&
pWriter
->
bDataW
);
}
pWriter
->
pBlockData
=
&
pWriter
->
bDataR
;
code
=
tsdbReadBlockData
(
pWriter
->
pDataFReader
,
pWriter
->
pBlockIdx
,
&
block
,
pWriter
->
pBlockData
,
NULL
,
NULL
);
code
=
tMapDataPutItem
(
&
pWriter
->
mBlockW
,
&
block
,
tPutBlock
);
if
(
code
)
goto
_err
;
pWriter
->
i
Row
=
0
;
pWriter
->
i
Block
++
;
}
if
(
pWriter
->
pBlockData
)
continue
;
code
=
tBlockDataAppendRow
(
&
pWriter
->
bDataW
,
pRow
,
NULL
);
if
(
code
)
goto
_err
;
iRow
++
;
if
(
iRow
<
p
Writer
->
p
BlockData
->
nRow
)
{
pRow
=
&
tsdbRowFromBlockData
(
pBlockData
,
iRow
);
if
(
iRow
<
pBlockData
->
nRow
)
{
*
pRow
=
tsdbRowFromBlockData
(
pBlockData
,
iRow
);
}
else
{
pRow
=
NULL
;
}
...
...
@@ -713,6 +650,91 @@ static int32_t tsdbSnapWriteDataImpl(STsdbSnapWriter* pWriter, TABLEID id) {
tBlockDataClearData
(
&
pWriter
->
bDataW
);
}
return
code
;
_err:
return
code
;
}
static
int32_t
tsdbSnapWriteTableData
(
STsdbSnapWriter
*
pWriter
,
TABLEID
id
)
{
int32_t
code
=
0
;
SBlockData
*
pBlockData
=
&
pWriter
->
bData
;
TSDBKEY
keyFirst
=
tBlockDataFirstKey
(
pBlockData
);
TSDBKEY
keyLast
=
tBlockDataLastKey
(
pBlockData
);
// end last table write if should
if
(
pWriter
->
pBlockIdxW
)
{
int32_t
c
=
tTABLEIDCmprFn
(
pWriter
->
pBlockIdxW
,
&
id
);
if
(
c
<
0
)
{
// end
code
=
tsdbSnapWriteTableDataEnd
(
pWriter
);
if
(
code
)
goto
_err
;
// reset
pWriter
->
pBlockIdxW
=
NULL
;
}
else
if
(
c
>
0
)
{
ASSERT
(
0
);
}
}
// start new table data write if need
if
(
pWriter
->
pBlockIdxW
==
NULL
)
{
// write table data ahead
while
(
true
)
{
if
(
pWriter
->
iBlockIdx
>=
taosArrayGetSize
(
pWriter
->
aBlockIdx
))
break
;
SBlockIdx
*
pBlockIdx
=
(
SBlockIdx
*
)
taosArrayGet
(
pWriter
->
aBlockIdx
,
pWriter
->
iBlockIdx
);
int32_t
c
=
tTABLEIDCmprFn
(
pBlockIdx
,
&
id
);
if
(
c
>=
0
)
break
;
code
=
tsdbSnapMoveWriteTableData
(
pWriter
,
pBlockIdx
);
if
(
code
)
goto
_err
;
pWriter
->
iBlockIdx
++
;
}
// reader
if
(
pWriter
->
iBlockIdx
<
taosArrayGetSize
(
pWriter
->
aBlockIdx
))
{
ASSERT
(
pWriter
->
pDataFReader
);
SBlockIdx
*
pBlockIdx
=
(
SBlockIdx
*
)
taosArrayGet
(
pWriter
->
aBlockIdx
,
pWriter
->
iBlock
);
int32_t
c
=
tTABLEIDCmprFn
(
pBlockIdx
,
&
id
);
ASSERT
(
c
>=
0
);
if
(
c
==
0
)
{
pWriter
->
pBlockIdx
=
pBlockIdx
;
pWriter
->
iBlockIdx
++
;
}
}
if
(
pWriter
->
pBlockIdx
)
{
code
=
tsdbReadBlock
(
pWriter
->
pDataFReader
,
pWriter
->
pBlockIdx
,
&
pWriter
->
mBlock
,
NULL
);
if
(
code
)
goto
_err
;
}
else
{
tMapDataReset
(
&
pWriter
->
mBlock
);
}
pWriter
->
iBlock
=
0
;
pWriter
->
pBlockData
=
NULL
;
pWriter
->
iRow
=
0
;
// writer
pWriter
->
pBlockIdxW
=
&
pWriter
->
blockIdxW
;
pWriter
->
pBlockIdxW
->
suid
=
id
.
suid
;
pWriter
->
pBlockIdxW
->
uid
=
id
.
uid
;
tBlockReset
(
&
pWriter
->
blockW
);
tBlockDataReset
(
&
pWriter
->
bDataW
);
tMapDataReset
(
&
pWriter
->
mBlockW
);
}
ASSERT
(
pWriter
->
pBlockIdxW
&&
pWriter
->
pBlockIdxW
->
suid
==
id
.
suid
&&
pWriter
->
pBlockIdxW
->
uid
==
id
.
uid
);
ASSERT
(
pWriter
->
pBlockIdx
==
NULL
||
(
pWriter
->
pBlockIdx
->
suid
==
id
.
suid
&&
pWriter
->
pBlockIdx
->
uid
==
id
.
uid
));
code
=
tsdbSnapWriteTableDataImpl
(
pWriter
);
if
(
code
)
goto
_err
;
_exit:
return
code
;
...
...
@@ -723,11 +745,10 @@ _err:
}
static
int32_t
tsdbSnapWriteData
(
STsdbSnapWriter
*
pWriter
,
uint8_t
*
pData
,
uint32_t
nData
)
{
int32_t
code
=
0
;
STsdb
*
pTsdb
=
pWriter
->
pTsdb
;
SSnapDataHdr
*
pHdr
=
(
SSnapDataHdr
*
)
pData
;
TABLEID
id
=
*
(
TABLEID
*
)(
&
pHdr
[
1
]);
int64_t
n
;
int32_t
code
=
0
;
STsdb
*
pTsdb
=
pWriter
->
pTsdb
;
TABLEID
id
=
*
(
TABLEID
*
)(
pData
+
sizeof
(
SSnapDataHdr
));
int64_t
n
;
// decode
SBlockData
*
pBlockData
=
&
pWriter
->
bData
;
...
...
@@ -742,13 +763,13 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
ASSERT
(
fid
==
tsdbKeyFid
(
keyLast
.
ts
,
pWriter
->
minutes
,
pWriter
->
precision
));
if
(
pWriter
->
pDataFWriter
==
NULL
||
pWriter
->
fid
!=
fid
)
{
// end last file data write if need
code
=
tsdbSnapWriteDataEnd
(
pWriter
);
// todo
code
=
tsdbSnapWriteDataEnd
(
pWriter
);
if
(
code
)
goto
_err
;
pWriter
->
fid
=
fid
;
// read
SDFileSet
*
pSet
=
tsdbFSStateGetDFileSet
(
pTsdb
->
fs
->
nState
,
fid
,
TD_EQ
);
// todo: check nState is valid
SDFileSet
*
pSet
=
tsdbFSStateGetDFileSet
(
pTsdb
->
fs
->
nState
,
fid
,
TD_EQ
);
if
(
pSet
)
{
code
=
tsdbDataFReaderOpen
(
&
pWriter
->
pDataFReader
,
pTsdb
,
pSet
);
if
(
code
)
goto
_err
;
...
...
@@ -763,8 +784,9 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
pWriter
->
pBlockIdx
=
NULL
;
tMapDataReset
(
&
pWriter
->
mBlock
);
pWriter
->
iBlock
=
0
;
tBlockDataReset
(
&
pWriter
->
bDataR
)
;
pWriter
->
pBlockData
=
NULL
;
pWriter
->
iRow
=
0
;
tBlockDataReset
(
&
pWriter
->
bDataR
);
// write
SDFileSet
wSet
;
...
...
@@ -789,12 +811,12 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
if
(
code
)
goto
_err
;
taosArrayClear
(
pWriter
->
aBlockIdxW
);
pWriter
->
pBlockIdxW
=
NULL
;
tMapDataReset
(
&
pWriter
->
mBlockW
);
pWriter
->
pBlockIdxW
=
NULL
;
tBlockDataReset
(
&
pWriter
->
bDataW
);
}
code
=
tsdbSnapWrite
DataImpl
(
pWriter
,
id
);
code
=
tsdbSnapWrite
TableData
(
pWriter
,
id
);
if
(
code
)
goto
_err
;
tsdbInfo
(
"vgId:%d vnode snapshot tsdb write data, fid:%d suid:%"
PRId64
" uid:%"
PRId64
" nRow:%d"
,
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录