Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
0359254b
T
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1181
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
0359254b
编写于
6月 20, 2022
作者:
H
Hongze Cheng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
more work
上级
607f698f
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
176 addition
and
41 deletion
+176
-41
source/dnode/vnode/src/inc/tsdb.h
source/dnode/vnode/src/inc/tsdb.h
+1
-0
source/dnode/vnode/src/tsdb/tsdbCommit.c
source/dnode/vnode/src/tsdb/tsdbCommit.c
+24
-13
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
+123
-26
source/dnode/vnode/src/tsdb/tsdbUtil.c
source/dnode/vnode/src/tsdb/tsdbUtil.c
+28
-2
未找到文件。
source/dnode/vnode/src/inc/tsdb.h
浏览文件 @
0359254b
...
...
@@ -101,6 +101,7 @@ int32_t tPutBlock(uint8_t *p, void *ph);
int32_t
tGetBlock
(
uint8_t
*
p
,
void
*
ph
);
int32_t
tBlockCmprFn
(
const
void
*
p1
,
const
void
*
p2
);
// SBlockIdx
#define tBlockIdxInit(SUID, UID) ((SBlockIdx){.suid = (SUID), .uid = (UID), .info = tKEYINFOInit()})
int32_t
tPutBlockIdx
(
uint8_t
*
p
,
void
*
ph
);
int32_t
tGetBlockIdx
(
uint8_t
*
p
,
void
*
ph
);
// SColdata
...
...
source/dnode/vnode/src/tsdb/tsdbCommit.c
浏览文件 @
0359254b
...
...
@@ -463,7 +463,8 @@ _err:
return
code
;
}
static
int32_t
tsdbMergeCommit
(
SCommitter
*
pCommitter
,
SBlockIdx
*
pBlockIdx
,
STbDataIter
*
pIter
,
SBlock
*
pBlock
)
{
static
int32_t
tsdbMergeCommit
(
SCommitter
*
pCommitter
,
SBlockIdx
*
pBlockIdx
,
STbDataIter
*
pIter
,
SBlock
*
pBlock
,
int8_t
isLastBlock
)
{
int32_t
code
=
0
;
TSDBROW
*
pRow
;
SBlock
block
=
tBlockInit
();
...
...
@@ -523,12 +524,18 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData, SBl
STbDataIter
iter
;
STbDataIter
*
pIter
=
&
iter
;
TSDBROW
*
pRow
;
SBlockIdx
blockIdx
;
// TODO
int64_t
suid
;
int64_t
uid
;
SBlockIdx
blockIdx
;
// create iter
if
(
pTbData
)
{
suid
=
pTbData
->
suid
;
uid
=
pTbData
->
uid
;
tsdbTbDataIterOpen
(
pTbData
,
&
(
TSDBKEY
){.
ts
=
pCommitter
->
minKey
,
.
version
=
0
},
0
,
pIter
);
}
else
{
suid
=
pBlockIdx
->
suid
;
uid
=
pBlockIdx
->
uid
;
pIter
=
NULL
;
}
...
...
@@ -538,23 +545,27 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData, SBl
// start ================================
tMapDataReset
(
&
pCommitter
->
oBlockMap
);
tMapDataReset
(
&
pCommitter
->
nBlockMap
);
tBlockReset
(
&
pCommitter
->
oBlock
);
tBlockDataReset
(
&
pCommitter
->
oBlockData
);
if
(
pBlockIdx
)
{
code
=
tsdbReadBlock
(
pCommitter
->
pReader
,
pBlockIdx
,
&
pCommitter
->
oBlockMap
,
NULL
);
if
(
code
)
goto
_err
;
}
blockIdx
=
tBlockIdxInit
(
suid
,
uid
);
tMapDataReset
(
&
pCommitter
->
nBlockMap
);
tBlockReset
(
&
pCommitter
->
nBlock
);
tBlockDataReset
(
&
pCommitter
->
nBlockData
);
// impl ===============================
SBlock
block
;
SBlock
*
pBlock
=
&
block
;
int32_t
iBlock
=
0
;
int32_t
nBlock
Map
=
pCommitter
->
oBlockMap
.
nItem
;
int32_t
nBlock
=
pCommitter
->
oBlockMap
.
nItem
;
// merge
pRow
=
tsdbTbDataIterGet
(
pIter
);
while
(
!
ROW_END
(
pRow
,
pCommitter
->
maxKey
)
&&
iBlock
<
nBlock
Map
)
{
tMapDataGetItemByIdx
(
&
pCommitter
->
oBlockMap
,
iBlock
,
p
Block
,
tGetBlock
);
code
=
tsdbMergeCommit
(
pCommitter
,
&
blockIdx
,
pIter
,
pBlock
);
while
(
!
ROW_END
(
pRow
,
pCommitter
->
maxKey
)
&&
iBlock
<
nBlock
)
{
tMapDataGetItemByIdx
(
&
pCommitter
->
oBlockMap
,
iBlock
,
&
pCommitter
->
o
Block
,
tGetBlock
);
code
=
tsdbMergeCommit
(
pCommitter
,
&
blockIdx
,
pIter
,
&
pCommitter
->
oBlock
,
iBlock
==
(
nBlock
-
1
)
);
if
(
code
)
goto
_err
;
pRow
=
tsdbTbDataIterGet
(
pIter
);
...
...
@@ -564,17 +575,17 @@ static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData, SBl
// mem
pRow
=
tsdbTbDataIterGet
(
pIter
);
while
(
!
ROW_END
(
pRow
,
pCommitter
->
maxKey
))
{
code
=
tsdbMergeCommit
(
pCommitter
,
&
blockIdx
,
pIter
,
NULL
);
code
=
tsdbMergeCommit
(
pCommitter
,
&
blockIdx
,
pIter
,
NULL
,
0
);
if
(
code
)
goto
_err
;
pRow
=
tsdbTbDataIterGet
(
pIter
);
}
// disk
while
(
iBlock
<
nBlock
Map
)
{
tMapDataGetItemByIdx
(
&
pCommitter
->
oBlockMap
,
iBlock
,
p
Block
,
tGetBlock
);
while
(
iBlock
<
nBlock
)
{
tMapDataGetItemByIdx
(
&
pCommitter
->
oBlockMap
,
iBlock
,
&
pCommitter
->
o
Block
,
tGetBlock
);
code
=
tsdbMergeCommit
(
pCommitter
,
&
blockIdx
,
NULL
,
pBlock
);
code
=
tsdbMergeCommit
(
pCommitter
,
&
blockIdx
,
NULL
,
&
pCommitter
->
oBlock
,
0
);
if
(
code
)
goto
_err
;
iBlock
++
;
...
...
source/dnode/vnode/src/tsdb/tsdbReaderWriter.c
浏览文件 @
0359254b
...
...
@@ -489,17 +489,17 @@ _err:
return
code
;
}
int32_t
tsdbReadBlock
(
SDataFReader
*
pReader
,
SBlockIdx
*
pBlockIdx
,
SMapData
*
mBlock
Idx
,
uint8_t
**
ppBuf
)
{
int32_t
tsdbReadBlock
(
SDataFReader
*
pReader
,
SBlockIdx
*
pBlockIdx
,
SMapData
*
mBlock
,
uint8_t
**
ppBuf
)
{
int32_t
code
=
0
;
int64_t
offset
=
pBlockIdx
->
offset
;
int64_t
size
=
pBlockIdx
->
size
;
int64_t
n
;
uint32_t
delimiter
;
tb_uid_t
suid
;
tb_uid_t
uid
;
int64_t
suid
;
int64_t
uid
;
// alloc
if
(
!
ppBuf
)
ppBuf
=
&
mBlock
Idx
->
pBuf
;
if
(
!
ppBuf
)
ppBuf
=
&
mBlock
->
pBuf
;
code
=
tsdbRealloc
(
ppBuf
,
size
);
if
(
code
)
goto
_err
;
...
...
@@ -533,7 +533,7 @@ int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBl
ASSERT
(
suid
==
pBlockIdx
->
suid
);
n
+=
tGetI64
(
*
ppBuf
+
n
,
&
uid
);
ASSERT
(
uid
==
pBlockIdx
->
uid
);
n
+=
tGetMapData
(
*
ppBuf
+
n
,
mBlock
Idx
);
n
+=
tGetMapData
(
*
ppBuf
+
n
,
mBlock
);
ASSERT
(
n
+
sizeof
(
TSCKSUM
)
==
size
);
return
code
;
...
...
@@ -625,20 +625,112 @@ _err:
}
int32_t
tsdbUpdateDFileSetHeader
(
SDataFWriter
*
pWriter
,
uint8_t
**
ppBuf
)
{
int32_t
code
=
0
;
// TODO
int32_t
code
=
0
;
int64_t
size
=
TSDB_FHDR_SIZE
;
int64_t
n
;
uint8_t
*
pBuf
=
NULL
;
SHeadFile
*
pHeadFile
=
pWriter
->
pSet
->
pHeadFile
;
SDataFile
*
pDataFile
=
pWriter
->
pSet
->
pDataFile
;
SLastFile
*
pLastFile
=
pWriter
->
pSet
->
pLastFile
;
SSmaFile
*
pSmaFile
=
pWriter
->
pSet
->
pSmaFile
;
// alloc
if
(
!
ppBuf
)
ppBuf
=
&
pBuf
;
code
=
tsdbRealloc
(
ppBuf
,
size
);
if
(
code
)
goto
_err
;
// head ==============
// build
memset
(
*
ppBuf
,
0
,
size
);
// tPutHeadFileHdr(*ppBuf, pHeadFile);
taosCalcChecksumAppend
(
0
,
*
ppBuf
,
size
);
// seek
if
(
taosLSeekFile
(
pWriter
->
pHeadFD
,
0
,
SEEK_SET
)
<
0
)
{
code
=
TAOS_SYSTEM_ERROR
(
errno
);
goto
_err
;
}
// write
n
=
taosWriteFile
(
pWriter
->
pHeadFD
,
*
ppBuf
,
size
);
if
(
n
<
0
)
{
code
=
TAOS_SYSTEM_ERROR
(
errno
);
goto
_err
;
}
// data ==============
memset
(
*
ppBuf
,
0
,
size
);
// tPutDataFileHdr(*ppBuf, pDataFile);
taosCalcChecksumAppend
(
0
,
*
ppBuf
,
size
);
// seek
if
(
taosLSeekFile
(
pWriter
->
pDataFD
,
0
,
SEEK_SET
)
<
0
)
{
code
=
TAOS_SYSTEM_ERROR
(
errno
);
goto
_err
;
}
// write
n
=
taosWriteFile
(
pWriter
->
pDataFD
,
*
ppBuf
,
size
);
if
(
n
<
0
)
{
code
=
TAOS_SYSTEM_ERROR
(
errno
);
goto
_err
;
}
// last ==============
memset
(
*
ppBuf
,
0
,
size
);
// tPutLastFileHdr(*ppBuf, pLastFile);
taosCalcChecksumAppend
(
0
,
*
ppBuf
,
size
);
// seek
if
(
taosLSeekFile
(
pWriter
->
pLastFD
,
0
,
SEEK_SET
)
<
0
)
{
code
=
TAOS_SYSTEM_ERROR
(
errno
);
goto
_err
;
}
// write
n
=
taosWriteFile
(
pWriter
->
pLastFD
,
*
ppBuf
,
size
);
if
(
n
<
0
)
{
code
=
TAOS_SYSTEM_ERROR
(
errno
);
goto
_err
;
}
// sma ==============
memset
(
*
ppBuf
,
0
,
size
);
// tPutSmaFileHdr(*ppBuf, pSmaFile);
taosCalcChecksumAppend
(
0
,
*
ppBuf
,
size
);
// seek
if
(
taosLSeekFile
(
pWriter
->
pSmaFD
,
0
,
SEEK_SET
)
<
0
)
{
code
=
TAOS_SYSTEM_ERROR
(
errno
);
goto
_err
;
}
// write
n
=
taosWriteFile
(
pWriter
->
pSmaFD
,
*
ppBuf
,
size
);
if
(
n
<
0
)
{
code
=
TAOS_SYSTEM_ERROR
(
errno
);
goto
_err
;
}
tsdbFree
(
pBuf
);
return
code
;
_err:
tsdbFree
(
pBuf
);
tsdbError
(
"vgId:%d update DFileSet header failed since %s"
,
TD_VID
(
pWriter
->
pTsdb
->
pVnode
),
tstrerror
(
code
));
return
code
;
}
int32_t
tsdbWriteBlockIdx
(
SDataFWriter
*
pWriter
,
SMapData
*
pBlockIdxMap
,
uint8_t
**
ppBuf
)
{
int32_t
code
=
0
;
int64_t
size
=
0
;
int64_t
n
=
0
;
uint8_t
*
pBuf
=
NULL
;
int32_t
tsdbWriteBlockIdx
(
SDataFWriter
*
pWriter
,
SMapData
*
mBlockIdx
,
uint8_t
**
ppBuf
)
{
int32_t
code
=
0
;
int64_t
size
=
0
;
SHeadFile
*
pHeadFile
=
pWriter
->
pSet
->
pHeadFile
;
int64_t
n
=
0
;
uint8_t
*
pBuf
=
NULL
;
// prepare
size
+=
tPutU32
(
NULL
,
TSDB_FILE_DLMT
);
size
=
size
+
tPutMapData
(
NULL
,
pBlockIdxMap
)
+
sizeof
(
TSCKSUM
);
size
=
size
+
tPutMapData
(
NULL
,
mBlockIdx
)
+
sizeof
(
TSCKSUM
);
// alloc
if
(
!
ppBuf
)
ppBuf
=
&
pBuf
;
...
...
@@ -647,7 +739,7 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SMapData *pBlockIdxMap, uint8_t
// build
n
+=
tPutU32
(
*
ppBuf
+
n
,
TSDB_FILE_DLMT
);
n
+=
tPutMapData
(
*
ppBuf
,
pBlockIdxMap
);
n
+=
tPutMapData
(
*
ppBuf
,
mBlockIdx
);
taosCalcChecksumAppend
(
0
,
*
ppBuf
,
size
);
ASSERT
(
n
+
sizeof
(
TSCKSUM
)
==
size
);
...
...
@@ -659,7 +751,9 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SMapData *pBlockIdxMap, uint8_t
goto
_err
;
}
// update (todo)
// update
pHeadFile
->
offset
=
pHeadFile
->
size
;
pHeadFile
->
size
+=
size
;
tsdbFree
(
pBuf
);
return
code
;
...
...
@@ -670,20 +764,21 @@ _err:
return
code
;
}
int32_t
tsdbWriteBlock
(
SDataFWriter
*
pWriter
,
SMapData
*
pBlockMap
,
uint8_t
**
ppBuf
,
SBlockIdx
*
pBlockIdx
)
{
int32_t
code
=
0
;
uint8_t
*
pBuf
=
NULL
;
int64_t
size
;
int64_t
n
;
int32_t
tsdbWriteBlock
(
SDataFWriter
*
pWriter
,
SMapData
*
mBlock
,
uint8_t
**
ppBuf
,
SBlockIdx
*
pBlockIdx
)
{
int32_t
code
=
0
;
SHeadFile
*
pHeadFile
=
pWriter
->
pSet
->
pHeadFile
;
uint8_t
*
pBuf
=
NULL
;
int64_t
size
;
int64_t
n
;
ASSERT
(
pBlockMap
->
nItem
>
0
);
ASSERT
(
mBlock
->
nItem
>
0
);
// prepare
size
=
0
;
size
+=
tPutU32
(
NULL
,
TSDB_FILE_DLMT
);
size
+=
tPutI64
(
NULL
,
pBlockIdx
->
suid
);
size
+=
tPutI64
(
NULL
,
pBlockIdx
->
uid
);
size
=
size
+
tPutMapData
(
NULL
,
pBlockMap
)
+
sizeof
(
TSCKSUM
);
size
=
size
+
tPutMapData
(
NULL
,
mBlock
)
+
sizeof
(
TSCKSUM
);
// alloc
if
(
!
ppBuf
)
ppBuf
=
&
pBuf
;
...
...
@@ -695,7 +790,7 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pBlockMap, uint8_t **ppB
n
+=
tPutU32
(
*
ppBuf
+
n
,
TSDB_FILE_DLMT
);
n
+=
tPutI64
(
*
ppBuf
+
n
,
pBlockIdx
->
suid
);
n
+=
tPutI64
(
*
ppBuf
+
n
,
pBlockIdx
->
uid
);
n
+=
tPutMapData
(
*
ppBuf
+
n
,
pBlockMap
);
n
+=
tPutMapData
(
*
ppBuf
+
n
,
mBlock
);
taosCalcChecksumAppend
(
0
,
*
ppBuf
,
size
);
ASSERT
(
n
+
sizeof
(
TSCKSUM
)
==
size
);
...
...
@@ -707,16 +802,18 @@ int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pBlockMap, uint8_t **ppB
goto
_err
;
}
// update
(todo)
// pBlockIdx->offset = -1
;
// update
pBlockIdx
->
offset
=
pHeadFile
->
size
;
pBlockIdx
->
size
=
size
;
// pWriter->pSet->pHeadF.offset
pHeadFile
->
size
+=
size
;
tsdbFree
(
pBuf
);
tsdbTrace
(
"vgId:%d write block, offset:%"
PRId64
" size:%"
PRId64
,
TD_VID
(
pWriter
->
pTsdb
->
pVnode
),
pBlockIdx
->
offset
,
pBlockIdx
->
size
);
return
code
;
_err:
tsdbFree
(
pBuf
);
tsdbError
(
"vgId:%d write block failed since %s"
,
TD_VID
(
pWriter
->
pTsdb
->
pVnode
),
tstrerror
(
code
));
return
code
;
}
...
...
source/dnode/vnode/src/tsdb/tsdbUtil.c
浏览文件 @
0359254b
...
...
@@ -298,14 +298,40 @@ void tBlockClear(SBlock *pBlock) {
int32_t
tPutBlock
(
uint8_t
*
p
,
void
*
ph
)
{
int32_t
n
=
0
;
SBlock
*
pBlock
=
(
SBlock
*
)
ph
;
// TODO
n
+=
tPutKEYINFO
(
p
?
p
+
n
:
p
,
&
pBlock
->
info
);
n
+=
tPutI32v
(
p
?
p
+
n
:
p
,
pBlock
->
nRow
);
n
+=
tPutI8
(
p
?
p
+
n
:
p
,
pBlock
->
last
);
n
+=
tPutI8
(
p
?
p
+
n
:
p
,
pBlock
->
hasDup
);
n
+=
tPutI8
(
p
?
p
+
n
:
p
,
pBlock
->
cmprAlg
);
n
+=
tPutI8
(
p
?
p
+
n
:
p
,
pBlock
->
nSubBlock
);
for
(
int8_t
iSubBlock
=
0
;
iSubBlock
<
pBlock
->
nSubBlock
;
iSubBlock
++
)
{
n
+=
tPutI64v
(
p
?
p
+
n
:
p
,
pBlock
->
aSubBlock
[
iSubBlock
].
offset
);
n
+=
tPutI64v
(
p
?
p
+
n
:
p
,
pBlock
->
aSubBlock
[
iSubBlock
].
ksize
);
n
+=
tPutI64v
(
p
?
p
+
n
:
p
,
pBlock
->
aSubBlock
[
iSubBlock
].
bsize
);
n
+=
tPutMapData
(
p
?
p
+
n
:
p
,
&
pBlock
->
aSubBlock
[
iSubBlock
].
mBlockCol
);
}
return
n
;
}
int32_t
tGetBlock
(
uint8_t
*
p
,
void
*
ph
)
{
int32_t
n
=
0
;
SBlock
*
pBlock
=
(
SBlock
*
)
ph
;
// TODO
n
+=
tGetKEYINFO
(
p
+
n
,
&
pBlock
->
info
);
n
+=
tGetI32v
(
p
+
n
,
&
pBlock
->
nRow
);
n
+=
tGetI8
(
p
+
n
,
&
pBlock
->
last
);
n
+=
tGetI8
(
p
+
n
,
&
pBlock
->
hasDup
);
n
+=
tGetI8
(
p
+
n
,
&
pBlock
->
cmprAlg
);
n
+=
tGetI8
(
p
+
n
,
&
pBlock
->
nSubBlock
);
for
(
int8_t
iSubBlock
=
0
;
iSubBlock
<
pBlock
->
nSubBlock
;
iSubBlock
++
)
{
n
+=
tGetI64v
(
p
+
n
,
&
pBlock
->
aSubBlock
[
iSubBlock
].
offset
);
n
+=
tGetI64v
(
p
+
n
,
&
pBlock
->
aSubBlock
[
iSubBlock
].
ksize
);
n
+=
tGetI64v
(
p
+
n
,
&
pBlock
->
aSubBlock
[
iSubBlock
].
bsize
);
n
+=
tGetMapData
(
p
+
n
,
&
pBlock
->
aSubBlock
[
iSubBlock
].
mBlockCol
);
}
return
n
;
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录