Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
d9e24566
T
TDengine
项目概览
taosdata
/
TDengine
大约 2 年 前同步成功
通知
1193
Star
22018
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
d9e24566
编写于
6月 20, 2023
作者:
Y
yihaoDeng
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
add checkpoint
上级
f1fcc1d2
变更
2
隐藏空白更改
内联
并排
Showing
2 changed file
with
144 addition
and
30 deletion
+144
-30
include/libs/stream/streamSnapshot.h
include/libs/stream/streamSnapshot.h
+8
-7
source/libs/stream/src/streamSnapshot.c
source/libs/stream/src/streamSnapshot.c
+136
-23
未找到文件。
include/libs/stream/streamSnapshot.h
浏览文件 @
d9e24566
...
...
@@ -19,15 +19,16 @@
typedef
struct
SStreamSnapReader
SStreamSnapReader
;
typedef
struct
StreamSnapWriter
StreamSnapWriter
;
typedef
struct
SStreamSnapHandle
SStreamSnapHandle
;
typedef
struct
SStreamSnapHandle
SStreamSnapHandle
;
typedef
struct
SStreamSnapBlockHdr
SStreamSnapBlockHdr
;
int32_t
streamSnapReaderOpen
(
void
*
pMeta
,
int64_t
sver
,
int64_t
ever
,
void
**
ppReader
);
int32_t
streamSnapReaderClose
(
void
**
p
pReader
);
int32_t
streamSnapRead
(
void
*
pReader
,
uint8_t
**
ppData
);
int32_t
streamSnapReaderOpen
(
void
*
pMeta
,
int64_t
sver
,
int64_t
ever
,
SStreamSnapReader
**
ppReader
);
int32_t
streamSnapReaderClose
(
SStreamSnapReader
*
pReader
);
int32_t
streamSnapRead
(
SStreamSnapReader
*
pReader
,
uint8_t
**
ppData
,
int64_t
*
size
);
// SMetaSnapWriter ========================================
int32_t
streamSnapWriterOpen
(
void
*
pMeta
,
int64_t
sver
,
int64_t
ever
,
void
**
ppWriter
);
int32_t
streamSnapWrite
(
void
*
pWriter
,
uint8_t
*
pData
,
uint32_t
nData
);
int32_t
streamSnapWriterClose
(
void
**
ppWriter
,
int8_t
rollback
);
int32_t
streamSnapWriterOpen
(
void
*
pMeta
,
int64_t
sver
,
int64_t
ever
,
StreamSnapWriter
**
ppWriter
);
int32_t
streamSnapWrite
(
StreamSnapWriter
*
pWriter
,
uint8_t
*
pData
,
uint32_t
nData
);
int32_t
streamSnapWriterClose
(
StreamSnapWriter
**
ppWriter
,
int8_t
rollback
);
#endif
\ No newline at end of file
source/libs/stream/src/streamSnapshot.c
浏览文件 @
d9e24566
...
...
@@ -17,42 +17,69 @@
#include "rocksdb/c.h"
#include "tcommon.h"
enum
SBackendFileType
{
ROCKSDB_OPTIONS_TYPE
=
1
,
ROCKSDB_MAINFEST_TYPE
=
2
,
ROCKSDB_SST_TYPE
=
3
,
ROCKSDB_CURRENT_TYPE
=
4
,
ROCKSDB_CHECKPOINT_META_TYPE
=
5
,
};
typedef
struct
SBackendFileItem
{
char
*
name
;
int8_t
type
;
}
SBackendFileItem
;
typedef
struct
SBackendFile
{
SArray
*
pSst
;
char
*
pCurrent
;
char
*
pMainfest
;
char
*
pOptions
;
SArray
*
pSst
;
char
*
pCheckpointMeta
;
char
*
path
;
}
SBanckendFile
;
struct
SStreamSnapHandle
{
void
*
handle
;
SArray
*
fileList
;
SBanckendFile
*
pBackendFile
;
int64_t
checkpointId
;
int64_t
seraial
;
int64_t
offset
;
TdFilePtr
fd
;
int8_t
filetype
;
SArray
*
pFileList
;
int32_t
currFileIdx
;
};
struct
SStreamSnapBlockHdr
{
int8_t
type
;
int8_t
flag
;
int64_t
index
;
char
name
[
128
];
int64_t
size
;
uint8_t
data
[];
};
struct
SStreamSnapReader
{
void
*
pMeta
;
int64_t
sver
;
int64_t
ever
;
void
*
pMeta
;
int64_t
sver
;
int64_t
ever
;
SStreamSnapHandle
handle
;
};
// SMetaSnapWriter ========================================
struct
StreamSnapWriter
{
void
*
pMeta
;
int64_t
sver
;
int64_t
ever
;
};
const
char
*
ROCKSDB_OPTIONS
=
"OPTIONS"
;
const
char
*
ROCKSDB_MAINFEST
=
"MANIFEST"
;
const
char
*
ROCKSDB_SST
=
"sst"
;
const
char
*
ROCKSDB_CURRENT
=
"CURRENT"
;
const
char
*
ROCKSDB_CHECKPOINT_META
=
"CHECKPOINT"
;
static
int64_t
kBlockSize
=
64
*
1024
;
const
char
*
ROCKSDB_OPTIONS
=
"OPTIONS"
;
const
char
*
ROCKSDB_MAINFEST
=
"MANIFEST"
;
const
char
*
ROCKSDB_SST
=
"sst"
;
const
char
*
ROCKSDB_CURRENT
=
"CURRENT"
;
const
char
*
ROCKSDB_CHECKPOINT_META
=
"CHECKPOINT"
;
int32_t
streamSnapHandleInit
(
SStreamSnapHandle
*
handle
,
char
*
path
);
void
streamSnapHandleDestroy
(
SStreamSnapHandle
*
handle
);
int32_t
streamSnapHandleInit
(
SStreamSnapHandle
*
handle
,
char
*
path
)
{
// impl later
int32_t
code
=
0
;
handle
->
fileList
=
taosArrayInit
(
32
,
sizeof
(
void
*
));
TdDirPtr
pDir
=
taosOpenDir
(
path
);
if
(
NULL
==
pDir
)
{
...
...
@@ -60,6 +87,9 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path) {
}
SBanckendFile
*
pFile
=
taosMemoryCalloc
(
1
,
sizeof
(
SBanckendFile
));
handle
->
checkpointId
=
0
;
handle
->
seraial
=
0
;
pFile
->
path
=
taosStrdup
(
path
);
pFile
->
pSst
=
taosArrayInit
(
16
,
sizeof
(
void
*
));
TdDirEntryPtr
pDirEntry
;
...
...
@@ -90,8 +120,47 @@ int32_t streamSnapHandleInit(SStreamSnapHandle* handle, char* path) {
}
taosCloseDir
(
&
pDir
);
SArray
*
list
=
taosArrayInit
(
64
,
sizeof
(
SBackendFileItem
));
SBackendFileItem
item
;
item
.
name
=
pFile
->
pCurrent
;
item
.
type
=
ROCKSDB_CURRENT_TYPE
;
taosArrayPush
(
list
,
&
item
);
item
.
name
=
pFile
->
pMainfest
;
item
.
type
=
ROCKSDB_MAINFEST_TYPE
;
taosArrayPush
(
list
,
&
item
);
item
.
name
=
pFile
->
pOptions
;
item
.
type
=
ROCKSDB_OPTIONS_TYPE
;
taosArrayPush
(
list
,
&
item
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
pFile
->
pSst
);
i
++
)
{
char
*
sst
=
taosArrayGetP
(
pFile
->
pSst
,
i
);
item
.
name
=
sst
;
item
.
type
=
ROCKSDB_SST_TYPE
;
taosArrayPush
(
list
,
&
item
);
}
item
.
name
=
pFile
->
pCheckpointMeta
;
item
.
type
=
ROCKSDB_CHECKPOINT_META_TYPE
;
taosArrayPush
(
list
,
&
item
);
handle
->
pBackendFile
=
pFile
;
handle
->
currFileIdx
=
0
;
handle
->
pFileList
=
list
;
handle
->
fd
=
taosOpenFile
(
taosArrayGetP
(
handle
->
pFileList
,
handle
->
currFileIdx
),
TD_FILE_READ
);
if
(
handle
->
fd
==
NULL
)
{
goto
_err
;
}
handle
->
seraial
=
0
;
handle
->
offset
=
0
;
return
0
;
_err:
streamSnapHandleDestroy
(
handle
);
code
=
-
1
;
return
code
;
}
...
...
@@ -101,37 +170,81 @@ void streamSnapHandleDestroy(SStreamSnapHandle* handle) {
taosMemoryFree
(
pFile
->
pCurrent
);
taosMemoryFree
(
pFile
->
pMainfest
);
taosMemoryFree
(
pFile
->
pOptions
);
for
(
int
i
=
0
;
i
<
taosArrayGetSize
(
pFile
->
pSst
);
i
++
)
{
taosMemoryFree
(
pFile
->
path
);
for
(
int
i
=
0
;
pFile
->
pSst
!=
NULL
&&
i
<
taosArrayGetSize
(
pFile
->
pSst
);
i
++
)
{
char
*
sst
=
taosArrayGetP
(
pFile
->
pSst
,
i
);
taosMemoryFree
(
sst
);
}
taosArrayDestroy
(
pFile
->
pSst
);
taosArrayDestroy
(
handle
->
pFileList
);
taosMemoryFree
(
pFile
);
return
;
}
int32_t
streamSnapReaderOpen
(
void
*
pMeta
,
int64_t
sver
,
int64_t
ever
,
void
**
ppReader
)
{
int32_t
streamSnapReaderOpen
(
void
*
pMeta
,
int64_t
sver
,
int64_t
ever
,
SStreamSnapReader
**
ppReader
)
{
// impl later
rocksdb_t
*
db
=
NULL
;
SStreamSnapReader
*
pReader
=
taosMemoryCalloc
(
1
,
sizeof
(
SStreamSnapReader
));
if
(
pReader
==
NULL
)
{
return
TSDB_CODE_OUT_OF_MEMORY
;
}
const
char
*
path
=
NULL
;
if
(
streamSnapHandleInit
(
&
pReader
->
handle
,
(
char
*
)
path
)
<
0
)
{
return
-
1
;
}
*
ppReader
=
pReader
;
return
0
;
}
int32_t
streamSnapReaderClose
(
void
**
ppReader
)
{
// impl later
int32_t
streamSnapReaderClose
(
SStreamSnapReader
*
pReader
)
{
if
(
pReader
==
NULL
)
return
0
;
streamSnapHandleDestroy
(
&
pReader
->
handle
);
taosMemoryFree
(
pReader
);
return
0
;
}
int32_t
streamSnapRead
(
void
*
pReader
,
uint8_t
**
ppData
)
{
int32_t
streamSnapRead
(
SStreamSnapReader
*
pReader
,
uint8_t
**
ppData
,
int64_t
*
size
)
{
// impl later
SStreamSnapHandle
*
pHandle
=
&
pReader
->
handle
;
SBackendFileItem
*
item
=
taosArrayGet
(
pHandle
->
pFileList
,
pHandle
->
currFileIdx
);
uint8_t
*
buf
=
taosMemoryCalloc
(
1
,
sizeof
(
SStreamSnapBlockHdr
)
+
kBlockSize
);
int64_t
nread
=
taosReadFile
(
pHandle
->
fd
,
buf
,
kBlockSize
);
if
(
nread
==
-
1
)
{
// handle later
return
-
1
;
}
else
{
taosCloseFile
(
&
pHandle
->
fd
);
pHandle
->
currFileIdx
+=
1
;
if
(
pHandle
->
currFileIdx
>=
taosArrayGetSize
(
pHandle
->
pFileList
))
{
// finish
return
0
;
}
item
=
taosArrayGet
(
pHandle
->
pFileList
,
pHandle
->
currFileIdx
);
pHandle
->
fd
=
taosOpenFile
(
item
->
name
,
TD_FILE_READ
);
// handle err later
nread
=
taosReadFile
(
pHandle
->
fd
,
buf
,
kBlockSize
);
}
SStreamSnapBlockHdr
*
pHdr
=
(
SStreamSnapBlockHdr
*
)
buf
;
pHdr
->
size
=
nread
;
pHdr
->
type
=
item
->
type
;
memcpy
(
pHdr
->
name
,
item
->
name
,
strlen
(
item
->
name
));
pHandle
->
seraial
+=
nread
;
*
ppData
=
buf
;
*
size
=
sizeof
(
SStreamSnapBlockHdr
)
+
nread
;
return
0
;
}
// SMetaSnapWriter ========================================
int32_t
streamSnapWriterOpen
(
void
*
pMeta
,
int64_t
sver
,
int64_t
ever
,
void
**
ppWriter
)
{
int32_t
streamSnapWriterOpen
(
void
*
pMeta
,
int64_t
sver
,
int64_t
ever
,
StreamSnapWriter
**
ppWriter
)
{
// impl later
return
0
;
}
int32_t
streamSnapWrite
(
void
*
pWriter
,
uint8_t
*
pData
,
uint32_t
nData
)
{
int32_t
streamSnapWrite
(
StreamSnapWriter
*
pWriter
,
uint8_t
*
pData
,
uint32_t
nData
)
{
// impl later
return
0
;
}
int32_t
streamSnapWriterClose
(
void
**
ppWriter
,
int8_t
rollback
)
{
return
0
;
}
int32_t
streamSnapWriterClose
(
StreamSnapWriter
**
ppWriter
,
int8_t
rollback
)
{
return
0
;
}
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录