Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
16bf52bf
T
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1180
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
未验证
提交
16bf52bf
编写于
8月 01, 2022
作者:
L
Liu Jicong
提交者:
GitHub
8月 01, 2022
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #15611 from taosdata/feature/stream
refactor(stream): use vnode meta
上级
a73ea6f9
bb6fabcc
变更
9
隐藏空白更改
内联
并排
Showing
9 changed file
with
50 addition
and
24 deletion
+50
-24
include/common/tcommon.h
include/common/tcommon.h
+1
-2
include/libs/wal/wal.h
include/libs/wal/wal.h
+1
-1
source/dnode/vnode/src/inc/meta.h
source/dnode/vnode/src/inc/meta.h
+4
-1
source/dnode/vnode/src/inc/vnodeInt.h
source/dnode/vnode/src/inc/vnodeInt.h
+1
-1
source/dnode/vnode/src/meta/metaOpen.c
source/dnode/vnode/src/meta/metaOpen.c
+22
-0
source/dnode/vnode/src/tq/tq.c
source/dnode/vnode/src/tq/tq.c
+2
-2
source/dnode/vnode/src/tq/tqMeta.c
source/dnode/vnode/src/tq/tqMeta.c
+1
-1
source/dnode/vnode/src/tq/tqPush.c
source/dnode/vnode/src/tq/tqPush.c
+1
-1
source/libs/wal/src/walWrite.c
source/libs/wal/src/walWrite.c
+17
-15
未找到文件。
include/common/tcommon.h
浏览文件 @
16bf52bf
...
...
@@ -56,7 +56,6 @@ enum {
STREAM_INPUT__DATA_SUBMIT
=
1
,
STREAM_INPUT__DATA_BLOCK
,
STREAM_INPUT__MERGED_SUBMIT
,
// STREAM_INPUT__TABLE_SCAN,
STREAM_INPUT__TQ_SCAN
,
STREAM_INPUT__DATA_RETRIEVE
,
STREAM_INPUT__GET_RES
,
...
...
@@ -154,7 +153,7 @@ typedef struct SQueryTableDataCond {
int32_t
order
;
// desc|asc order to iterate the data block
int32_t
numOfCols
;
SColumnInfo
*
colList
;
int32_t
type
;
// data block load type:
int32_t
type
;
// data block load type:
STimeWindow
twindows
;
int64_t
startVersion
;
int64_t
endVersion
;
...
...
include/libs/wal/wal.h
浏览文件 @
16bf52bf
...
...
@@ -41,7 +41,7 @@ extern "C" {
#define WAL_REFRESH_MS 1000
#define WAL_PATH_LEN (TSDB_FILENAME_LEN + 12)
#define WAL_FILE_LEN (WAL_PATH_LEN + 32)
#define WAL_MAGIC 0xFAFBFCFDULL
#define WAL_MAGIC 0xFAFBFCFD
F4F3F2F1
ULL
#define WAL_SCAN_BUF_SIZE (1024 * 1024 * 3)
typedef
enum
{
...
...
source/dnode/vnode/src/inc/meta.h
浏览文件 @
16bf52bf
...
...
@@ -78,7 +78,10 @@ struct SMeta {
TTB
*
pTagIdx
;
TTB
*
pTtlIdx
;
TTB
*
pSmaIdx
;
TTB
*
pSmaIdx
;
TTB
*
pTaskIdx
;
SMetaIdx
*
pIdx
;
};
...
...
source/dnode/vnode/src/inc/vnodeInt.h
浏览文件 @
16bf52bf
...
...
@@ -153,7 +153,7 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, char* msg, int32_t msgLen);
int32_t
tqProcessPollReq
(
STQ
*
pTq
,
SRpcMsg
*
pMsg
);
int32_t
tqProcessTaskDeployReq
(
STQ
*
pTq
,
char
*
msg
,
int32_t
msgLen
);
int32_t
tqProcessTaskDropReq
(
STQ
*
pTq
,
char
*
msg
,
int32_t
msgLen
);
int32_t
tqProcessStreamTrigger
(
STQ
*
pTq
,
SSubmitReq
*
data
);
int32_t
tqProcessStreamTrigger
(
STQ
*
pTq
,
SSubmitReq
*
data
,
int64_t
ver
);
int32_t
tqProcessTaskRunReq
(
STQ
*
pTq
,
SRpcMsg
*
pMsg
);
int32_t
tqProcessTaskDispatchReq
(
STQ
*
pTq
,
SRpcMsg
*
pMsg
);
int32_t
tqProcessTaskRecoverReq
(
STQ
*
pTq
,
SRpcMsg
*
pMsg
);
...
...
source/dnode/vnode/src/meta/metaOpen.c
浏览文件 @
16bf52bf
...
...
@@ -22,6 +22,7 @@ static int tagIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
static
int
ttlIdxKeyCmpr
(
const
void
*
pKey1
,
int
kLen1
,
const
void
*
pKey2
,
int
kLen2
);
static
int
uidIdxKeyCmpr
(
const
void
*
pKey1
,
int
kLen1
,
const
void
*
pKey2
,
int
kLen2
);
static
int
smaIdxKeyCmpr
(
const
void
*
pKey1
,
int
kLen1
,
const
void
*
pKey2
,
int
kLen2
);
static
int
taskIdxKeyCmpr
(
const
void
*
pKey1
,
int
kLen1
,
const
void
*
pKey2
,
int
kLen2
);
static
int32_t
metaInitLock
(
SMeta
*
pMeta
)
{
return
taosThreadRwlockInit
(
&
pMeta
->
lock
,
NULL
);
}
static
int32_t
metaDestroyLock
(
SMeta
*
pMeta
)
{
return
taosThreadRwlockDestroy
(
&
pMeta
->
lock
);
}
...
...
@@ -130,6 +131,12 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
goto
_err
;
}
ret
=
tdbTbOpen
(
"stream.task.db"
,
sizeof
(
int64_t
),
-
1
,
taskIdxKeyCmpr
,
pMeta
->
pEnv
,
&
pMeta
->
pTaskIdx
);
if
(
ret
<
0
)
{
metaError
(
"vgId: %d, failed to open meta stream task index since %s"
,
TD_VID
(
pVnode
),
tstrerror
(
terrno
));
goto
_err
;
}
// open index
if
(
metaOpenIdx
(
pMeta
)
<
0
)
{
metaError
(
"vgId:%d, failed to open meta index since %s"
,
TD_VID
(
pVnode
),
tstrerror
(
terrno
));
...
...
@@ -143,6 +150,7 @@ int metaOpen(SVnode *pVnode, SMeta **ppMeta) {
_err:
if
(
pMeta
->
pIdx
)
metaCloseIdx
(
pMeta
);
if
(
pMeta
->
pTaskIdx
)
tdbTbClose
(
pMeta
->
pTaskIdx
);
if
(
pMeta
->
pSmaIdx
)
tdbTbClose
(
pMeta
->
pSmaIdx
);
if
(
pMeta
->
pTtlIdx
)
tdbTbClose
(
pMeta
->
pTtlIdx
);
if
(
pMeta
->
pTagIvtIdx
)
indexClose
(
pMeta
->
pTagIvtIdx
);
...
...
@@ -162,6 +170,7 @@ _err:
int
metaClose
(
SMeta
*
pMeta
)
{
if
(
pMeta
)
{
if
(
pMeta
->
pIdx
)
metaCloseIdx
(
pMeta
);
if
(
pMeta
->
pTaskIdx
)
tdbTbClose
(
pMeta
->
pTaskIdx
);
if
(
pMeta
->
pSmaIdx
)
tdbTbClose
(
pMeta
->
pSmaIdx
);
if
(
pMeta
->
pTtlIdx
)
tdbTbClose
(
pMeta
->
pTtlIdx
);
if
(
pMeta
->
pTagIvtIdx
)
indexClose
(
pMeta
->
pTagIvtIdx
);
...
...
@@ -378,3 +387,16 @@ static int smaIdxKeyCmpr(const void *pKey1, int kLen1, const void *pKey2, int kL
return
0
;
}
static
int
taskIdxKeyCmpr
(
const
void
*
pKey1
,
int
kLen1
,
const
void
*
pKey2
,
int
kLen2
)
{
int32_t
uid1
=
*
(
int32_t
*
)
pKey1
;
int32_t
uid2
=
*
(
int32_t
*
)
pKey2
;
if
(
uid1
>
uid2
)
{
return
1
;
}
else
if
(
uid1
<
uid2
)
{
return
-
1
;
}
return
0
;
}
source/dnode/vnode/src/tq/tq.c
浏览文件 @
16bf52bf
...
...
@@ -695,7 +695,7 @@ FAIL:
return
-
1
;
}
int32_t
tqProcessStreamTrigger
(
STQ
*
pTq
,
SSubmitReq
*
pReq
)
{
int32_t
tqProcessStreamTrigger
(
STQ
*
pTq
,
SSubmitReq
*
pReq
,
int64_t
ver
)
{
void
*
pIter
=
NULL
;
bool
failed
=
false
;
SStreamDataSubmit
*
pSubmit
=
NULL
;
...
...
@@ -713,7 +713,7 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) {
SStreamTask
*
pTask
=
*
(
SStreamTask
**
)
pIter
;
if
(
!
pTask
->
isDataScan
)
continue
;
qDebug
(
"data submit enqueue stream task: %d
"
,
pTask
->
taskId
);
qDebug
(
"data submit enqueue stream task: %d
, ver: %ld"
,
pTask
->
taskId
,
ver
);
if
(
!
failed
)
{
if
(
streamTaskInput
(
pTask
,
(
SStreamQueueItem
*
)
pSubmit
)
<
0
)
{
...
...
source/dnode/vnode/src/tq/tqMeta.c
浏览文件 @
16bf52bf
...
...
@@ -48,7 +48,7 @@ int32_t tqMetaOpen(STQ* pTq) {
ASSERT
(
0
);
}
if
(
tdbTbOpen
(
"handles"
,
-
1
,
-
1
,
0
,
pTq
->
pMetaStore
,
&
pTq
->
pExecStore
)
<
0
)
{
if
(
tdbTbOpen
(
"handles"
,
-
1
,
-
1
,
NULL
,
pTq
->
pMetaStore
,
&
pTq
->
pExecStore
)
<
0
)
{
ASSERT
(
0
);
}
...
...
source/dnode/vnode/src/tq/tqPush.c
浏览文件 @
16bf52bf
...
...
@@ -252,7 +252,7 @@ int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver)
SSubmitReq
*
pReq
=
(
SSubmitReq
*
)
data
;
pReq
->
version
=
ver
;
tqProcessStreamTrigger
(
pTq
,
data
);
tqProcessStreamTrigger
(
pTq
,
data
,
ver
);
}
return
0
;
...
...
source/libs/wal/src/walWrite.c
浏览文件 @
16bf52bf
...
...
@@ -116,15 +116,15 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
walBuildIdxName
(
pWal
,
walGetCurFileFirstVer
(
pWal
),
fnameStr
);
TdFilePtr
pIdx
T
File
=
taosOpenFile
(
fnameStr
,
TD_FILE_WRITE
|
TD_FILE_READ
|
TD_FILE_APPEND
);
TdFilePtr
pIdxFile
=
taosOpenFile
(
fnameStr
,
TD_FILE_WRITE
|
TD_FILE_READ
|
TD_FILE_APPEND
);
if
(
pIdx
T
File
==
NULL
)
{
if
(
pIdxFile
==
NULL
)
{
ASSERT
(
0
);
taosThreadMutexUnlock
(
&
pWal
->
mutex
);
return
-
1
;
}
int64_t
idxOff
=
walGetVerIdxOffset
(
pWal
,
ver
);
code
=
taosLSeekFile
(
pIdx
T
File
,
idxOff
,
SEEK_SET
);
code
=
taosLSeekFile
(
pIdxFile
,
idxOff
,
SEEK_SET
);
if
(
code
<
0
)
{
ASSERT
(
0
);
taosThreadMutexUnlock
(
&
pWal
->
mutex
);
...
...
@@ -132,7 +132,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
// read idx file and get log file pos
SWalIdxEntry
entry
;
if
(
taosReadFile
(
pIdx
T
File
,
&
entry
,
sizeof
(
SWalIdxEntry
))
!=
sizeof
(
SWalIdxEntry
))
{
if
(
taosReadFile
(
pIdxFile
,
&
entry
,
sizeof
(
SWalIdxEntry
))
!=
sizeof
(
SWalIdxEntry
))
{
ASSERT
(
0
);
taosThreadMutexUnlock
(
&
pWal
->
mutex
);
return
-
1
;
...
...
@@ -140,24 +140,24 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
ASSERT
(
entry
.
ver
==
ver
);
walBuildLogName
(
pWal
,
walGetCurFileFirstVer
(
pWal
),
fnameStr
);
TdFilePtr
pLogTFile
=
taosOpenFile
(
fnameStr
,
TD_FILE_WRITE
|
TD_FILE_READ
|
TD_FILE_APPEND
);
if
(
pLogTFile
==
NULL
)
{
ASSERT
(
0
);
TdFilePtr
pLogFile
=
taosOpenFile
(
fnameStr
,
TD_FILE_WRITE
|
TD_FILE_READ
|
TD_FILE_APPEND
);
if
(
pLogFile
==
NULL
)
{
// TODO
terrno
=
TAOS_SYSTEM_ERROR
(
errno
);
taosThreadMutexUnlock
(
&
pWal
->
mutex
);
return
-
1
;
}
code
=
taosLSeekFile
(
pLog
T
File
,
entry
.
offset
,
SEEK_SET
);
code
=
taosLSeekFile
(
pLogFile
,
entry
.
offset
,
SEEK_SET
);
if
(
code
<
0
)
{
ASSERT
(
0
);
// TODO
terrno
=
TAOS_SYSTEM_ERROR
(
errno
);
taosThreadMutexUnlock
(
&
pWal
->
mutex
);
return
-
1
;
}
// validate offset
SWalCkHead
head
;
ASSERT
(
taosValidFile
(
pLog
T
File
));
int64_t
size
=
taosReadFile
(
pLog
T
File
,
&
head
,
sizeof
(
SWalCkHead
));
ASSERT
(
taosValidFile
(
pLogFile
));
int64_t
size
=
taosReadFile
(
pLogFile
,
&
head
,
sizeof
(
SWalCkHead
));
if
(
size
!=
sizeof
(
SWalCkHead
))
{
ASSERT
(
0
);
taosThreadMutexUnlock
(
&
pWal
->
mutex
);
...
...
@@ -180,14 +180,14 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
}
// truncate old files
code
=
taosFtruncateFile
(
pLog
T
File
,
entry
.
offset
);
code
=
taosFtruncateFile
(
pLogFile
,
entry
.
offset
);
if
(
code
<
0
)
{
ASSERT
(
0
);
terrno
=
TAOS_SYSTEM_ERROR
(
errno
);
taosThreadMutexUnlock
(
&
pWal
->
mutex
);
return
-
1
;
}
code
=
taosFtruncateFile
(
pIdx
T
File
,
idxOff
);
code
=
taosFtruncateFile
(
pIdxFile
,
idxOff
);
if
(
code
<
0
)
{
ASSERT
(
0
);
terrno
=
TAOS_SYSTEM_ERROR
(
errno
);
...
...
@@ -205,8 +205,10 @@ int32_t walRollback(SWal *pWal, int64_t ver) {
ASSERT
(((
SWalFileInfo
*
)
taosArrayGetLast
(
pWal
->
fileInfoSet
))
->
fileSize
==
0
);
((
SWalFileInfo
*
)
taosArrayGetLast
(
pWal
->
fileInfoSet
))
->
firstVer
=
-
1
;
}
taosCloseFile
(
&
pIdxTFile
);
taosCloseFile
(
&
pLogTFile
);
taosCloseFile
(
&
pIdxFile
);
taosCloseFile
(
&
pLogFile
);
walSaveMeta
(
pWal
);
// unlock
taosThreadMutexUnlock
(
&
pWal
->
mutex
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录