Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
18479d81
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1184
Star
22015
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
18479d81
编写于
4月 11, 2023
作者:
H
Haojun Liao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor: do some internall refactor.
上级
15cceb5a
变更
10
显示空白变更内容
内联
并排
Showing
10 changed file
with
40 addition
and
35 deletion
+40
-35
include/libs/stream/tstream.h
include/libs/stream/tstream.h
+4
-4
source/dnode/mnode/impl/src/mndDef.c
source/dnode/mnode/impl/src/mndDef.c
+2
-2
source/dnode/mnode/impl/src/mndScheduler.c
source/dnode/mnode/impl/src/mndScheduler.c
+5
-5
source/dnode/mnode/impl/src/mndStream.c
source/dnode/mnode/impl/src/mndStream.c
+4
-4
source/dnode/snode/src/snode.c
source/dnode/snode/src/snode.c
+1
-1
source/dnode/vnode/src/tq/tq.c
source/dnode/vnode/src/tq/tq.c
+5
-5
source/dnode/vnode/src/tq/tqRestore.c
source/dnode/vnode/src/tq/tqRestore.c
+3
-2
source/libs/stream/src/streamExec.c
source/libs/stream/src/streamExec.c
+9
-5
source/libs/stream/src/streamMeta.c
source/libs/stream/src/streamMeta.c
+4
-4
source/libs/stream/src/streamTask.c
source/libs/stream/src/streamTask.c
+3
-3
未找到文件。
include/libs/stream/tstream.h
浏览文件 @
18479d81
...
...
@@ -50,7 +50,7 @@ enum {
TASK_STATUS__RECOVER_PREPARE
,
TASK_STATUS__RECOVER1
,
TASK_STATUS__RECOVER2
,
TASK_STATUS_RESTORE
,
// only available for source task to replay WAL from the checkpoint
TASK_STATUS_
_
RESTORE
,
// only available for source task to replay WAL from the checkpoint
};
enum
{
...
...
@@ -353,9 +353,9 @@ struct SStreamTask {
int32_t
tEncodeStreamEpInfo
(
SEncoder
*
pEncoder
,
const
SStreamChildEpInfo
*
pInfo
);
int32_t
tDecodeStreamEpInfo
(
SDecoder
*
pDecoder
,
SStreamChildEpInfo
*
pInfo
);
SStreamTask
*
tNewS
S
treamTask
(
int64_t
streamId
);
int32_t
tEncodeS
S
treamTask
(
SEncoder
*
pEncoder
,
const
SStreamTask
*
pTask
);
int32_t
tDecodeS
S
treamTask
(
SDecoder
*
pDecoder
,
SStreamTask
*
pTask
);
SStreamTask
*
tNewStreamTask
(
int64_t
streamId
);
int32_t
tEncodeStreamTask
(
SEncoder
*
pEncoder
,
const
SStreamTask
*
pTask
);
int32_t
tDecodeStreamTask
(
SDecoder
*
pDecoder
,
SStreamTask
*
pTask
);
void
tFreeStreamTask
(
SStreamTask
*
pTask
);
int32_t
tAppendDataForStream
(
SStreamTask
*
pTask
,
SStreamQueueItem
*
pItem
);
bool
tInputQueueIsFull
(
const
SStreamTask
*
pTask
);
...
...
source/dnode/mnode/impl/src/mndDef.c
浏览文件 @
18479d81
...
...
@@ -70,7 +70,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if
(
tEncodeI32
(
pEncoder
,
innerSz
)
<
0
)
return
-
1
;
for
(
int32_t
j
=
0
;
j
<
innerSz
;
j
++
)
{
SStreamTask
*
pTask
=
taosArrayGetP
(
pArray
,
j
);
if
(
tEncodeS
S
treamTask
(
pEncoder
,
pTask
)
<
0
)
return
-
1
;
if
(
tEncodeStreamTask
(
pEncoder
,
pTask
)
<
0
)
return
-
1
;
}
}
...
...
@@ -130,7 +130,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
taosArrayDestroy
(
pArray
);
return
-
1
;
}
if
(
tDecodeS
S
treamTask
(
pDecoder
,
pTask
)
<
0
)
{
if
(
tDecodeStreamTask
(
pDecoder
,
pTask
)
<
0
)
{
taosMemoryFree
(
pTask
);
taosArrayDestroy
(
pArray
);
return
-
1
;
...
...
source/dnode/mnode/impl/src/mndScheduler.c
浏览文件 @
18479d81
...
...
@@ -224,7 +224,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
continue
;
}
SStreamTask
*
pTask
=
tNewS
S
treamTask
(
pStream
->
uid
);
SStreamTask
*
pTask
=
tNewStreamTask
(
pStream
->
uid
);
if
(
pTask
==
NULL
)
{
sdbRelease
(
pSdb
,
pVgroup
);
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
...
...
@@ -260,7 +260,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
int32_t
mndAddFixedSinkTaskToStream
(
SMnode
*
pMnode
,
SStreamObj
*
pStream
)
{
SArray
*
tasks
=
taosArrayGetP
(
pStream
->
tasks
,
0
);
SStreamTask
*
pTask
=
tNewS
S
treamTask
(
pStream
->
uid
);
SStreamTask
*
pTask
=
tNewStreamTask
(
pStream
->
uid
);
if
(
pTask
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
return
-
1
;
...
...
@@ -350,7 +350,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
return
-
1
;
}
pInnerTask
=
tNewS
S
treamTask
(
pStream
->
uid
);
pInnerTask
=
tNewStreamTask
(
pStream
->
uid
);
if
(
pInnerTask
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
qDestroyQueryPlan
(
pPlan
);
...
...
@@ -421,7 +421,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
continue
;
}
SStreamTask
*
pTask
=
tNewS
S
treamTask
(
pStream
->
uid
);
SStreamTask
*
pTask
=
tNewStreamTask
(
pStream
->
uid
);
if
(
pTask
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
sdbRelease
(
pSdb
,
pVgroup
);
...
...
@@ -491,7 +491,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
continue
;
}
SStreamTask
*
pTask
=
tNewS
S
treamTask
(
pStream
->
uid
);
SStreamTask
*
pTask
=
tNewStreamTask
(
pStream
->
uid
);
if
(
pTask
==
NULL
)
{
sdbRelease
(
pSdb
,
pVgroup
);
qDestroyQueryPlan
(
pPlan
);
...
...
source/dnode/mnode/impl/src/mndStream.c
浏览文件 @
18479d81
...
...
@@ -39,8 +39,8 @@ static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStream
static
int32_t
mndProcessCreateStreamReq
(
SRpcMsg
*
pReq
);
static
int32_t
mndProcessDropStreamReq
(
SRpcMsg
*
pReq
);
static
int32_t
mndProcessStreamCheckpointTmr
(
SRpcMsg
*
pReq
);
//
static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq);
/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/
static
int32_t
mndProcessStreamDoCheckpoint
(
SRpcMsg
*
pReq
);
static
int32_t
mndProcessRecoverStreamReq
(
SRpcMsg
*
pReq
);
static
int32_t
mndProcessStreamMetaReq
(
SRpcMsg
*
pReq
);
static
int32_t
mndGetStreamMeta
(
SRpcMsg
*
pReq
,
SShowObj
*
pShow
,
STableMetaRsp
*
pMeta
);
static
int32_t
mndRetrieveStream
(
SRpcMsg
*
pReq
,
SShowObj
*
pShow
,
SSDataBlock
*
pBlock
,
int32_t
rows
);
...
...
@@ -418,7 +418,7 @@ FAIL:
int32_t
mndPersistTaskDeployReq
(
STrans
*
pTrans
,
const
SStreamTask
*
pTask
)
{
SEncoder
encoder
;
tEncoderInit
(
&
encoder
,
NULL
,
0
);
tEncodeS
S
treamTask
(
&
encoder
,
pTask
);
tEncodeStreamTask
(
&
encoder
,
pTask
);
int32_t
size
=
encoder
.
pos
;
int32_t
tlen
=
sizeof
(
SMsgHead
)
+
size
;
tEncoderClear
(
&
encoder
);
...
...
@@ -430,7 +430,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) {
((
SMsgHead
*
)
buf
)
->
vgId
=
htonl
(
pTask
->
nodeId
);
void
*
abuf
=
POINTER_SHIFT
(
buf
,
sizeof
(
SMsgHead
));
tEncoderInit
(
&
encoder
,
abuf
,
size
);
tEncodeS
S
treamTask
(
&
encoder
,
pTask
);
tEncodeStreamTask
(
&
encoder
,
pTask
);
tEncoderClear
(
&
encoder
);
STransAction
action
=
{
0
};
...
...
source/dnode/snode/src/snode.c
浏览文件 @
18479d81
...
...
@@ -139,7 +139,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
}
SDecoder
decoder
;
tDecoderInit
(
&
decoder
,
(
uint8_t
*
)
msg
,
msgLen
);
code
=
tDecodeS
S
treamTask
(
&
decoder
,
pTask
);
code
=
tDecodeStreamTask
(
&
decoder
,
pTask
);
if
(
code
<
0
)
{
tDecoderClear
(
&
decoder
);
taosMemoryFree
(
pTask
);
...
...
source/dnode/vnode/src/tq/tq.c
浏览文件 @
18479d81
...
...
@@ -906,14 +906,13 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) {
pTask
->
inputStatus
=
TASK_INPUT_STATUS__NORMAL
;
pTask
->
outputStatus
=
TASK_OUTPUT_STATUS__NORMAL
;
pTask
->
pMsgCb
=
&
pTq
->
pVnode
->
msgCb
;
pTask
->
chkInfo
.
version
=
ver
;
pTask
->
pMeta
=
pTq
->
pStreamMeta
;
// expand executor
if
(
pTask
->
fillHistory
)
{
pTask
->
taskStatus
=
TASK_STATUS__WAIT_DOWNSTREAM
;
}
else
{
pTask
->
taskStatus
=
TASK_STATUS_RESTORE
;
pTask
->
taskStatus
=
TASK_STATUS_
_
RESTORE
;
}
if
(
pTask
->
taskLevel
==
TASK_LEVEL__SOURCE
)
{
...
...
@@ -1089,7 +1088,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
SDecoder
decoder
;
tDecoderInit
(
&
decoder
,
(
uint8_t
*
)
msg
,
msgLen
);
code
=
tDecodeS
S
treamTask
(
&
decoder
,
pTask
);
code
=
tDecodeStreamTask
(
&
decoder
,
pTask
);
if
(
code
<
0
)
{
tDecoderClear
(
&
decoder
);
taosMemoryFree
(
pTask
);
...
...
@@ -1485,8 +1484,9 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
if
(
pTask
->
taskStatus
==
TASK_STATUS__NORMAL
)
{
tqDebug
(
"vgId:%d s-task:%s start to process run req"
,
vgId
,
pTask
->
id
.
idStr
);
streamProcessRunReq
(
pTask
);
}
else
if
(
pTask
->
taskStatus
==
TASK_STATUS_RESTORE
)
{
tqDebug
(
"vgId:%d s-task:%s start to restore from last ck"
,
vgId
,
pTask
->
id
.
idStr
);
}
else
if
(
pTask
->
taskStatus
==
TASK_STATUS__RESTORE
)
{
tqDebug
(
"vgId:%d s-task:%s start to process in restore procedure from last chk point:%"
PRId64
,
vgId
,
pTask
->
id
.
idStr
,
pTask
->
chkInfo
.
version
);
streamProcessRunReq
(
pTask
);
}
else
{
tqDebug
(
"vgId:%d s-task:%s ignore run req since not in ready state"
,
vgId
,
pTask
->
id
.
idStr
);
...
...
source/dnode/vnode/src/tq/tqRestore.c
浏览文件 @
18479d81
...
...
@@ -49,9 +49,10 @@ int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList) {
}
// todo: add lock
for
(
int32_t
i
=
0
;
i
<
numOfTask
;
++
i
)
{
for
(
int32_t
i
=
0
;
i
<
numOfTask
;
++
i
)
{
SStreamTask
*
pTask
=
taosArrayGetP
(
pTaskList
,
i
);
tqDebug
(
"vgId:%d transfer s-task:%s state restore -> ready"
,
pStreamMeta
->
vgId
,
pTask
->
id
.
idStr
);
tqDebug
(
"vgId:%d transfer s-task:%s state restore -> ready, checkpoint:%"
PRId64
" checkpoint id:%"
PRId64
,
pStreamMeta
->
vgId
,
pTask
->
id
.
idStr
,
pTask
->
chkInfo
.
version
,
pTask
->
chkInfo
.
id
);
taosHashRemove
(
pStreamMeta
->
pRestoreTasks
,
&
pTask
->
id
.
taskId
,
sizeof
(
pTask
->
id
.
taskId
));
// NOTE: do not change the following order
...
...
source/libs/stream/src/streamExec.c
浏览文件 @
18479d81
...
...
@@ -21,11 +21,15 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
int32_t
code
=
TSDB_CODE_SUCCESS
;
void
*
pExecutor
=
pTask
->
exec
.
pExecutor
;
while
(
pTask
->
taskLevel
==
TASK_LEVEL__SOURCE
&&
atomic_load_8
(
&
pTask
->
taskStatus
)
!=
TASK_STATUS__NORMAL
)
{
while
(
pTask
->
taskLevel
==
TASK_LEVEL__SOURCE
)
{
int8_t
status
=
atomic_load_8
(
&
pTask
->
taskStatus
);
if
(
status
!=
TASK_STATUS__NORMAL
&&
status
!=
TASK_STATUS__RESTORE
)
{
qError
(
"stream task wait for the end of fill history, s-task:%s, status:%d"
,
pTask
->
id
.
idStr
,
atomic_load_8
(
&
pTask
->
taskStatus
));
taosMsleep
(
2
);
continue
;
}
else
{
break
;
}
}
// set input
...
...
source/libs/stream/src/streamMeta.c
浏览文件 @
18479d81
...
...
@@ -109,7 +109,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg,
}
SDecoder decoder;
tDecoderInit(&decoder, (uint8_t*)msg, msgLen);
if (tDecodeS
S
treamTask(&decoder, pTask) < 0) {
if (tDecodeStreamTask(&decoder, pTask) < 0) {
tDecoderClear(&decoder);
goto FAIL;
}
...
...
@@ -142,7 +142,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
void
*
buf
=
NULL
;
int32_t
len
;
int32_t
code
;
tEncodeSize
(
tEncodeS
S
treamTask
,
pTask
,
len
,
code
);
tEncodeSize
(
tEncodeStreamTask
,
pTask
,
len
,
code
);
if
(
code
<
0
)
{
return
-
1
;
}
...
...
@@ -153,7 +153,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) {
SEncoder
encoder
=
{
0
};
tEncoderInit
(
&
encoder
,
buf
,
len
);
tEncodeS
S
treamTask
(
&
encoder
,
pTask
);
tEncodeStreamTask
(
&
encoder
,
pTask
);
tEncoderClear
(
&
encoder
);
if
(
tdbTbUpsert
(
pMeta
->
pTaskDb
,
&
pTask
->
id
.
taskId
,
sizeof
(
int32_t
),
buf
,
len
,
pMeta
->
txn
)
<
0
)
{
...
...
@@ -321,7 +321,7 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
return
-
1
;
}
tDecoderInit
(
&
decoder
,
(
uint8_t
*
)
pVal
,
vLen
);
tDecodeS
S
treamTask
(
&
decoder
,
pTask
);
tDecodeStreamTask
(
&
decoder
,
pTask
);
tDecoderClear
(
&
decoder
);
if
(
pMeta
->
expandFunc
(
pMeta
->
ahandle
,
pTask
,
-
1
)
<
0
)
{
...
...
source/libs/stream/src/streamTask.c
浏览文件 @
18479d81
...
...
@@ -16,7 +16,7 @@
#include "executor.h"
#include "tstream.h"
SStreamTask
*
tNewS
S
treamTask
(
int64_t
streamId
)
{
SStreamTask
*
tNewStreamTask
(
int64_t
streamId
)
{
SStreamTask
*
pTask
=
(
SStreamTask
*
)
taosMemoryCalloc
(
1
,
sizeof
(
SStreamTask
));
if
(
pTask
==
NULL
)
{
return
NULL
;
...
...
@@ -54,7 +54,7 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) {
return
0
;
}
int32_t
tEncodeS
S
treamTask
(
SEncoder
*
pEncoder
,
const
SStreamTask
*
pTask
)
{
int32_t
tEncodeStreamTask
(
SEncoder
*
pEncoder
,
const
SStreamTask
*
pTask
)
{
if
(
tStartEncode
(
pEncoder
)
<
0
)
return
-
1
;
if
(
tEncodeI64
(
pEncoder
,
pTask
->
id
.
streamId
)
<
0
)
return
-
1
;
if
(
tEncodeI32
(
pEncoder
,
pTask
->
id
.
taskId
)
<
0
)
return
-
1
;
...
...
@@ -107,7 +107,7 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) {
return
pEncoder
->
pos
;
}
int32_t
tDecodeS
S
treamTask
(
SDecoder
*
pDecoder
,
SStreamTask
*
pTask
)
{
int32_t
tDecodeStreamTask
(
SDecoder
*
pDecoder
,
SStreamTask
*
pTask
)
{
if
(
tStartDecode
(
pDecoder
)
<
0
)
return
-
1
;
if
(
tDecodeI64
(
pDecoder
,
&
pTask
->
id
.
streamId
)
<
0
)
return
-
1
;
if
(
tDecodeI32
(
pDecoder
,
&
pTask
->
id
.
taskId
)
<
0
)
return
-
1
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录