Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
a89ce1a2
T
TDengine
项目概览
taosdata
/
TDengine
1 年多 前同步成功
通知
1185
Star
22016
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
a89ce1a2
编写于
8月 12, 2023
作者:
H
Haojun Liao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix(stream): transfer state by using data block.
上级
dea30255
变更
4
显示空白变更内容
内联
并排
Showing
4 changed file
with
82 addition
and
9 deletion
+82
-9
include/common/tcommon.h
include/common/tcommon.h
+1
-0
include/libs/stream/tstream.h
include/libs/stream/tstream.h
+1
-0
source/libs/stream/src/streamExec.c
source/libs/stream/src/streamExec.c
+51
-9
source/libs/stream/src/streamRecover.c
source/libs/stream/src/streamRecover.c
+29
-0
未找到文件。
include/common/tcommon.h
浏览文件 @
a89ce1a2
...
...
@@ -152,6 +152,7 @@ enum {
STREAM_INPUT__DATA_RETRIEVE
,
STREAM_INPUT__GET_RES
,
STREAM_INPUT__CHECKPOINT
,
STREAM_INPUT__TRANS_STATE
,
STREAM_INPUT__REF_DATA_BLOCK
,
STREAM_INPUT__DESTROY
,
};
...
...
include/libs/stream/tstream.h
浏览文件 @
a89ce1a2
...
...
@@ -627,6 +627,7 @@ int32_t streamSourceScanHistoryData(SStreamTask* pTask);
int32_t
streamDispatchScanHistoryFinishMsg
(
SStreamTask
*
pTask
);
int32_t
streamDispatchTransferStateMsg
(
SStreamTask
*
pTask
);
int32_t
appendTranstateIntoInputQ
(
SStreamTask
*
pTask
);
// agg level
int32_t
streamTaskScanHistoryPrepare
(
SStreamTask
*
pTask
);
...
...
source/libs/stream/src/streamExec.c
浏览文件 @
a89ce1a2
...
...
@@ -390,6 +390,11 @@ static int32_t streamTransferStateToStreamTask(SStreamTask* pTask) {
if
(
level
==
TASK_LEVEL__SOURCE
)
{
streamTaskFillHistoryFinished
(
pTask
);
streamTaskEndScanWAL
(
pTask
);
code
=
streamDoTransferStateToStreamTask
(
pTask
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
// todo handle this
return
code
;
}
}
else
if
(
level
==
TASK_LEVEL__AGG
)
{
// do transfer task operator states.
code
=
streamDoTransferStateToStreamTask
(
pTask
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
// todo handle this
...
...
@@ -460,6 +465,40 @@ static int32_t extractMsgFromInputQ(SStreamTask* pTask, SStreamQueueItem** pInpu
}
}
int32_t
streamProcessTranstateBlock
(
SStreamTask
*
pTask
,
SStreamDataBlock
*
pBlock
)
{
const
char
*
id
=
pTask
->
id
.
idStr
;
int32_t
code
=
TSDB_CODE_SUCCESS
;
int32_t
level
=
pTask
->
info
.
taskLevel
;
if
(
level
==
TASK_LEVEL__AGG
||
level
==
TASK_LEVEL__SINK
)
{
int32_t
remain
=
streamAlignTransferState
(
pTask
);
if
(
remain
>
0
)
{
qDebug
(
"s-task:%s receive upstream transfer state msg, remain:%d"
,
id
,
remain
);
return
0
;
}
// transfer the ownership of executor state
qDebug
(
"s-task:%s all upstream tasks send transfer msg, open transfer state flag"
,
id
);
ASSERT
(
pTask
->
streamTaskId
.
taskId
!=
0
&&
pTask
->
info
.
fillHistory
==
1
);
pTask
->
status
.
transferState
=
true
;
}
// dispatch the transtate block to downstream task immediately
if
(
level
==
TASK_LEVEL__SOURCE
||
level
==
TASK_LEVEL__AGG
)
{
// pBlock-> = pTask->id.taskId;
pBlock
->
srcVgId
=
pTask
->
pMeta
->
vgId
;
code
=
taosWriteQitem
(
pTask
->
outputInfo
.
queue
->
queue
,
pBlock
);
if
(
code
==
0
)
{
streamDispatchStreamBlock
(
pTask
);
}
else
{
streamFreeQitem
((
SStreamQueueItem
*
)
pBlock
);
}
}
return
code
;
}
/**
* todo: the batch of blocks should be tuned dynamic, according to the total elapsed time of each batch of blocks, the
* appropriate batch of blocks should be handled in 5 to 10 sec.
...
...
@@ -484,6 +523,11 @@ int32_t streamExecForAll(SStreamTask* pTask) {
break
;
}
if
(
pInput
->
type
==
STREAM_INPUT__TRANS_STATE
)
{
streamProcessTranstateBlock
(
pTask
,
(
SStreamDataBlock
*
)
pInput
);
return
0
;
}
if
(
pTask
->
info
.
taskLevel
==
TASK_LEVEL__SINK
)
{
ASSERT
(
pInput
->
type
==
STREAM_INPUT__DATA_BLOCK
);
qDebug
(
"s-task:%s sink task start to sink %d blocks"
,
id
,
batchSize
);
...
...
@@ -557,17 +601,15 @@ int32_t streamTaskEndScanWAL(SStreamTask* pTask) {
qDebug
(
"s-task:%s scan-history from WAL stage(step 2) ended, elapsed time:%.2fs"
,
id
,
el
);
// 1. notify all downstream tasks to transfer executor state after handle all history blocks.
int32_t
code
=
streamDispatchTransferStateMsg
(
pTask
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
// todo handle error
}
// pTask->status.transferState = true;
appendTranstateIntoInputQ
(
pTask
);
// 2. do transfer stream task operator states.
pTask
->
status
.
transferState
=
true
;
code
=
streamDoTransferStateToStreamTask
(
pTask
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
// todo handle error
return
code
;
}
// todo remove this
// int32_t
code = streamDoTransferStateToStreamTask(pTask);
// if (code != TSDB_CODE_SUCCESS) {
// todo handle error
//
return code;
//
}
return
TSDB_CODE_SUCCESS
;
}
...
...
source/libs/stream/src/streamRecover.c
浏览文件 @
a89ce1a2
...
...
@@ -415,6 +415,35 @@ static int32_t doDispatchTransferMsg(SStreamTask* pTask, const SStreamTransferRe
return
0
;
}
int32_t
appendTranstateIntoInputQ
(
SStreamTask
*
pTask
)
{
SStreamDataBlock
*
pTranstate
=
taosAllocateQitem
(
sizeof
(
SStreamDataBlock
),
DEF_QITEM
,
sizeof
(
SSDataBlock
));
if
(
pTranstate
==
NULL
)
{
return
TSDB_CODE_OUT_OF_MEMORY
;
}
SSDataBlock
*
pBlock
=
taosMemoryCalloc
(
1
,
sizeof
(
SSDataBlock
));
if
(
pBlock
==
NULL
)
{
taosFreeQitem
(
pTranstate
);
return
TSDB_CODE_OUT_OF_MEMORY
;
}
pBlock
->
info
.
type
=
STREAM_TRANS_STATE
;
pBlock
->
info
.
rows
=
1
;
pBlock
->
info
.
childId
=
pTask
->
info
.
selfChildId
;
pTranstate
->
blocks
=
taosArrayInit
(
4
,
sizeof
(
SSDataBlock
));
//pBlock;
taosArrayPush
(
pTranstate
->
blocks
,
pBlock
);
taosMemoryFree
(
pBlock
);
if
(
tAppendDataToInputQueue
(
pTask
,
(
SStreamQueueItem
*
)
pTranstate
)
<
0
)
{
taosFreeQitem
(
pTranstate
);
return
TSDB_CODE_OUT_OF_MEMORY
;
}
streamSchedExec
(
pTask
);
return
TSDB_CODE_SUCCESS
;
}
int32_t
streamDispatchTransferStateMsg
(
SStreamTask
*
pTask
)
{
SStreamTransferReq
req
=
{
.
streamId
=
pTask
->
id
.
streamId
,
.
childId
=
pTask
->
info
.
selfChildId
};
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录