Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
c6a322e0
T
TDengine
项目概览
taosdata
/
TDengine
大约 2 年 前同步成功
通知
1192
Star
22018
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
c6a322e0
编写于
7月 16, 2023
作者:
H
Haojun Liao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor(stream): back pressure is active, which is trigger by dispatch rsp.
上级
725db16a
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
23 addition
and
24 deletion
+23
-24
include/libs/stream/tstream.h
include/libs/stream/tstream.h
+0
-3
source/dnode/vnode/src/tq/tq.c
source/dnode/vnode/src/tq/tq.c
+0
-8
source/dnode/vnode/src/tq/tqRestore.c
source/dnode/vnode/src/tq/tqRestore.c
+8
-1
source/libs/stream/src/streamCheckpoint.c
source/libs/stream/src/streamCheckpoint.c
+2
-2
source/libs/stream/src/streamDispatch.c
source/libs/stream/src/streamDispatch.c
+12
-9
source/libs/stream/src/streamExec.c
source/libs/stream/src/streamExec.c
+1
-1
未找到文件。
include/libs/stream/tstream.h
浏览文件 @
c6a322e0
...
...
@@ -62,15 +62,12 @@ enum {
enum
{
TASK_INPUT_STATUS__NORMAL
=
1
,
TASK_INPUT_STATUS__BLOCKED
,
TASK_INPUT_STATUS__RECOVER
,
TASK_INPUT_STATUS__STOP
,
TASK_INPUT_STATUS__FAILED
,
};
enum
{
TASK_OUTPUT_STATUS__NORMAL
=
1
,
TASK_OUTPUT_STATUS__WAIT
,
TASK_OUTPUT_STATUS__BLOCKED
,
};
enum
{
...
...
source/dnode/vnode/src/tq/tq.c
浏览文件 @
c6a322e0
...
...
@@ -761,17 +761,9 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver, int64_t checkpoi
pTask
->
pMsgCb
=
&
pTq
->
pVnode
->
msgCb
;
pTask
->
pMeta
=
pTq
->
pStreamMeta
;
// checkpoint exists, restore from the last checkpoint
// if (pTask->chkInfo.checkpointId != 0) {
// ASSERT(pTask->chkInfo.checkpointVer > 0);
// pTask->chkInfo.currentVer = pTask->chkInfo.checkpointVer;
// pTask->dataRange.range.maxVer = pTask->chkInfo.checkpointVer;
// pTask->dataRange.range.minVer = pTask->chkInfo.checkpointVer;
// } else {
pTask
->
chkInfo
.
currentVer
=
ver
;
pTask
->
dataRange
.
range
.
maxVer
=
ver
;
pTask
->
dataRange
.
range
.
minVer
=
ver
;
//}
if
(
pTask
->
info
.
taskLevel
==
TASK_LEVEL__SOURCE
)
{
SStreamTask
*
pSateTask
=
pTask
;
...
...
source/dnode/vnode/src/tq/tqRestore.c
浏览文件 @
c6a322e0
...
...
@@ -240,8 +240,9 @@ int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
}
int32_t
status
=
pTask
->
status
.
taskStatus
;
const
char
*
pStatus
=
streamGetTaskStatusStr
(
status
);
if
(
status
!=
TASK_STATUS__NORMAL
)
{
tqDebug
(
"s-task:%s not ready for new submit block from wal, status:%s"
,
pTask
->
id
.
idStr
,
streamGetTaskStatusStr
(
status
)
);
tqDebug
(
"s-task:%s not ready for new submit block from wal, status:%s"
,
pTask
->
id
.
idStr
,
pStatus
);
streamMetaReleaseTask
(
pStreamMeta
,
pTask
);
continue
;
}
...
...
@@ -252,6 +253,12 @@ int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
continue
;
}
if
(
pTask
->
inputStatus
==
TASK_INPUT_STATUS__BLOCKED
)
{
tqDebug
(
"s-task:%s inputQ is blocked, do nothing"
,
pTask
->
id
.
idStr
);
streamMetaReleaseTask
(
pStreamMeta
,
pTask
);
continue
;
}
*
pScanIdle
=
false
;
// seek the stored version and extract data from WAL
...
...
source/libs/stream/src/streamCheckpoint.c
浏览文件 @
c6a322e0
...
...
@@ -244,10 +244,10 @@ int32_t streamSaveTasks(SStreamMeta* pMeta, int64_t checkpointId) {
uint32_t
*
pTaskId
=
taosArrayGet
(
pMeta
->
pTaskList
,
i
);
SStreamTask
*
p
=
*
(
SStreamTask
**
)
taosHashGet
(
pMeta
->
pTasks
,
pTaskId
,
sizeof
(
*
pTaskId
));
int8_t
prev
=
p
->
status
.
taskStatus
;
ASSERT
(
p
->
chkInfo
.
checkpointId
<
p
->
checkpointingId
&&
p
->
checkpointingId
==
checkpointId
);
p
->
chkInfo
.
checkpointId
=
p
->
checkpointingId
;
int8_t
prev
=
p
->
status
.
taskStatus
;
p
->
chkInfo
.
checkpointId
=
p
->
checkpointingId
;
p
->
status
.
taskStatus
=
TASK_STATUS__NORMAL
;
// save the task
...
...
source/libs/stream/src/streamDispatch.c
浏览文件 @
c6a322e0
...
...
@@ -794,13 +794,13 @@ int32_t streamAddCheckpointReadyMsg(SStreamTask* pTask, int32_t upstreamTaskId,
return
0
;
}
// todo record the idle time for dispatch data
int32_t
streamProcessDispatchRsp
(
SStreamTask
*
pTask
,
SStreamDispatchRsp
*
pRsp
,
int32_t
code
)
{
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
// dispatch message failed: network error, or node not available.
// in case of the input queue is full, the code will be TSDB_CODE_SUCCESS, the and pRsp>inputStatus will be set
// flag. here we need to retry dispatch this message to downstream task immediately. handle the case the failure
// happened too fast. todo handle the shuffle dispatch failure
// happened too fast.
// todo handle the shuffle dispatch failure
qError
(
"s-task:%s failed to dispatch msg to task:0x%x, code:%s, retry cnt:%d"
,
pTask
->
id
.
idStr
,
pRsp
->
downstreamTaskId
,
tstrerror
(
code
),
++
pTask
->
msgInfo
.
retryCount
);
int32_t
ret
=
doDispatchAllBlocks
(
pTask
,
pTask
->
msgInfo
.
pData
);
...
...
@@ -810,7 +810,8 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
return
TSDB_CODE_SUCCESS
;
}
qDebug
(
"s-task:%s receive dispatch rsp, output status:%d code:%d"
,
pTask
->
id
.
idStr
,
pRsp
->
inputStatus
,
code
);
qDebug
(
"s-task:%s recv dispatch rsp, downstream task input status:%d code:%d"
,
pTask
->
id
.
idStr
,
pRsp
->
inputStatus
,
code
);
// there are other dispatch message not response yet
if
(
pTask
->
outputType
==
TASK_OUTPUT__SHUFFLE_DISPATCH
)
{
...
...
@@ -828,14 +829,12 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
// the input queue of the (down stream) task that receive the output data is full,
// so the TASK_INPUT_STATUS_BLOCKED is rsp
// todo blocking the output status
if
(
pRsp
->
inputStatus
==
TASK_INPUT_STATUS__BLOCKED
)
{
pTask
->
inputStatus
=
TASK_INPUT_STATUS__BLOCKED
;
// block the input of current task, to push pressure to upstream
pTask
->
msgInfo
.
blockingTs
=
taosGetTimestampMs
();
// record the blocking start time
int32_t
waitDuration
=
300
;
// 300 ms
qError
(
"s-task:%s inputQ of downstream task:0x%x is full, time:%"
PRId64
"wait for %dms and retry dispatch data"
,
pTask
->
id
.
idStr
,
pRsp
->
downstreamTaskId
,
pTask
->
msgInfo
.
blockingTs
,
waitDuration
);
streamRetryDispatchStreamBlock
(
pTask
,
waitDuration
);
pTask
->
id
.
idStr
,
pRsp
->
downstreamTaskId
,
pTask
->
msgInfo
.
blockingTs
,
DISPATCH_RETRY_INTERVAL_MS
);
streamRetryDispatchStreamBlock
(
pTask
,
DISPATCH_RETRY_INTERVAL_MS
);
}
else
{
// pipeline send data in output queue
// this message has been sent successfully, let's try next one.
destroyStreamDataBlock
(
pTask
->
msgInfo
.
pData
);
...
...
@@ -843,8 +842,12 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i
if
(
pTask
->
msgInfo
.
blockingTs
!=
0
)
{
int64_t
el
=
taosGetTimestampMs
()
-
pTask
->
msgInfo
.
blockingTs
;
qDebug
(
"s-task:%s resume to normal from inputQ blocking, idle time:%"
PRId64
"ms"
,
pTask
->
id
.
idStr
,
el
);
qDebug
(
"s-task:%s downstream task:0x%x resume to normal from inputQ blocking, blocking time:%"
PRId64
"ms"
,
pTask
->
id
.
idStr
,
pRsp
->
downstreamTaskId
,
el
);
pTask
->
msgInfo
.
blockingTs
=
0
;
// put data into inputQ of current task is also allowed
pTask
->
inputStatus
=
TASK_INPUT_STATUS__NORMAL
;
}
// now ready for next data output
...
...
source/libs/stream/src/streamExec.c
浏览文件 @
c6a322e0
...
...
@@ -512,7 +512,7 @@ bool streamTaskIsIdle(const SStreamTask* pTask) {
}
// blocked by downstream task
if
(
pTask
->
outputStatus
==
TASK_OUT
PUT_STATUS__BLOCKED
)
{
if
(
pTask
->
inputStatus
==
TASK_IN
PUT_STATUS__BLOCKED
)
{
return
false
;
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录