Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
8958aabe
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1180
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
8958aabe
编写于
5月 05, 2023
作者:
H
Haojun Liao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix(stream): opt task scan performance.
上级
8b12d4d3
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
49 addition
and
30 deletion
+49
-30
include/libs/stream/tstream.h
include/libs/stream/tstream.h
+1
-0
source/dnode/snode/src/snode.c
source/dnode/snode/src/snode.c
+4
-0
source/dnode/vnode/src/tq/tq.c
source/dnode/vnode/src/tq/tq.c
+6
-3
source/dnode/vnode/src/tq/tqPush.c
source/dnode/vnode/src/tq/tqPush.c
+3
-2
source/dnode/vnode/src/tq/tqRestore.c
source/dnode/vnode/src/tq/tqRestore.c
+10
-24
source/libs/stream/src/streamMeta.c
source/libs/stream/src/streamMeta.c
+25
-1
未找到文件。
include/libs/stream/tstream.h
浏览文件 @
8958aabe
...
...
@@ -340,6 +340,7 @@ typedef struct SStreamMeta {
TTB
*
pTaskDb
;
TTB
*
pCheckpointDb
;
SHashObj
*
pTasks
;
SArray
*
pTaskList
;
// SArray<task_id*>
void
*
ahandle
;
TXN
*
txn
;
FTaskExpand
*
expandFunc
;
...
...
source/dnode/snode/src/snode.c
浏览文件 @
8958aabe
...
...
@@ -153,11 +153,15 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) {
ASSERT
(
pTask
->
taskLevel
==
TASK_LEVEL__AGG
);
// 2.save task
taosWLockLatch
(
&
pSnode
->
pMeta
->
lock
);
code
=
streamMetaAddDeployedTask
(
pSnode
->
pMeta
,
-
1
,
pTask
);
if
(
code
<
0
)
{
taosWUnLockLatch
(
&
pSnode
->
pMeta
->
lock
);
return
-
1
;
}
taosWUnLockLatch
(
&
pSnode
->
pMeta
->
lock
);
// 3.go through recover steps to fill history
if
(
pTask
->
fillHistory
)
{
streamSetParamForRecover
(
pTask
);
...
...
source/dnode/vnode/src/tq/tq.c
浏览文件 @
8958aabe
...
...
@@ -781,13 +781,17 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
tDecoderClear
(
&
decoder
);
// 2.save task, use the newest commit version as the initial start version of stream task.
taosWLockLatch
(
&
pTq
->
pStreamMeta
->
lock
);
code
=
streamMetaAddDeployedTask
(
pTq
->
pStreamMeta
,
sversion
,
pTask
);
if
(
code
<
0
)
{
tqError
(
"vgId:%d failed to add s-task:%s, total:%d"
,
TD_VID
(
pTq
->
pVnode
),
pTask
->
id
.
idStr
,
streamMetaGetNumOfTasks
(
pTq
->
pStreamMeta
));
taosWUnLockLatch
(
&
pTq
->
pStreamMeta
->
lock
);
return
-
1
;
}
taosWUnLockLatch
(
&
pTq
->
pStreamMeta
->
lock
);
// 3.go through recover steps to fill history
if
(
pTask
->
fillHistory
)
{
streamTaskCheckDownstream
(
pTask
,
sversion
);
...
...
@@ -1323,13 +1327,12 @@ FAIL:
int32_t
tqCheckLogInWal
(
STQ
*
pTq
,
int64_t
sversion
)
{
return
sversion
<=
pTq
->
walLogLastVer
;
}
int32_t
tqStartStreamTasks
(
STQ
*
pTq
)
{
int32_t
vgId
=
TD_VID
(
pTq
->
pVnode
);
int32_t
vgId
=
TD_VID
(
pTq
->
pVnode
);
SStreamMeta
*
pMeta
=
pTq
->
pStreamMeta
;
taosWLockLatch
(
&
pMeta
->
lock
);
int32_t
numOfTasks
=
taos
HashGetSize
(
pTq
->
pStreamMeta
->
pTasks
);
int32_t
numOfTasks
=
taos
ArrayGetSize
(
pMeta
->
pTaskList
);
if
(
numOfTasks
==
0
)
{
tqInfo
(
"vgId:%d no stream tasks exists"
,
vgId
);
taosWUnLockLatch
(
&
pTq
->
pStreamMeta
->
lock
);
...
...
source/dnode/vnode/src/tq/tqPush.c
浏览文件 @
8958aabe
...
...
@@ -262,14 +262,15 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v
// taosWUnLockLatch(&pTq->lock);
}
tqDebug
(
"handle submit, restore:%d, size:%d"
,
pTq
->
pVnode
->
restored
,
(
int
)
taosHashGetSize
(
pTq
->
pStreamMeta
->
pTasks
));
int32_t
numOfTasks
=
streamMetaGetNumOfTasks
(
pTq
->
pStreamMeta
);
tqDebug
(
"handle submit, restore:%d, size:%d"
,
pTq
->
pVnode
->
restored
,
numOfTasks
);
// push data for stream processing:
// 1. the vnode has already been restored.
// 2. the vnode should be the leader.
// 3. the stream is not suspended yet.
if
(
!
tsDisableStream
&&
vnodeIsRoleLeader
(
pTq
->
pVnode
)
&&
pTq
->
pVnode
->
restored
)
{
if
(
taosHashGetSize
(
pTq
->
pStreamMeta
->
pTasks
)
==
0
)
{
if
(
numOfTasks
==
0
)
{
return
0
;
}
...
...
source/dnode/vnode/src/tq/tqRestore.c
浏览文件 @
8958aabe
...
...
@@ -57,42 +57,28 @@ int32_t tqStreamTasksScanWal(STQ* pTq) {
return
0
;
}
static
SArray
*
extractTaskIdList
(
SStreamMeta
*
pStreamMeta
,
int32_t
numOfTasks
)
{
SArray
*
pTaskIdList
=
taosArrayInit
(
numOfTasks
,
sizeof
(
int32_t
));
void
*
pIter
=
NULL
;
taosWLockLatch
(
&
pStreamMeta
->
lock
);
while
(
1
)
{
pIter
=
taosHashIterate
(
pStreamMeta
->
pTasks
,
pIter
);
if
(
pIter
==
NULL
)
{
break
;
}
SStreamTask
*
pTask
=
*
(
SStreamTask
**
)
pIter
;
taosArrayPush
(
pTaskIdList
,
&
pTask
->
id
.
taskId
);
}
taosWUnLockLatch
(
&
pStreamMeta
->
lock
);
return
pTaskIdList
;
}
int32_t
createStreamRunReq
(
SStreamMeta
*
pStreamMeta
,
bool
*
pScanIdle
)
{
*
pScanIdle
=
true
;
bool
noNewDataInWal
=
true
;
int32_t
vgId
=
pStreamMeta
->
vgId
;
int32_t
numOfTasks
=
taos
HashGetSize
(
pStreamMeta
->
pTasks
);
int32_t
numOfTasks
=
taos
ArrayGetSize
(
pStreamMeta
->
pTaskList
);
if
(
numOfTasks
==
0
)
{
return
TSDB_CODE_SUCCESS
;
}
SArray
*
pTaskList
=
NULL
;
taosWLockLatch
(
&
pStreamMeta
->
lock
);
pTaskList
=
taosArrayDup
(
pStreamMeta
->
pTaskList
,
NULL
);
taosWUnLockLatch
(
&
pStreamMeta
->
lock
);
tqDebug
(
"vgId:%d start to check wal to extract new submit block for %d tasks"
,
vgId
,
numOfTasks
);
SArray
*
pTaskIdList
=
extractTaskIdList
(
pStreamMeta
,
numOfTasks
);
// update the new task number
numOfTasks
=
taosArrayGetSize
(
pTaskIdList
);
numOfTasks
=
taosArrayGetSize
(
pTaskList
);
for
(
int32_t
i
=
0
;
i
<
numOfTasks
;
++
i
)
{
int32_t
*
pTaskId
=
taosArrayGet
(
pTask
Id
List
,
i
);
int32_t
*
pTaskId
=
taosArrayGet
(
pTaskList
,
i
);
SStreamTask
*
pTask
=
streamMetaAcquireTask
(
pStreamMeta
,
*
pTaskId
);
if
(
pTask
==
NULL
)
{
continue
;
...
...
@@ -166,7 +152,7 @@ int32_t createStreamRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
*
pScanIdle
=
true
;
}
taosArrayDestroy
(
pTask
Id
List
);
taosArrayDestroy
(
pTaskList
);
return
0
;
}
source/libs/stream/src/streamMeta.c
浏览文件 @
8958aabe
...
...
@@ -57,6 +57,13 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
goto
_err
;
}
// task list
pMeta
->
pTaskList
=
taosArrayInit
(
4
,
sizeof
(
int32_t
));
if
(
pMeta
->
pTaskList
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
goto
_err
;
}
if
(
streamMetaBegin
(
pMeta
)
<
0
)
{
goto
_err
;
}
...
...
@@ -70,6 +77,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF
_err:
taosMemoryFree
(
pMeta
->
path
);
if
(
pMeta
->
pTasks
)
taosHashCleanup
(
pMeta
->
pTasks
);
if
(
pMeta
->
pTaskList
)
taosArrayDestroy
(
pMeta
->
pTaskList
);
if
(
pMeta
->
pTaskDb
)
tdbTbClose
(
pMeta
->
pTaskDb
);
if
(
pMeta
->
pCheckpointDb
)
tdbTbClose
(
pMeta
->
pCheckpointDb
);
if
(
pMeta
->
db
)
tdbClose
(
pMeta
->
db
);
...
...
@@ -100,6 +108,7 @@ void streamMetaClose(SStreamMeta* pMeta) {
}
taosHashCleanup
(
pMeta
->
pTasks
);
pMeta
->
pTaskList
=
taosArrayDestroy
(
pMeta
->
pTaskList
);
taosMemoryFree
(
pMeta
->
path
);
taosMemoryFree
(
pMeta
);
}
...
...
@@ -180,11 +189,15 @@ int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask*
}
taosHashPut
(
pMeta
->
pTasks
,
&
pTask
->
id
.
taskId
,
sizeof
(
int32_t
),
&
pTask
,
POINTER_BYTES
);
taosArrayPush
(
pMeta
->
pTaskList
,
&
pTask
->
id
.
taskId
);
return
0
;
}
int32_t
streamMetaGetNumOfTasks
(
const
SStreamMeta
*
pMeta
)
{
return
(
int32_t
)
taosHashGetSize
(
pMeta
->
pTasks
);
size_t
size
=
taosHashGetSize
(
pMeta
->
pTasks
);
ASSERT
(
taosArrayGetSize
(
pMeta
->
pTaskList
)
==
taosHashGetSize
(
pMeta
->
pTasks
));
return
(
int32_t
)
size
;
}
SStreamTask
*
streamMetaAcquireTask
(
SStreamMeta
*
pMeta
,
int32_t
taskId
)
{
...
...
@@ -224,6 +237,15 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) {
atomic_store_8
(
&
pTask
->
status
.
taskStatus
,
TASK_STATUS__STOP
);
int32_t
num
=
taosArrayGetSize
(
pMeta
->
pTaskList
);
for
(
int32_t
i
=
0
;
i
<
num
;
++
i
)
{
int32_t
*
pTaskId
=
taosArrayGet
(
pMeta
->
pTaskList
,
i
);
if
(
*
pTaskId
==
taskId
)
{
taosArrayRemove
(
pMeta
->
pTaskList
,
i
);
break
;
}
}
streamMetaReleaseTask
(
pMeta
,
pTask
);
taosWUnLockLatch
(
&
pMeta
->
lock
);
}
...
...
@@ -308,6 +330,8 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) {
return
-
1
;
}
taosArrayPush
(
pMeta
->
pTaskList
,
&
pTask
->
id
.
taskId
);
if
(
pTask
->
fillHistory
)
{
pTask
->
status
.
taskStatus
=
TASK_STATUS__WAIT_DOWNSTREAM
;
streamTaskCheckDownstream
(
pTask
,
ver
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录