Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
2977a4d0
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1181
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
2977a4d0
编写于
5月 30, 2023
作者:
H
Haojun Liao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor: do some internal refactor.
上级
3628888e
变更
7
显示空白变更内容
内联
并排
Showing
7 changed file
with
199 addition
and
92 deletion
+199
-92
source/dnode/mnode/impl/inc/mndDef.h
source/dnode/mnode/impl/inc/mndDef.h
+15
-8
source/dnode/mnode/impl/src/mndDef.c
source/dnode/mnode/impl/src/mndDef.c
+10
-10
source/dnode/mnode/impl/src/mndDump.c
source/dnode/mnode/impl/src/mndDump.c
+4
-4
source/dnode/mnode/impl/src/mndScheduler.c
source/dnode/mnode/impl/src/mndScheduler.c
+47
-48
source/dnode/mnode/impl/src/mndSma.c
source/dnode/mnode/impl/src/mndSma.c
+10
-10
source/dnode/mnode/impl/src/mndStream.c
source/dnode/mnode/impl/src/mndStream.c
+13
-12
source/libs/stream/src/streamTask.c
source/libs/stream/src/streamTask.c
+100
-0
未找到文件。
source/dnode/mnode/impl/inc/mndDef.h
浏览文件 @
2977a4d0
...
...
@@ -637,6 +637,14 @@ typedef struct {
SMqSubActionLogEntry
*
pLogEntry
;
}
SMqRebOutputObj
;
typedef
struct
SStreamConf
{
int8_t
igExpired
;
int8_t
trigger
;
int8_t
fillHistory
;
int64_t
triggerParam
;
int64_t
watermark
;
}
SStreamConf
;
typedef
struct
{
char
name
[
TSDB_STREAM_FNAME_LEN
];
// ctl
...
...
@@ -650,12 +658,7 @@ typedef struct {
// info
int64_t
uid
;
int8_t
status
;
// config
int8_t
igExpired
;
int8_t
trigger
;
int8_t
fillHistory
;
int64_t
triggerParam
;
int64_t
watermark
;
SStreamConf
conf
;
// source and target
int64_t
sourceDbUid
;
int64_t
targetDbUid
;
...
...
@@ -673,6 +676,10 @@ typedef struct {
char
*
ast
;
char
*
physicalPlan
;
SArray
*
tasks
;
// SArray<SArray<SStreamTask>>
SArray
*
pBatchTask
;
// generate the results for already stored ts data
int64_t
batchTaskUid
;
// stream task for history ts data
SSchemaWrapper
outputSchema
;
SSchemaWrapper
tagSchema
;
...
...
source/dnode/mnode/impl/src/mndDef.c
浏览文件 @
2977a4d0
...
...
@@ -30,11 +30,11 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) {
if
(
tEncodeI64
(
pEncoder
,
pObj
->
uid
)
<
0
)
return
-
1
;
if
(
tEncodeI8
(
pEncoder
,
pObj
->
status
)
<
0
)
return
-
1
;
if
(
tEncodeI8
(
pEncoder
,
pObj
->
igExpired
)
<
0
)
return
-
1
;
if
(
tEncodeI8
(
pEncoder
,
pObj
->
trigger
)
<
0
)
return
-
1
;
if
(
tEncodeI8
(
pEncoder
,
pObj
->
fillHistory
)
<
0
)
return
-
1
;
if
(
tEncodeI64
(
pEncoder
,
pObj
->
triggerParam
)
<
0
)
return
-
1
;
if
(
tEncodeI64
(
pEncoder
,
pObj
->
watermark
)
<
0
)
return
-
1
;
if
(
tEncodeI8
(
pEncoder
,
pObj
->
conf
.
igExpired
)
<
0
)
return
-
1
;
if
(
tEncodeI8
(
pEncoder
,
pObj
->
conf
.
trigger
)
<
0
)
return
-
1
;
if
(
tEncodeI8
(
pEncoder
,
pObj
->
conf
.
fillHistory
)
<
0
)
return
-
1
;
if
(
tEncodeI64
(
pEncoder
,
pObj
->
conf
.
triggerParam
)
<
0
)
return
-
1
;
if
(
tEncodeI64
(
pEncoder
,
pObj
->
conf
.
watermark
)
<
0
)
return
-
1
;
if
(
tEncodeI64
(
pEncoder
,
pObj
->
sourceDbUid
)
<
0
)
return
-
1
;
if
(
tEncodeI64
(
pEncoder
,
pObj
->
targetDbUid
)
<
0
)
return
-
1
;
...
...
@@ -97,11 +97,11 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) {
if
(
tDecodeI64
(
pDecoder
,
&
pObj
->
uid
)
<
0
)
return
-
1
;
if
(
tDecodeI8
(
pDecoder
,
&
pObj
->
status
)
<
0
)
return
-
1
;
if
(
tDecodeI8
(
pDecoder
,
&
pObj
->
igExpired
)
<
0
)
return
-
1
;
if
(
tDecodeI8
(
pDecoder
,
&
pObj
->
trigger
)
<
0
)
return
-
1
;
if
(
tDecodeI8
(
pDecoder
,
&
pObj
->
fillHistory
)
<
0
)
return
-
1
;
if
(
tDecodeI64
(
pDecoder
,
&
pObj
->
triggerParam
)
<
0
)
return
-
1
;
if
(
tDecodeI64
(
pDecoder
,
&
pObj
->
watermark
)
<
0
)
return
-
1
;
if
(
tDecodeI8
(
pDecoder
,
&
pObj
->
conf
.
igExpired
)
<
0
)
return
-
1
;
if
(
tDecodeI8
(
pDecoder
,
&
pObj
->
conf
.
trigger
)
<
0
)
return
-
1
;
if
(
tDecodeI8
(
pDecoder
,
&
pObj
->
conf
.
fillHistory
)
<
0
)
return
-
1
;
if
(
tDecodeI64
(
pDecoder
,
&
pObj
->
conf
.
triggerParam
)
<
0
)
return
-
1
;
if
(
tDecodeI64
(
pDecoder
,
&
pObj
->
conf
.
watermark
)
<
0
)
return
-
1
;
if
(
tDecodeI64
(
pDecoder
,
&
pObj
->
sourceDbUid
)
<
0
)
return
-
1
;
if
(
tDecodeI64
(
pDecoder
,
&
pObj
->
targetDbUid
)
<
0
)
return
-
1
;
...
...
source/dnode/mnode/impl/src/mndDump.c
浏览文件 @
2977a4d0
...
...
@@ -367,10 +367,10 @@ void dumpStream(SSdb *pSdb, SJson *json) {
tjsonAddStringToObject
(
item
,
"smaId"
,
i642str
(
pObj
->
smaId
));
tjsonAddStringToObject
(
item
,
"uid"
,
i642str
(
pObj
->
uid
));
tjsonAddStringToObject
(
item
,
"status"
,
i642str
(
pObj
->
status
));
tjsonAddStringToObject
(
item
,
"igExpired"
,
i642str
(
pObj
->
igExpired
));
tjsonAddStringToObject
(
item
,
"trigger"
,
i642str
(
pObj
->
trigger
));
tjsonAddStringToObject
(
item
,
"triggerParam"
,
i642str
(
pObj
->
triggerParam
));
tjsonAddStringToObject
(
item
,
"watermark"
,
i642str
(
pObj
->
watermark
));
tjsonAddStringToObject
(
item
,
"igExpired"
,
i642str
(
pObj
->
conf
.
igExpired
));
tjsonAddStringToObject
(
item
,
"trigger"
,
i642str
(
pObj
->
conf
.
trigger
));
tjsonAddStringToObject
(
item
,
"triggerParam"
,
i642str
(
pObj
->
conf
.
triggerParam
));
tjsonAddStringToObject
(
item
,
"watermark"
,
i642str
(
pObj
->
conf
.
watermark
));
tjsonAddStringToObject
(
item
,
"sourceDbUid"
,
i642str
(
pObj
->
sourceDbUid
));
tjsonAddStringToObject
(
item
,
"targetDbUid"
,
i642str
(
pObj
->
targetDbUid
));
tjsonAddStringToObject
(
item
,
"sourceDb"
,
mndGetDbStr
(
pObj
->
sourceDb
));
...
...
source/dnode/mnode/impl/src/mndScheduler.c
浏览文件 @
2977a4d0
...
...
@@ -22,9 +22,10 @@
#include "tname.h"
#include "tuuid.h"
#define SINK_NODE_LEVEL (0)
extern
bool
tsDeployOnSnode
;
static
int32_t
mndAddSinkTaskToStream
(
SStreamObj
*
pStream
,
SMnode
*
pMnode
,
int32_t
vgId
,
SVgObj
*
pVgroup
);
static
int32_t
mndAddSinkTaskToStream
(
SStreamObj
*
pStream
,
S
Array
*
pTaskList
,
S
Mnode
*
pMnode
,
int32_t
vgId
,
SVgObj
*
pVgroup
);
static
void
setFixedDownstreamEpInfo
(
SStreamTask
*
pDstTask
,
const
SStreamTask
*
pTask
);
int32_t
mndConvertRsmaTask
(
char
**
pDst
,
int32_t
*
pDstLen
,
const
char
*
ast
,
int64_t
uid
,
int8_t
triggerType
,
...
...
@@ -100,9 +101,7 @@ int32_t mndSetSinkTaskInfo(SStreamObj* pStream, SStreamTask* pTask) {
return
0
;
}
#define SINK_NODE_LEVEL (0)
int32_t
mndAddDispatcherForInnerTask
(
SMnode
*
pMnode
,
SStreamObj
*
pStream
,
SStreamTask
*
pTask
)
{
int32_t
mndAddDispatcherForInnerTask
(
SMnode
*
pMnode
,
SStreamObj
*
pStream
,
SArray
*
pSinkNodeList
,
SStreamTask
*
pTask
)
{
bool
isShuffle
=
false
;
if
(
pStream
->
fixedSinkVgId
==
0
)
{
...
...
@@ -120,7 +119,6 @@ int32_t mndAddDispatcherForInnerTask(SMnode* pMnode, SStreamObj* pStream, SStrea
sdbRelease
(
pMnode
->
pSdb
,
pDb
);
}
SArray
*
pSinkNodeList
=
taosArrayGetP
(
pStream
->
tasks
,
SINK_NODE_LEVEL
);
int32_t
numOfSinkNodes
=
taosArrayGetSize
(
pSinkNodeList
);
if
(
isShuffle
)
{
...
...
@@ -187,6 +185,7 @@ int32_t mndAssignTaskToSnode(SMnode* pMnode, SStreamTask* pTask, SSubplan* plan,
return
0
;
}
// todo random choose a node to do compute
SVgObj
*
mndSchedFetchOneVg
(
SMnode
*
pMnode
,
int64_t
dbUid
)
{
void
*
pIter
=
NULL
;
SVgObj
*
pVgroup
=
NULL
;
...
...
@@ -203,7 +202,7 @@ SVgObj* mndSchedFetchOneVg(SMnode* pMnode, int64_t dbUid) {
}
// create sink node for each vgroup.
int32_t
mndAddShuffleSinkTasksToStream
(
SMnode
*
pMnode
,
SStreamObj
*
pStream
)
{
int32_t
mndAddShuffleSinkTasksToStream
(
SMnode
*
pMnode
,
S
Array
*
pTaskList
,
S
StreamObj
*
pStream
)
{
SSdb
*
pSdb
=
pMnode
->
pSdb
;
void
*
pIter
=
NULL
;
...
...
@@ -219,17 +218,15 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) {
continue
;
}
mndAddSinkTaskToStream
(
pStream
,
pMnode
,
pVgroup
->
vgId
,
pVgroup
);
mndAddSinkTaskToStream
(
pStream
,
p
TaskList
,
p
Mnode
,
pVgroup
->
vgId
,
pVgroup
);
sdbRelease
(
pSdb
,
pVgroup
);
}
return
0
;
}
int32_t
mndAddSinkTaskToStream
(
SStreamObj
*
pStream
,
SMnode
*
pMnode
,
int32_t
vgId
,
SVgObj
*
pVgroup
)
{
SArray
*
pTaskList
=
taosArrayGetP
(
pStream
->
tasks
,
SINK_NODE_LEVEL
);
SStreamTask
*
pTask
=
tNewStreamTask
(
pStream
->
uid
,
TASK_LEVEL__SINK
,
pStream
->
fillHistory
,
0
,
pTaskList
);
int32_t
mndAddSinkTaskToStream
(
SStreamObj
*
pStream
,
SArray
*
pTaskList
,
SMnode
*
pMnode
,
int32_t
vgId
,
SVgObj
*
pVgroup
)
{
SStreamTask
*
pTask
=
tNewStreamTask
(
pStream
->
uid
,
TASK_LEVEL__SINK
,
pStream
->
conf
.
fillHistory
,
0
,
pTaskList
);
if
(
pTask
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
return
-
1
;
...
...
@@ -245,17 +242,18 @@ static int32_t mndScheduleFillHistoryStreamTask(SMnode* pMnode, SStreamObj* pStr
return
0
;
}
static
int32_t
addSourceStreamTask
(
SMnode
*
pMnode
,
SVgObj
*
pVgroup
,
SArray
*
pTaskList
,
SStreamObj
*
pStream
,
static
int32_t
addSourceStreamTask
(
SMnode
*
pMnode
,
SVgObj
*
pVgroup
,
SArray
*
pTaskList
,
S
Array
*
pSinkTaskList
,
S
StreamObj
*
pStream
,
SSubplan
*
plan
,
uint64_t
uid
,
int8_t
taskLevel
,
int8_t
fillHistory
,
bool
hasExtraSink
)
{
SStreamTask
*
pTask
=
tNewStreamTask
(
uid
,
taskLevel
,
fillHistory
,
pStream
->
triggerParam
,
pTaskList
);
SStreamTask
*
pTask
=
tNewStreamTask
(
uid
,
taskLevel
,
fillHistory
,
pStream
->
conf
.
triggerParam
,
pTaskList
);
if
(
pTask
==
NULL
)
{
return
terrno
;
}
// sink or dispatch
if
(
hasExtraSink
)
{
mndAddDispatcherForInnerTask
(
pMnode
,
pStream
,
pTask
);
mndAddDispatcherForInnerTask
(
pMnode
,
pStream
,
pSinkTaskList
,
pTask
);
}
else
{
mndSetSinkTaskInfo
(
pStream
,
pTask
);
}
...
...
@@ -292,31 +290,23 @@ void setFixedDownstreamEpInfo(SStreamTask* pDstTask, const SStreamTask* pTask) {
pDstTask
->
dispatchMsgType
=
TDMT_STREAM_TASK_DISPATCH
;
}
int32_t
appendTo
Upstream
(
SStreamTask
*
pTask
,
SStreamTask
*
pUp
stream
)
{
int32_t
appendTo
Downstream
(
SStreamTask
*
pTask
,
SStreamTask
*
pDown
stream
)
{
SStreamChildEpInfo
*
pEpInfo
=
createStreamTaskEpInfo
(
pTask
);
if
(
pEpInfo
==
NULL
)
{
return
TSDB_CODE_OUT_OF_MEMORY
;
}
if
(
p
Up
stream
->
childEpInfo
==
NULL
)
{
p
Up
stream
->
childEpInfo
=
taosArrayInit
(
4
,
POINTER_BYTES
);
if
(
p
Down
stream
->
childEpInfo
==
NULL
)
{
p
Down
stream
->
childEpInfo
=
taosArrayInit
(
4
,
POINTER_BYTES
);
}
taosArrayPush
(
p
Up
stream
->
childEpInfo
,
&
pEpInfo
);
taosArrayPush
(
p
Down
stream
->
childEpInfo
,
&
pEpInfo
);
return
TSDB_CODE_SUCCESS
;
}
int32_t
mndScheduleStream
(
SMnode
*
pMnode
,
SStreamObj
*
pStream
)
{
static
int32_t
doScheduleStream
(
uint64_t
uid
,
SArray
*
pTasksList
,
SStreamObj
*
pStream
,
SMnode
*
pMnode
,
SQueryPlan
*
pPlan
)
{
SSdb
*
pSdb
=
pMnode
->
pSdb
;
SQueryPlan
*
pPlan
=
qStringToQueryPlan
(
pStream
->
physicalPlan
);
if
(
pPlan
==
NULL
)
{
terrno
=
TSDB_CODE_QRY_INVALID_INPUT
;
return
-
1
;
}
int32_t
planTotLevel
=
LIST_LENGTH
(
pPlan
->
pSubplans
);
pStream
->
tasks
=
taosArrayInit
(
planTotLevel
,
POINTER_BYTES
);
bool
hasExtraSink
=
false
;
bool
externalTargetDB
=
strcmp
(
pStream
->
sourceDb
,
pStream
->
targetDb
)
!=
0
;
...
...
@@ -331,17 +321,17 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
if
(
planTotLevel
==
2
||
externalTargetDB
||
multiTarget
||
pStream
->
fixedSinkVgId
)
{
SArray
*
taskOneLevel
=
taosArrayInit
(
0
,
POINTER_BYTES
);
taosArrayPush
(
p
Stream
->
tasks
,
&
taskOneLevel
);
taosArrayPush
(
p
TasksList
,
&
taskOneLevel
);
// add extra sink
hasExtraSink
=
true
;
if
(
pStream
->
fixedSinkVgId
==
0
)
{
if
(
mndAddShuffleSinkTasksToStream
(
pMnode
,
pStream
)
<
0
)
{
if
(
mndAddShuffleSinkTasksToStream
(
pMnode
,
taskOneLevel
,
pStream
)
<
0
)
{
// TODO free
return
-
1
;
}
}
else
{
if
(
mndAddSinkTaskToStream
(
pStream
,
pMnode
,
pStream
->
fixedSinkVgId
,
&
pStream
->
fixedSinkVg
)
<
0
)
{
if
(
mndAddSinkTaskToStream
(
pStream
,
taskOneLevel
,
pMnode
,
pStream
->
fixedSinkVgId
,
&
pStream
->
fixedSinkVg
)
<
0
)
{
// TODO free
return
-
1
;
}
...
...
@@ -355,7 +345,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
// inner level
{
SArray
*
taskInnerLevel
=
taosArrayInit
(
0
,
POINTER_BYTES
);
taosArrayPush
(
p
Stream
->
tasks
,
&
taskInnerLevel
);
taosArrayPush
(
p
TasksList
,
&
taskInnerLevel
);
SNodeListNode
*
inner
=
(
SNodeListNode
*
)
nodesListGetNode
(
pPlan
->
pSubplans
,
0
);
SSubplan
*
plan
=
(
SSubplan
*
)
nodesListGetNode
(
inner
->
pNodeList
,
0
);
...
...
@@ -364,16 +354,15 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
return
-
1
;
}
pInnerTask
=
tNewStreamTask
(
pStream
->
uid
,
TASK_LEVEL__AGG
,
pStream
->
fillHistory
,
pStream
->
triggerParam
,
taskInnerLevel
);
pInnerTask
=
tNewStreamTask
(
uid
,
TASK_LEVEL__AGG
,
pStream
->
conf
.
fillHistory
,
pStream
->
conf
.
triggerParam
,
taskInnerLevel
);
if
(
pInnerTask
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
qDestroyQueryPlan
(
pPlan
);
return
-
1
;
}
// dispatch
if
(
mndAddDispatcherForInnerTask
(
pMnode
,
pStream
,
pInnerTask
)
<
0
)
{
qDestroyQueryPlan
(
pPlan
);
SArray
*
pSinkNodeList
=
taosArrayGet
(
pTasksList
,
SINK_NODE_LEVEL
);
if
(
mndAddDispatcherForInnerTask
(
pMnode
,
pStream
,
pSinkNodeList
,
pInnerTask
)
<
0
)
{
return
-
1
;
}
...
...
@@ -383,14 +372,12 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
SVgObj
*
pVgroup
=
mndSchedFetchOneVg
(
pMnode
,
pStream
->
sourceDbUid
);
if
(
mndAssignStreamTaskToVgroup
(
pMnode
,
pInnerTask
,
plan
,
pVgroup
)
<
0
)
{
sdbRelease
(
pSdb
,
pVgroup
);
qDestroyQueryPlan
(
pPlan
);
return
-
1
;
}
sdbRelease
(
pSdb
,
pVgroup
);
}
else
{
if
(
mndAssignTaskToSnode
(
pMnode
,
pInnerTask
,
plan
,
pSnode
)
<
0
)
{
sdbRelease
(
pSdb
,
pSnode
);
qDestroyQueryPlan
(
pPlan
);
return
-
1
;
}
}
...
...
@@ -398,7 +385,6 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
SVgObj
*
pVgroup
=
mndSchedFetchOneVg
(
pMnode
,
pStream
->
sourceDbUid
);
if
(
mndAssignStreamTaskToVgroup
(
pMnode
,
pInnerTask
,
plan
,
pVgroup
)
<
0
)
{
sdbRelease
(
pSdb
,
pVgroup
);
qDestroyQueryPlan
(
pPlan
);
return
-
1
;
}
...
...
@@ -408,7 +394,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
// source level
SArray
*
taskSourceLevel
=
taosArrayInit
(
0
,
POINTER_BYTES
);
taosArrayPush
(
p
Stream
->
tasks
,
&
taskSourceLevel
);
taosArrayPush
(
p
TasksList
,
&
taskSourceLevel
);
SNodeListNode
*
inner
=
(
SNodeListNode
*
)
nodesListGetNode
(
pPlan
->
pSubplans
,
1
);
SSubplan
*
plan
=
(
SSubplan
*
)
nodesListGetNode
(
inner
->
pNodeList
,
0
);
...
...
@@ -430,11 +416,10 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
continue
;
}
SStreamTask
*
pTask
=
tNewStreamTask
(
pStream
->
uid
,
TASK_LEVEL__SOURCE
,
pStream
->
fillHistory
,
0
,
taskSourceLevel
);
SStreamTask
*
pTask
=
tNewStreamTask
(
uid
,
TASK_LEVEL__SOURCE
,
pStream
->
conf
.
fillHistory
,
0
,
taskSourceLevel
);
if
(
pTask
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
sdbRelease
(
pSdb
,
pVgroup
);
qDestroyQueryPlan
(
pPlan
);
return
-
1
;
}
...
...
@@ -443,23 +428,21 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
if
(
mndAssignStreamTaskToVgroup
(
pMnode
,
pTask
,
plan
,
pVgroup
)
<
0
)
{
sdbRelease
(
pSdb
,
pVgroup
);
qDestroyQueryPlan
(
pPlan
);
return
-
1
;
}
int32_t
code
=
appendTo
Up
stream
(
pTask
,
pInnerTask
);
int32_t
code
=
appendTo
Down
stream
(
pTask
,
pInnerTask
);
sdbRelease
(
pSdb
,
pVgroup
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
terrno
=
code
;
qDestroyQueryPlan
(
pPlan
);
return
-
1
;
}
}
}
else
if
(
planTotLevel
==
1
)
{
// create exec stream task, since only one level, the exec task is also the source task
SArray
*
pTaskList
=
taosArrayInit
(
0
,
POINTER_BYTES
);
taosArrayPush
(
p
Stream
->
tasks
,
&
pTaskList
);
taosArrayPush
(
p
TasksList
,
&
pTaskList
);
SNodeListNode
*
inner
=
(
SNodeListNode
*
)
nodesListGetNode
(
pPlan
->
pSubplans
,
0
);
if
(
LIST_LENGTH
(
inner
->
pNodeList
)
!=
1
)
{
...
...
@@ -487,20 +470,36 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) {
}
// new stream task
int32_t
code
=
addSourceStreamTask
(
pMnode
,
pVgroup
,
pTaskList
,
pStream
,
plan
,
pStream
->
uid
,
TASK_LEVEL__SOURCE
,
pStream
->
fillHistory
,
hasExtraSink
);
SArray
*
pSinkNodeTaskList
=
taosArrayGet
(
pTasksList
,
SINK_NODE_LEVEL
);
int32_t
code
=
addSourceStreamTask
(
pMnode
,
pVgroup
,
pTaskList
,
pSinkNodeTaskList
,
pStream
,
plan
,
uid
,
TASK_LEVEL__SOURCE
,
pStream
->
conf
.
fillHistory
,
hasExtraSink
);
sdbRelease
(
pSdb
,
pVgroup
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
qDestroyQueryPlan
(
pPlan
);
return
-
1
;
}
}
}
qDestroyQueryPlan
(
pPlan
);
return
0
;
}
int32_t
mndScheduleStream
(
SMnode
*
pMnode
,
SStreamObj
*
pStream
)
{
SQueryPlan
*
pPlan
=
qStringToQueryPlan
(
pStream
->
physicalPlan
);
if
(
pPlan
==
NULL
)
{
terrno
=
TSDB_CODE_QRY_INVALID_INPUT
;
return
-
1
;
}
int32_t
code
=
doScheduleStream
(
pStream
->
uid
,
pStream
->
tasks
,
pStream
,
pMnode
,
pPlan
);
if
(
code
==
TSDB_CODE_SUCCESS
)
{
code
=
doScheduleStream
(
pStream
->
batchTaskUid
,
pStream
->
pBatchTask
,
pStream
,
pMnode
,
pPlan
);
}
qDestroyQueryPlan
(
pPlan
);
return
code
;
}
int32_t
mndSchedInitSubEp
(
SMnode
*
pMnode
,
const
SMqTopicObj
*
pTopic
,
SMqSubscribeObj
*
pSub
)
{
SSdb
*
pSdb
=
pMnode
->
pSdb
;
SVgObj
*
pVgroup
=
NULL
;
...
...
source/dnode/mnode/impl/src/mndSma.c
浏览文件 @
2977a4d0
...
...
@@ -555,20 +555,20 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
streamObj
.
version
=
1
;
streamObj
.
sql
=
taosStrdup
(
pCreate
->
sql
);
streamObj
.
smaId
=
smaObj
.
uid
;
streamObj
.
watermark
=
pCreate
->
watermark
;
streamObj
.
conf
.
watermark
=
pCreate
->
watermark
;
streamObj
.
deleteMark
=
pCreate
->
deleteMark
;
streamObj
.
fillHistory
=
STREAM_FILL_HISTORY_ON
;
streamObj
.
trigger
=
STREAM_TRIGGER_WINDOW_CLOSE
;
streamObj
.
triggerParam
=
pCreate
->
maxDelay
;
streamObj
.
conf
.
fillHistory
=
STREAM_FILL_HISTORY_ON
;
streamObj
.
conf
.
trigger
=
STREAM_TRIGGER_WINDOW_CLOSE
;
streamObj
.
conf
.
triggerParam
=
pCreate
->
maxDelay
;
streamObj
.
ast
=
taosStrdup
(
smaObj
.
ast
);
// check the maxDelay
if
(
streamObj
.
triggerParam
<
TSDB_MIN_ROLLUP_MAX_DELAY
)
{
if
(
streamObj
.
conf
.
triggerParam
<
TSDB_MIN_ROLLUP_MAX_DELAY
)
{
int64_t
msInterval
=
convertTimeFromPrecisionToUnit
(
pCreate
->
interval
,
pDb
->
cfg
.
precision
,
TIME_UNIT_MILLISECOND
);
streamObj
.
triggerParam
=
msInterval
>
TSDB_MIN_ROLLUP_MAX_DELAY
?
msInterval
:
TSDB_MIN_ROLLUP_MAX_DELAY
;
streamObj
.
conf
.
triggerParam
=
msInterval
>
TSDB_MIN_ROLLUP_MAX_DELAY
?
msInterval
:
TSDB_MIN_ROLLUP_MAX_DELAY
;
}
if
(
streamObj
.
triggerParam
>
TSDB_MAX_ROLLUP_MAX_DELAY
)
{
streamObj
.
triggerParam
=
TSDB_MAX_ROLLUP_MAX_DELAY
;
if
(
streamObj
.
conf
.
triggerParam
>
TSDB_MAX_ROLLUP_MAX_DELAY
)
{
streamObj
.
conf
.
triggerParam
=
TSDB_MAX_ROLLUP_MAX_DELAY
;
}
if
(
mndAllocSmaVgroup
(
pMnode
,
pDb
,
&
streamObj
.
fixedSinkVg
)
!=
0
)
{
...
...
@@ -597,8 +597,8 @@ static int32_t mndCreateSma(SMnode *pMnode, SRpcMsg *pReq, SMCreateSmaReq *pCrea
.
pAstRoot
=
pAst
,
.
topicQuery
=
false
,
.
streamQuery
=
true
,
.
triggerType
=
streamObj
.
trigger
,
.
watermark
=
streamObj
.
watermark
,
.
triggerType
=
streamObj
.
conf
.
trigger
,
.
watermark
=
streamObj
.
conf
.
watermark
,
.
deleteMark
=
streamObj
.
deleteMark
,
};
...
...
source/dnode/mnode/impl/src/mndStream.c
浏览文件 @
2977a4d0
...
...
@@ -239,7 +239,7 @@ static void mndShowStreamStatus(char *dst, SStreamObj *pStream) {
}
static
void
mndShowStreamTrigger
(
char
*
dst
,
SStreamObj
*
pStream
)
{
int8_t
trigger
=
pStream
->
trigger
;
int8_t
trigger
=
pStream
->
conf
.
trigger
;
if
(
trigger
==
STREAM_TRIGGER_AT_ONCE
)
{
strcpy
(
dst
,
"at once"
);
}
else
if
(
trigger
==
STREAM_TRIGGER_WINDOW_CLOSE
)
{
...
...
@@ -301,11 +301,11 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
pObj
->
uid
=
mndGenerateUid
(
pObj
->
name
,
strlen
(
pObj
->
name
));
pObj
->
status
=
0
;
pObj
->
igExpired
=
pCreate
->
igExpired
;
pObj
->
trigger
=
pCreate
->
triggerType
;
pObj
->
triggerParam
=
pCreate
->
maxDelay
;
pObj
->
watermark
=
pCreate
->
watermark
;
pObj
->
fillHistory
=
pCreate
->
fillHistory
;
pObj
->
conf
.
igExpired
=
pCreate
->
igExpired
;
pObj
->
conf
.
trigger
=
pCreate
->
triggerType
;
pObj
->
conf
.
triggerParam
=
pCreate
->
maxDelay
;
pObj
->
conf
.
watermark
=
pCreate
->
watermark
;
pObj
->
conf
.
fillHistory
=
pCreate
->
fillHistory
;
pObj
->
deleteMark
=
pCreate
->
deleteMark
;
pObj
->
igCheckUpdate
=
pCreate
->
igUpdate
;
...
...
@@ -387,9 +387,9 @@ static int32_t mndBuildStreamObjFromCreateReq(SMnode *pMnode, SStreamObj *pObj,
.
pAstRoot
=
pAst
,
.
topicQuery
=
false
,
.
streamQuery
=
true
,
.
triggerType
=
pObj
->
trigger
==
STREAM_TRIGGER_MAX_DELAY
?
STREAM_TRIGGER_WINDOW_CLOSE
:
pObj
->
trigger
,
.
watermark
=
pObj
->
watermark
,
.
igExpired
=
pObj
->
igExpired
,
.
triggerType
=
pObj
->
conf
.
trigger
==
STREAM_TRIGGER_MAX_DELAY
?
STREAM_TRIGGER_WINDOW_CLOSE
:
pObj
->
conf
.
trigger
,
.
watermark
=
pObj
->
conf
.
watermark
,
.
igExpired
=
pObj
->
conf
.
igExpired
,
.
deleteMark
=
pObj
->
deleteMark
,
.
igCheckUpdate
=
pObj
->
igCheckUpdate
,
};
...
...
@@ -459,8 +459,9 @@ int32_t mndPersistStreamTasks(SMnode *pMnode, STrans *pTrans, SStreamObj *pStrea
int32_t
level
=
taosArrayGetSize
(
pStream
->
tasks
);
for
(
int32_t
i
=
0
;
i
<
level
;
i
++
)
{
SArray
*
pLevel
=
taosArrayGetP
(
pStream
->
tasks
,
i
);
int32_t
sz
=
taosArrayGetSize
(
pLevel
);
for
(
int32_t
j
=
0
;
j
<
sz
;
j
++
)
{
int32_t
numOfTasks
=
taosArrayGetSize
(
pLevel
);
for
(
int32_t
j
=
0
;
j
<
numOfTasks
;
j
++
)
{
SStreamTask
*
pTask
=
taosArrayGetP
(
pLevel
,
j
);
if
(
mndPersistTaskDeployReq
(
pTrans
,
pTask
)
<
0
)
{
return
-
1
;
...
...
@@ -1157,7 +1158,7 @@ static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB
}
pColInfo
=
taosArrayGet
(
pBlock
->
pDataBlock
,
cols
++
);
colDataSetVal
(
pColInfo
,
numOfRows
,
(
const
char
*
)
&
pStream
->
watermark
,
false
);
colDataSetVal
(
pColInfo
,
numOfRows
,
(
const
char
*
)
&
pStream
->
conf
.
watermark
,
false
);
char
trigger
[
20
+
VARSTR_HEADER_SIZE
]
=
{
0
};
char
trigger2
[
20
]
=
{
0
};
...
...
source/libs/stream/src/streamTask.c
浏览文件 @
2977a4d0
...
...
@@ -49,6 +49,106 @@ SStreamTask* tNewStreamTask(int64_t streamId, int8_t taskLevel, int8_t fillHisto
return
pTask
;
}
SStreamTask
*
streamTaskClone
(
SStreamTask
*
pTask
)
{
SStreamTask
*
pDst
=
taosMemoryCalloc
(
1
,
sizeof
(
SStreamTask
));
/* pDst->
SStreamId id;
int32_t totalLevel;
int8_t taskLevel;
int8_t outputType;
int16_t dispatchMsgType;
SStreamStatus status;
int32_t selfChildId;
int32_t nodeId; // vgroup id
SEpSet epSet;
SCheckpointInfo chkInfo;
STaskExec exec;
int8_t fillHistory; // fill history
int64_t ekey; // end ts key
int64_t endVer; // end version
// children info
SArray* childEpInfo; // SArray<SStreamChildEpInfo*>
int32_t nextCheckId;
SArray* checkpointInfo; // SArray<SStreamCheckpointInfo>
// output
union {
STaskDispatcherFixedEp fixedEpDispatcher;
STaskDispatcherShuffle shuffleDispatcher;
STaskSinkTb tbSink;
STaskSinkSma smaSink;
STaskSinkFetch fetchSink;
};
int8_t inputStatus;
int8_t outputStatus;
SStreamQueue* inputQueue;
SStreamQueue* outputQueue;
// trigger
int8_t triggerStatus;
int64_t triggerParam;
void* timer;
SMsgCb* pMsgCb; // msg handle
SStreamState* pState; // state backend
// the followings attributes don't be serialized
int32_t recoverTryingDownstream;
int32_t recoverWaitingUpstream;
int64_t checkReqId;
SArray* checkReqIds; // shuffle
int32_t refCnt;
int64_t checkpointingId;
int32_t checkpointAlignCnt;
struct SStreamMeta* pMeta;
int32_t status = atomic_load_8((int8_t*)&(pTask->status.taskStatus));
if (pTask->inputQueue) {
streamQueueClose(pTask->inputQueue);
}
if (pTask->outputQueue) {
streamQueueClose(pTask->outputQueue);
}
if (pTask->exec.qmsg) {
taosMemoryFree(pTask->exec.qmsg);
}
if (pTask->exec.pExecutor) {
qDestroyTask(pTask->exec.pExecutor);
pTask->exec.pExecutor = NULL;
}
if (pTask->exec.pWalReader != NULL) {
walCloseReader(pTask->exec.pWalReader);
}
taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree);
if (pTask->outputType == TASK_OUTPUT__TABLE) {
tDeleteSchemaWrapper(pTask->tbSink.pSchemaWrapper);
taosMemoryFree(pTask->tbSink.pTSchema);
tSimpleHashCleanup(pTask->tbSink.pTblInfo);
}
if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) {
taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos);
taosArrayDestroy(pTask->checkReqIds);
pTask->checkReqIds = NULL;
}
if (pTask->pState) {
streamStateClose(pTask->pState, status == TASK_STATUS__DROPPING);
}
if (pTask->id.idStr != NULL) {
taosMemoryFree((void*)pTask->id.idStr);
}
taosMemoryFree(pTask);*/
return
NULL
;
}
int32_t
tEncodeStreamEpInfo
(
SEncoder
*
pEncoder
,
const
SStreamChildEpInfo
*
pInfo
)
{
if
(
tEncodeI32
(
pEncoder
,
pInfo
->
taskId
)
<
0
)
return
-
1
;
if
(
tEncodeI32
(
pEncoder
,
pInfo
->
nodeId
)
<
0
)
return
-
1
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录