Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
b3fea32a
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1184
Star
22015
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
b3fea32a
编写于
5月 16, 2022
作者:
L
Liu Jicong
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor(stream)
上级
f77f9102
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
79 addition
and
166 deletion
+79
-166
include/common/tmsg.h
include/common/tmsg.h
+6
-2
source/dnode/vnode/src/tq/tq.c
source/dnode/vnode/src/tq/tq.c
+1
-30
source/libs/stream/src/tstream.c
source/libs/stream/src/tstream.c
+72
-134
未找到文件。
include/common/tmsg.h
浏览文件 @
b3fea32a
...
...
@@ -2494,11 +2494,15 @@ static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) {
}
typedef
struct
{
void
*
data
;
int64_t
streamId
;
int32_t
taskId
;
int32_t
sourceVg
;
int64_t
sourceVer
;
SArray
*
data
;
// SArray<SSDataBlock>
}
SStreamDispatchReq
;
typedef
struct
{
int8_t
s
tatus
;
int8_t
inputS
tatus
;
}
SStreamDispatchRsp
;
#define TD_AUTO_CREATE_TABLE 0x1
...
...
source/dnode/vnode/src/tq/tq.c
浏览文件 @
b3fea32a
...
...
@@ -1087,35 +1087,6 @@ int32_t tqProcessStreamTrigger2(STQ* pTq, SSubmitReq* pReq, int64_t ver) {
}
int32_t
tqProcessTaskExec2
(
STQ
*
pTq
,
char
*
msg
,
int32_t
msgLen
)
{
SStreamTaskExecReq
req
=
{
0
};
tDecodeSStreamTaskExecReq
(
msg
,
&
req
);
int32_t
taskId
=
req
.
taskId
;
SStreamTask
*
pTask
=
taosHashGet
(
pTq
->
pStreamTasks
,
&
taskId
,
sizeof
(
int32_t
));
ASSERT
(
pTask
);
ASSERT
(
pTask
->
inputType
==
TASK_INPUT_TYPE__DATA_BLOCK
);
// enqueue
int32_t
inputStatus
=
streamEnqueueDataBlk
(
pTask
,
(
SStreamDataBlock
*
)
req
.
data
);
if
(
inputStatus
==
TASK_INPUT_STATUS__BLOCKED
)
{
// TODO rsp blocked
return
0
;
}
// try exec
int8_t
execStatus
=
atomic_val_compare_exchange_8
(
&
pTask
->
status
,
TASK_STATUS__IDLE
,
TASK_STATUS__EXECUTING
);
if
(
execStatus
==
TASK_STATUS__IDLE
)
{
if
(
streamTaskRun
(
pTask
)
<
0
)
{
atomic_store_8
(
&
pTask
->
status
,
TASK_STATUS__CLOSING
);
goto
FAIL
;
}
}
else
if
(
execStatus
==
TASK_STATUS__EXECUTING
)
{
return
0
;
}
// TODO rsp success
//
return
0
;
FAIL:
return
-
1
;
}
source/libs/stream/src/tstream.c
浏览文件 @
b3fea32a
...
...
@@ -35,7 +35,7 @@ void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) {
return
(
void
*
)
buf
;
}
static
int32_t
streamBuild
Dispatch
Msg
(
SStreamTask
*
pTask
,
SArray
*
data
,
SRpcMsg
*
pMsg
,
SEpSet
**
ppEpSet
)
{
static
int32_t
streamBuild
Exec
Msg
(
SStreamTask
*
pTask
,
SArray
*
data
,
SRpcMsg
*
pMsg
,
SEpSet
**
ppEpSet
)
{
SStreamTaskExecReq
req
=
{
.
streamId
=
pTask
->
streamId
,
.
data
=
data
,
...
...
@@ -107,7 +107,7 @@ static int32_t streamShuffleDispatch(SStreamTask* pTask, SMsgCb* pMsgCb, SHashOb
SArray
*
pData
=
*
(
SArray
**
)
pIter
;
SRpcMsg
dispatchMsg
=
{
0
};
SEpSet
*
pEpSet
;
if
(
streamBuild
Dispatch
Msg
(
pTask
,
pData
,
&
dispatchMsg
,
&
pEpSet
)
<
0
)
{
if
(
streamBuild
Exec
Msg
(
pTask
,
pData
,
&
dispatchMsg
,
&
pEpSet
)
<
0
)
{
ASSERT
(
0
);
return
-
1
;
}
...
...
@@ -133,7 +133,7 @@ int32_t streamEnqueueDataBlk(SStreamTask* pTask, SStreamDataBlock* input) {
return
inputStatus
;
}
int32_t
streamTaskExecImpl
(
SStreamTask
*
pTask
,
void
*
data
,
SArray
*
pRes
)
{
static
int32_t
streamTaskExecImpl
(
SStreamTask
*
pTask
,
void
*
data
,
SArray
*
pRes
)
{
void
*
exec
=
pTask
->
exec
.
runners
[
0
].
executor
;
// set input
...
...
@@ -265,87 +265,42 @@ FAIL:
return
-
1
;
}
int32_t
streamTaskDispatchDown
(
SStreamTask
*
pTask
,
SMsgCb
*
pMsgCb
)
{
//
return
0
;
}
int32_t
streamTaskSink
(
SStreamTask
*
pTask
)
{
//
return
0
;
}
int32_t
streamTaskProcessInputReq
(
SStreamTask
*
pTask
,
SMsgCb
*
pMsgCb
,
SStreamDataBlock
*
pBlock
,
SRpcMsg
*
pRsp
)
{
// 1. handle input
// 1.1 enqueue
taosWriteQitem
(
pTask
->
inputQ
,
pBlock
);
// 1.2 calc back pressure
// 1.3 rsp by input status
int8_t
inputStatus
=
atomic_load_8
(
&
pTask
->
inputStatus
);
SStreamDispatchRsp
*
pCont
=
rpcMallocCont
(
sizeof
(
SStreamDispatchRsp
));
pCont
->
status
=
inputStatus
;
pRsp
->
pCont
=
pCont
;
pRsp
->
contLen
=
sizeof
(
SStreamDispatchRsp
);
tmsgSendRsp
(
pRsp
);
// 2. try exec
// 2.1. idle: exec
// 2.2. executing: return
// 2.3. closing: keep trying
int32_t
streamTaskSink
(
SStreamTask
*
pTask
,
SMsgCb
*
pMsgCb
)
{
bool
firstRun
=
1
;
while
(
1
)
{
int8_t
execStatus
=
atomic_val_compare_exchange_8
(
&
pTask
->
status
,
TASK_STATUS__IDLE
,
TASK_STATUS__EXECUTING
);
if
(
execStatus
==
TASK_STATUS__IDLE
)
{
void
*
exec
=
pTask
->
exec
.
runners
[
0
].
executor
;
SArray
*
pRes
=
taosArrayInit
(
0
,
sizeof
(
void
*
));
const
SArray
*
blocks
=
pBlock
->
blocks
;
qSetMultiStreamInput
(
exec
,
blocks
->
pData
,
blocks
->
size
,
STREAM_DATA_TYPE_SSDATA_BLOCK
);
while
(
1
)
{
SSDataBlock
*
output
;
uint64_t
ts
=
0
;
if
(
qExecTask
(
exec
,
&
output
,
&
ts
)
<
0
)
{
ASSERT
(
false
);
}
if
(
output
==
NULL
)
break
;
taosArrayPush
(
pRes
,
&
output
);
}
// TODO: wrap destroy block
taosArrayDestroyP
(
pBlock
->
blocks
,
(
FDelete
)
blockDataDestroy
);
if
(
taosArrayGetSize
(
pRes
)
!=
0
)
{
SArray
**
resQ
=
taosAllocateQitem
(
sizeof
(
void
**
),
DEF_QITEM
);
*
resQ
=
pRes
;
taosWriteQitem
(
pTask
->
outputQ
,
resQ
);
SStreamDataBlock
*
pBlock
=
NULL
;
if
(
!
firstRun
)
{
taosReadAllQitems
(
pTask
->
outputQ
,
pTask
->
outputQAll
);
}
taosGetQitem
(
pTask
->
outputQAll
,
(
void
**
)
&
pBlock
);
if
(
pBlock
==
NULL
)
{
if
(
firstRun
)
{
firstRun
=
0
;
continue
;
}
else
{
break
;
}
}
else
if
(
execStatus
==
TASK_STATUS__CLOSING
)
{
continue
;
}
else
if
(
execStatus
==
TASK_STATUS__EXECUTING
)
break
;
else
{
ASSERT
(
0
);
}
}
// 3. handle output
// 3.1 check and set status
// 3.2 dispatch / sink
STaosQall
*
qall
=
taosAllocateQall
();
taosReadAllQitems
(
pTask
->
outputQ
,
qall
);
SArray
**
ppRes
=
NULL
;
while
(
1
)
{
taosGetQitem
(
qall
,
(
void
**
)
&
ppRes
);
if
(
ppRes
==
NULL
)
break
;
SArray
*
pRes
=
*
ppRes
;
SArray
*
pRes
=
pBlock
->
blocks
;
// sink
if
(
pTask
->
sinkType
==
TASK_SINK__TABLE
)
{
pTask
->
tbSink
.
tbSinkFunc
(
pTask
,
pTask
->
tbSink
.
vnode
,
pBlock
->
sourceVer
,
pRes
);
// blockDebugShowData(pRes);
pTask
->
tbSink
.
tbSinkFunc
(
pTask
,
pTask
->
tbSink
.
vnode
,
0
,
pRes
);
}
else
if
(
pTask
->
sinkType
==
TASK_SINK__SMA
)
{
pTask
->
smaSink
.
smaSink
(
pTask
->
ahandle
,
pTask
->
smaSink
.
smaId
,
pRes
);
//
}
else
if
(
pTask
->
sinkType
==
TASK_SINK__FETCH
)
{
//
}
else
{
ASSERT
(
pTask
->
sinkType
==
TASK_SINK__NONE
);
}
// dispatch
if
(
pTask
->
dispatchType
==
TASK_DISPATCH__INPLACE
)
{
SRpcMsg
dispatchMsg
=
{
0
};
if
(
streamBuild
Dispatch
Msg
(
pTask
,
pRes
,
&
dispatchMsg
,
NULL
)
<
0
)
{
if
(
streamBuild
Exec
Msg
(
pTask
,
pRes
,
&
dispatchMsg
,
NULL
)
<
0
)
{
ASSERT
(
0
);
return
-
1
;
}
...
...
@@ -366,7 +321,7 @@ int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDat
}
else
if
(
pTask
->
dispatchType
==
TASK_DISPATCH__FIXED
)
{
SRpcMsg
dispatchMsg
=
{
0
};
SEpSet
*
pEpSet
=
NULL
;
if
(
streamBuild
Dispatch
Msg
(
pTask
,
pRes
,
&
dispatchMsg
,
&
pEpSet
)
<
0
)
{
if
(
streamBuild
Exec
Msg
(
pTask
,
pRes
,
&
dispatchMsg
,
&
pEpSet
)
<
0
)
{
ASSERT
(
0
);
return
-
1
;
}
...
...
@@ -401,75 +356,58 @@ int32_t streamTaskProcessInputReq(SStreamTask* pTask, SMsgCb* pMsgCb, SStreamDat
ASSERT
(
pTask
->
dispatchType
==
TASK_DISPATCH__NONE
);
}
}
//
return
0
;
}
int32_t
streamTaskProcessDispatchRsp
(
SStreamTask
*
pTask
,
char
*
msg
,
int32_t
msgLen
)
{
//
return
0
;
}
int32_t
streamTaskEnqueue
(
SStreamTask
*
pTask
,
SStreamDispatchReq
*
pReq
,
SRpcMsg
*
pRsp
)
{
SStreamDataBlock
*
pBlock
=
taosAllocateQitem
(
sizeof
(
SStreamDataBlock
),
DEF_QITEM
);
int8_t
status
;
// 1.1 update status
// TODO cal backpressure
if
(
pBlock
==
NULL
)
{
atomic_store_8
(
&
pTask
->
inputStatus
,
TASK_INPUT_STATUS__FAILED
);
status
=
TASK_INPUT_STATUS__FAILED
;
}
else
{
status
=
atomic_load_8
(
&
pTask
->
inputStatus
);
}
// 1.2 enqueue
pBlock
->
type
=
STREAM_DATA_TYPE_SSDATA_BLOCK
;
pBlock
->
sourceVg
=
pReq
->
sourceVg
;
pBlock
->
sourceVer
=
pReq
->
sourceVer
;
taosWriteQitem
(
pTask
->
inputQ
,
pBlock
);
// 1.3 rsp by input status
SStreamDispatchRsp
*
pCont
=
rpcMallocCont
(
sizeof
(
SStreamDispatchRsp
));
pCont
->
inputStatus
=
status
;
pRsp
->
pCont
=
pCont
;
pRsp
->
contLen
=
sizeof
(
SStreamDispatchRsp
);
tmsgSendRsp
(
pRsp
);
int32_t
streamTaskProcessRecoverReq
(
SStreamTask
*
pTask
,
char
*
msg
)
{
//
return
0
;
}
int32_t
streamTaskRun
(
SStreamTask
*
pTask
)
{
SArray
*
pRes
=
NULL
;
if
(
pTask
->
execType
==
TASK_EXEC__PIPE
||
pTask
->
execType
==
TASK_EXEC__MERGE
)
{
// TODO remove multi runner
void
*
exec
=
pTask
->
exec
.
runners
[
0
].
executor
;
int8_t
status
=
atomic_val_compare_exchange_8
(
&
pTask
->
status
,
TASK_STATUS__IDLE
,
TASK_STATUS__EXECUTING
);
if
(
status
==
TASK_STATUS__IDLE
)
{
pRes
=
taosArrayInit
(
0
,
sizeof
(
void
*
));
if
(
pRes
==
NULL
)
{
return
-
1
;
}
int32_t
streamTaskProcessDispatchReq
(
SStreamTask
*
pTask
,
SMsgCb
*
pMsgCb
,
SStreamDispatchReq
*
pReq
,
SRpcMsg
*
pRsp
)
{
// 1. handle input
streamTaskEnqueue
(
pTask
,
pReq
,
pRsp
);
void
*
input
=
NULL
;
taosWriteQitem
(
pTask
->
inputQ
,
&
input
);
if
(
input
==
NULL
)
return
0
;
// TODO: fix type
if
(
pTask
->
sourceType
==
TASK_SOURCE__SCAN
)
{
SStreamDataSubmit
*
pSubmit
=
(
SStreamDataSubmit
*
)
input
;
qSetStreamInput
(
exec
,
pSubmit
->
data
,
STREAM_DATA_TYPE_SUBMIT_BLOCK
);
while
(
1
)
{
SSDataBlock
*
output
;
uint64_t
ts
=
0
;
if
(
qExecTask
(
exec
,
&
output
,
&
ts
)
<
0
)
{
ASSERT
(
false
);
}
if
(
output
==
NULL
)
break
;
taosArrayPush
(
pRes
,
&
output
);
}
streamDataSubmitRefDec
(
pSubmit
);
}
else
{
SStreamDataBlock
*
pStreamBlock
=
(
SStreamDataBlock
*
)
input
;
const
SArray
*
blocks
=
pStreamBlock
->
blocks
;
qSetMultiStreamInput
(
exec
,
blocks
->
pData
,
blocks
->
size
,
STREAM_DATA_TYPE_SSDATA_BLOCK
);
while
(
1
)
{
SSDataBlock
*
output
;
uint64_t
ts
=
0
;
if
(
qExecTask
(
exec
,
&
output
,
&
ts
)
<
0
)
{
ASSERT
(
false
);
}
if
(
output
==
NULL
)
break
;
taosArrayPush
(
pRes
,
&
output
);
}
// TODO: wrap destroy block
taosArrayDestroyP
(
pStreamBlock
->
blocks
,
(
FDelete
)
blockDataDestroy
);
}
// 2. try exec
// 2.1. idle: exec
// 2.2. executing: return
// 2.3. closing: keep trying
streamTaskExec2
(
pTask
,
pMsgCb
);
if
(
taosArrayGetSize
(
pRes
)
!=
0
)
{
SArray
**
resQ
=
taosAllocateQitem
(
sizeof
(
void
**
),
DEF_QITEM
);
*
resQ
=
pRes
;
taosWriteQitem
(
pTask
->
outputQ
,
resQ
);
}
}
}
// 3. handle output
// 3.1 check and set status
// 3.2 dispatch / sink
streamTaskSink
(
pTask
,
pMsgCb
);
return
0
;
}
int32_t
streamTaskProcessRecoverReq
(
SStreamTask
*
pTask
,
char
*
msg
)
{
//
return
0
;
}
...
...
@@ -545,7 +483,7 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
if
(
pTask
->
dispatchType
==
TASK_DISPATCH__INPLACE
)
{
SRpcMsg
dispatchMsg
=
{
0
};
if
(
streamBuild
Dispatch
Msg
(
pTask
,
pRes
,
&
dispatchMsg
,
NULL
)
<
0
)
{
if
(
streamBuild
Exec
Msg
(
pTask
,
pRes
,
&
dispatchMsg
,
NULL
)
<
0
)
{
ASSERT
(
0
);
return
-
1
;
}
...
...
@@ -566,7 +504,7 @@ int32_t streamExecTask(SStreamTask* pTask, SMsgCb* pMsgCb, const void* input, in
}
else
if
(
pTask
->
dispatchType
==
TASK_DISPATCH__FIXED
)
{
SRpcMsg
dispatchMsg
=
{
0
};
SEpSet
*
pEpSet
=
NULL
;
if
(
streamBuild
Dispatch
Msg
(
pTask
,
pRes
,
&
dispatchMsg
,
&
pEpSet
)
<
0
)
{
if
(
streamBuild
Exec
Msg
(
pTask
,
pRes
,
&
dispatchMsg
,
&
pEpSet
)
<
0
)
{
ASSERT
(
0
);
return
-
1
;
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录