Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
3ef7d43d
T
TDengine
项目概览
taosdata
/
TDengine
接近 2 年 前同步成功
通知
1192
Star
22018
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
3ef7d43d
编写于
4月 03, 2023
作者:
H
Haojun Liao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor: do some internal refactor.
上级
08bd021a
变更
13
隐藏空白更改
内联
并排
Showing
13 changed file
with
115 addition
and
127 deletion
+115
-127
include/libs/stream/tstream.h
include/libs/stream/tstream.h
+1
-37
source/client/src/clientTmq.c
source/client/src/clientTmq.c
+3
-11
source/dnode/vnode/src/tq/tq.c
source/dnode/vnode/src/tq/tq.c
+8
-12
source/dnode/vnode/src/tq/tqPush.c
source/dnode/vnode/src/tq/tqPush.c
+26
-25
source/dnode/vnode/src/vnd/vnodeSvr.c
source/dnode/vnode/src/vnd/vnodeSvr.c
+1
-0
source/libs/executor/inc/executorimpl.h
source/libs/executor/inc/executorimpl.h
+0
-20
source/libs/executor/src/aggregateoperator.c
source/libs/executor/src/aggregateoperator.c
+10
-0
source/libs/executor/src/executor.c
source/libs/executor/src/executor.c
+2
-4
source/libs/executor/src/scanoperator.c
source/libs/executor/src/scanoperator.c
+10
-1
source/libs/executor/src/timewindowoperator.c
source/libs/executor/src/timewindowoperator.c
+1
-0
source/libs/stream/src/stream.c
source/libs/stream/src/stream.c
+40
-9
source/libs/stream/src/streamData.c
source/libs/stream/src/streamData.c
+9
-5
source/libs/stream/src/streamExec.c
source/libs/stream/src/streamExec.c
+4
-3
未找到文件。
include/libs/stream/tstream.h
浏览文件 @
3ef7d43d
...
...
@@ -371,43 +371,7 @@ SStreamTask* tNewSStreamTask(int64_t streamId);
int32_t
tEncodeSStreamTask
(
SEncoder
*
pEncoder
,
const
SStreamTask
*
pTask
);
int32_t
tDecodeSStreamTask
(
SDecoder
*
pDecoder
,
SStreamTask
*
pTask
);
void
tFreeSStreamTask
(
SStreamTask
*
pTask
);
static
FORCE_INLINE
int32_t
streamTaskInput
(
SStreamTask
*
pTask
,
SStreamQueueItem
*
pItem
)
{
int8_t
type
=
pItem
->
type
;
if
(
type
==
STREAM_INPUT__DATA_SUBMIT
)
{
SStreamDataSubmit2
*
pSubmitClone
=
streamSubmitRefClone
((
SStreamDataSubmit2
*
)
pItem
);
if
(
pSubmitClone
==
NULL
)
{
qDebug
(
"task %d %p submit enqueue failed since out of memory"
,
pTask
->
taskId
,
pTask
);
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
atomic_store_8
(
&
pTask
->
inputStatus
,
TASK_INPUT_STATUS__FAILED
);
return
-
1
;
}
qDebug
(
"task %d %p submit enqueue %p %p %p %d %"
PRId64
,
pTask
->
taskId
,
pTask
,
pItem
,
pSubmitClone
,
pSubmitClone
->
submit
.
msgStr
,
pSubmitClone
->
submit
.
msgLen
,
pSubmitClone
->
submit
.
ver
);
taosWriteQitem
(
pTask
->
inputQueue
->
queue
,
pSubmitClone
);
// qStreamInput(pTask->exec.executor, pSubmitClone);
}
else
if
(
type
==
STREAM_INPUT__DATA_BLOCK
||
type
==
STREAM_INPUT__DATA_RETRIEVE
||
type
==
STREAM_INPUT__REF_DATA_BLOCK
)
{
taosWriteQitem
(
pTask
->
inputQueue
->
queue
,
pItem
);
// qStreamInput(pTask->exec.executor, pItem);
}
else
if
(
type
==
STREAM_INPUT__CHECKPOINT
)
{
taosWriteQitem
(
pTask
->
inputQueue
->
queue
,
pItem
);
// qStreamInput(pTask->exec.executor, pItem);
}
else
if
(
type
==
STREAM_INPUT__GET_RES
)
{
taosWriteQitem
(
pTask
->
inputQueue
->
queue
,
pItem
);
// qStreamInput(pTask->exec.executor, pItem);
}
if
(
type
!=
STREAM_INPUT__GET_RES
&&
type
!=
STREAM_INPUT__CHECKPOINT
&&
pTask
->
triggerParam
!=
0
)
{
atomic_val_compare_exchange_8
(
&
pTask
->
triggerStatus
,
TASK_TRIGGER_STATUS__INACTIVE
,
TASK_TRIGGER_STATUS__ACTIVE
);
}
#if 0
// TODO: back pressure
atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL);
#endif
return
0
;
}
int32_t
streamTaskInput
(
SStreamTask
*
pTask
,
SStreamQueueItem
*
pItem
);
static
FORCE_INLINE
void
streamTaskInputFail
(
SStreamTask
*
pTask
)
{
atomic_store_8
(
&
pTask
->
inputStatus
,
TASK_INPUT_STATUS__FAILED
);
...
...
source/client/src/clientTmq.c
浏览文件 @
3ef7d43d
...
...
@@ -107,7 +107,6 @@ struct tmq_t {
STaosQueue
*
mqueue
;
// queue of rsp
STaosQall
*
qall
;
STaosQueue
*
delayedTask
;
// delayed task queue for heartbeat and auto commit
TdThreadMutex
lock
;
// used to protect the operation on each topic, when updating the epsets.
tsem_t
rspSem
;
};
...
...
@@ -188,7 +187,6 @@ typedef struct {
SMqClientVg
*
pVg
;
SMqClientTopic
*
pTopic
;
int32_t
vgId
;
tsem_t
rspSem
;
uint64_t
requestId
;
// request id for debug purpose
}
SMqPollCbParam
;
...
...
@@ -979,7 +977,6 @@ void tmqFreeImpl(void* handle) {
taosFreeQall
(
tmq
->
qall
);
tsem_destroy
(
&
tmq
->
rspSem
);
taosThreadMutexDestroy
(
&
tmq
->
lock
);
taosArrayDestroyEx
(
tmq
->
clientTopics
,
freeClientVgImpl
);
taos_close_internal
(
tmq
->
pTscObj
);
...
...
@@ -1024,7 +1021,6 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) {
pTmq
->
delayedTask
=
taosOpenQueue
();
pTmq
->
qall
=
taosAllocateQall
();
taosThreadMutexInit
(
&
pTmq
->
lock
,
NULL
);
if
(
pTmq
->
clientTopics
==
NULL
||
pTmq
->
mqueue
==
NULL
||
pTmq
->
qall
==
NULL
||
pTmq
->
delayedTask
==
NULL
||
conf
->
groupId
[
0
]
==
0
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
...
...
@@ -1233,7 +1229,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) {
tmq_t
*
tmq
=
taosAcquireRef
(
tmqMgmt
.
rsetId
,
refId
);
if
(
tmq
==
NULL
)
{
tsem_destroy
(
&
pParam
->
rspSem
);
taosMemoryFree
(
pParam
);
taosMemoryFree
(
pMsg
->
pData
);
taosMemoryFree
(
pMsg
->
pEpSet
);
...
...
@@ -1419,7 +1414,7 @@ static void freeClientVgInfo(void* param) {
taosArrayDestroy
(
pTopic
->
vgs
);
}
static
bool
tmqUpdate
Ep
(
tmq_t
*
tmq
,
int32_t
epoch
,
const
SMqAskEpRsp
*
pRsp
)
{
static
bool
doUpdateLocal
Ep
(
tmq_t
*
tmq
,
int32_t
epoch
,
const
SMqAskEpRsp
*
pRsp
)
{
bool
set
=
false
;
int32_t
topicNumCur
=
taosArrayGetSize
(
tmq
->
clientTopics
);
...
...
@@ -1471,14 +1466,11 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) {
taosHashCleanup
(
pVgOffsetHashMap
);
taosThreadMutexLock
(
&
tmq
->
lock
);
// destroy current buffered existed topics info
if
(
tmq
->
clientTopics
)
{
taosArrayDestroyEx
(
tmq
->
clientTopics
,
freeClientVgInfo
);
}
tmq
->
clientTopics
=
newTopics
;
taosThreadMutexUnlock
(
&
tmq
->
lock
);
int8_t
flag
=
(
topicNumGet
==
0
)
?
TMQ_CONSUMER_STATUS__NO_TOPIC
:
TMQ_CONSUMER_STATUS__READY
;
atomic_store_8
(
&
tmq
->
status
,
flag
);
...
...
@@ -1742,7 +1734,7 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p
if
(
rspWrapper
->
epoch
>
atomic_load_32
(
&
tmq
->
epoch
))
{
SMqAskEpRspWrapper
*
pEpRspWrapper
=
(
SMqAskEpRspWrapper
*
)
rspWrapper
;
SMqAskEpRsp
*
rspMsg
=
&
pEpRspWrapper
->
msg
;
tmqUpdate
Ep
(
tmq
,
rspWrapper
->
epoch
,
rspMsg
);
doUpdateLocal
Ep
(
tmq
,
rspWrapper
->
epoch
,
rspMsg
);
/*tmqClearUnhandleMsg(tmq);*/
tDeleteSMqAskEpRsp
(
rspMsg
);
*
pReset
=
true
;
...
...
@@ -2163,7 +2155,7 @@ void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* par
SMqAskEpRsp
rsp
;
tDecodeSMqAskEpRsp
(
POINTER_SHIFT
(
pDataBuf
->
pData
,
sizeof
(
SMqRspHead
)),
&
rsp
);
tmqUpdate
Ep
(
pTmq
,
head
->
epoch
,
&
rsp
);
doUpdateLocal
Ep
(
pTmq
,
head
->
epoch
,
&
rsp
);
tDeleteSMqAskEpRsp
(
&
rsp
);
}
...
...
source/dnode/vnode/src/tq/tq.c
浏览文件 @
3ef7d43d
...
...
@@ -819,13 +819,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
pHandle
->
pRef
=
pRef
;
SReadHandle
handle
=
{
.
meta
=
pVnode
->
pMeta
,
.
vnode
=
pVnode
,
.
initTableReader
=
true
,
.
initTqReader
=
true
,
.
version
=
ver
,
};
.
meta
=
pVnode
->
pMeta
,
.
vnode
=
pVnode
,
.
initTableReader
=
true
,
.
initTqReader
=
true
,
.
version
=
ver
};
pHandle
->
snapshotVer
=
ver
;
if
(
pHandle
->
execHandle
.
subType
==
TOPIC_SUB_TYPE__COLUMN
)
{
...
...
@@ -1393,11 +1387,10 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) {
}
int32_t
tqProcessSubmitReq
(
STQ
*
pTq
,
SPackedData
submit
)
{
void
*
pIter
=
NULL
;
bool
failed
=
false
;
SStreamDataSubmit2
*
pSubmit
=
NULL
;
void
*
pIter
=
NULL
;
bool
failed
=
false
;
pSubmit
=
streamDataSubmitNew
(
submit
);
SStreamDataSubmit2
*
pSubmit
=
streamDataSubmitNew
(
submit
);
if
(
pSubmit
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
tqError
(
"failed to create data submit for stream since out of memory"
);
...
...
@@ -1411,7 +1404,10 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) {
}
SStreamTask
*
pTask
=
*
(
SStreamTask
**
)
pIter
;
if
(
pTask
->
taskLevel
!=
TASK_LEVEL__SOURCE
)
continue
;
if
(
pTask
->
taskLevel
!=
TASK_LEVEL__SOURCE
)
{
continue
;
}
if
(
pTask
->
taskStatus
==
TASK_STATUS__RECOVER_PREPARE
||
pTask
->
taskStatus
==
TASK_STATUS__WAIT_DOWNSTREAM
)
{
tqDebug
(
"skip push task %d, task status %d"
,
pTask
->
taskId
,
pTask
->
taskStatus
);
continue
;
...
...
source/dnode/vnode/src/tq/tqPush.c
浏览文件 @
3ef7d43d
...
...
@@ -207,33 +207,18 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_
#endif
typedef
struct
{
void
*
pKey
;
void
*
pKey
;
int64_t
keyLen
;
}
SItem
;
static
void
recordPushedEntry
(
SArray
*
cachedKey
,
void
*
pIter
);
static
void
doRemovePushedEntry
(
SArray
*
pCachedKeys
,
STQ
*
pTq
);
static
void
freeItem
(
void
*
param
)
{
SItem
*
p
=
(
SItem
*
)
param
;
taosMemoryFree
(
p
->
pKey
);
}
static
void
doRemovePushedEntry
(
SArray
*
pCachedKeys
,
STQ
*
pTq
)
{
int32_t
vgId
=
TD_VID
(
pTq
->
pVnode
);
int32_t
numOfKeys
=
(
int32_t
)
taosArrayGetSize
(
pCachedKeys
);
for
(
int32_t
i
=
0
;
i
<
numOfKeys
;
i
++
)
{
SItem
*
pItem
=
taosArrayGet
(
pCachedKeys
,
i
);
if
(
taosHashRemove
(
pTq
->
pPushMgr
,
pItem
->
pKey
,
pItem
->
keyLen
)
!=
0
)
{
tqError
(
"vgId:%d, tq push hash remove key error, key: %s"
,
vgId
,
(
char
*
)
pItem
->
pKey
);
}
}
if
(
numOfKeys
>
0
)
{
tqDebug
(
"vgId:%d, pushed %d items and remain:%d"
,
vgId
,
numOfKeys
,
(
int32_t
)
taosHashGetSize
(
pTq
->
pPushMgr
));
}
}
static
void
doPushDataForEntry
(
void
*
pIter
,
STqExecHandle
*
pExec
,
STQ
*
pTq
,
int64_t
ver
,
int32_t
vgId
,
char
*
pData
,
int32_t
dataLen
,
SArray
*
pCachedKey
)
{
STqPushEntry
*
pPushEntry
=
*
(
STqPushEntry
**
)
pIter
;
...
...
@@ -347,7 +332,7 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v
void
*
data
=
taosMemoryMalloc
(
len
);
if
(
data
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
tqError
(
"
failed to copy data for stream since out of memory"
);
tqError
(
"
vgId:%d, failed to copy submit data for stream processing, since out of memory"
,
vgId
);
return
-
1
;
}
...
...
@@ -366,13 +351,6 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v
return
0
;
}
void
recordPushedEntry
(
SArray
*
cachedKey
,
void
*
pIter
)
{
size_t
kLen
=
0
;
void
*
key
=
taosHashGetKey
(
pIter
,
&
kLen
);
SItem
item
=
{.
pKey
=
strndup
(
key
,
kLen
),
.
keyLen
=
kLen
};
taosArrayPush
(
cachedKey
,
&
item
);
}
int32_t
tqRegisterPushEntry
(
STQ
*
pTq
,
void
*
pHandle
,
const
SMqPollReq
*
pRequest
,
SRpcMsg
*
pRpcMsg
,
SMqDataRsp
*
pDataRsp
,
int32_t
type
)
{
uint64_t
consumerId
=
pRequest
->
consumerId
;
...
...
@@ -430,3 +408,26 @@ int32_t tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64
return
0
;
}
void
recordPushedEntry
(
SArray
*
cachedKey
,
void
*
pIter
)
{
size_t
kLen
=
0
;
void
*
key
=
taosHashGetKey
(
pIter
,
&
kLen
);
SItem
item
=
{.
pKey
=
strndup
(
key
,
kLen
),
.
keyLen
=
kLen
};
taosArrayPush
(
cachedKey
,
&
item
);
}
void
doRemovePushedEntry
(
SArray
*
pCachedKeys
,
STQ
*
pTq
)
{
int32_t
vgId
=
TD_VID
(
pTq
->
pVnode
);
int32_t
numOfKeys
=
(
int32_t
)
taosArrayGetSize
(
pCachedKeys
);
for
(
int32_t
i
=
0
;
i
<
numOfKeys
;
i
++
)
{
SItem
*
pItem
=
taosArrayGet
(
pCachedKeys
,
i
);
if
(
taosHashRemove
(
pTq
->
pPushMgr
,
pItem
->
pKey
,
pItem
->
keyLen
)
!=
0
)
{
tqError
(
"vgId:%d, tq push hash remove key error, key: %s"
,
vgId
,
(
char
*
)
pItem
->
pKey
);
}
}
if
(
numOfKeys
>
0
)
{
tqDebug
(
"vgId:%d, pushed %d items and remain:%d"
,
vgId
,
numOfKeys
,
(
int32_t
)
taosHashGetSize
(
pTq
->
pPushMgr
));
}
}
source/dnode/vnode/src/vnd/vnodeSvr.c
浏览文件 @
3ef7d43d
...
...
@@ -541,6 +541,7 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) {
return
vnodeGetBatchMeta
(
pVnode
,
pMsg
);
case
TDMT_VND_TMQ_CONSUME
:
return
tqProcessPollReq
(
pVnode
->
pTq
,
pMsg
);
case
TDMT_STREAM_TASK_RUN
:
return
tqProcessTaskRunReq
(
pVnode
->
pTq
,
pMsg
);
#if 1
...
...
source/libs/executor/inc/executorimpl.h
浏览文件 @
3ef7d43d
...
...
@@ -502,32 +502,12 @@ typedef struct STableCountScanSupp {
char
stbNameFilter
[
TSDB_TABLE_NAME_LEN
];
}
STableCountScanSupp
;
typedef
struct
STableCountScanOperatorInfo
{
SReadHandle
readHandle
;
SSDataBlock
*
pRes
;
STableCountScanSupp
supp
;
int32_t
currGrpIdx
;
SArray
*
stbUidList
;
// when group by db_name and/or stable_name
}
STableCountScanOperatorInfo
;
typedef
struct
SOptrBasicInfo
{
SResultRowInfo
resultRowInfo
;
SSDataBlock
*
pRes
;
bool
mergeResultBlock
;
}
SOptrBasicInfo
;
typedef
struct
SAggOperatorInfo
{
SOptrBasicInfo
binfo
;
SAggSupporter
aggSup
;
STableQueryInfo
*
current
;
uint64_t
groupId
;
SGroupResInfo
groupResInfo
;
SExprSupp
scalarExprSup
;
bool
groupKeyOptimized
;
}
SAggOperatorInfo
;
typedef
struct
SIntervalAggOperatorInfo
{
SOptrBasicInfo
binfo
;
// basic info
SAggSupporter
aggSup
;
// aggregate supporter
...
...
source/libs/executor/src/aggregateoperator.c
浏览文件 @
3ef7d43d
...
...
@@ -40,6 +40,16 @@ typedef struct {
int32_t
startOffset
;
}
SFunctionCtxStatus
;
typedef
struct
SAggOperatorInfo
{
SOptrBasicInfo
binfo
;
SAggSupporter
aggSup
;
STableQueryInfo
*
current
;
uint64_t
groupId
;
SGroupResInfo
groupResInfo
;
SExprSupp
scalarExprSup
;
bool
groupKeyOptimized
;
}
SAggOperatorInfo
;
static
void
destroyAggOperatorInfo
(
void
*
param
);
static
void
setExecutionContext
(
SOperatorInfo
*
pOperator
,
int32_t
numOfOutput
,
uint64_t
groupId
);
...
...
source/libs/executor/src/executor.c
浏览文件 @
3ef7d43d
...
...
@@ -127,12 +127,10 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
pOperator
->
status
=
OP_NOT_OPENED
;
SStreamScanInfo
*
pInfo
=
pOperator
->
info
;
qDebug
(
"stream set total blocks:%d, task id:%s"
PRIx64
,
(
int32_t
)
numOfBlocks
,
id
);
ASSERT
(
pInfo
->
validBlockIndex
==
0
);
ASSERT
(
taosArrayGetSize
(
pInfo
->
pBlockLists
)
==
0
);
qDebug
(
"task stream set total blocks:%d %s"
,
(
int32_t
)
numOfBlocks
,
id
);
ASSERT
(
pInfo
->
validBlockIndex
==
0
&&
taosArrayGetSize
(
pInfo
->
pBlockLists
)
==
0
);
if
(
type
==
STREAM_INPUT__MERGED_SUBMIT
)
{
// ASSERT(numOfBlocks > 1);
for
(
int32_t
i
=
0
;
i
<
numOfBlocks
;
i
++
)
{
SPackedData
*
pReq
=
POINTER_SHIFT
(
input
,
i
*
sizeof
(
SPackedData
));
taosArrayPush
(
pInfo
->
pBlockLists
,
pReq
);
...
...
source/libs/executor/src/scanoperator.c
浏览文件 @
3ef7d43d
...
...
@@ -33,7 +33,6 @@
int32_t
scanDebug
=
0
;
#define MULTI_READER_MAX_TABLE_NUM 5000
#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
...
...
@@ -52,6 +51,16 @@ typedef struct STableMergeScanSortSourceParam {
STsdbReader
*
dataReader
;
}
STableMergeScanSortSourceParam
;
typedef
struct
STableCountScanOperatorInfo
{
SReadHandle
readHandle
;
SSDataBlock
*
pRes
;
STableCountScanSupp
supp
;
int32_t
currGrpIdx
;
SArray
*
stbUidList
;
// when group by db_name and/or stable_name
}
STableCountScanOperatorInfo
;
static
bool
processBlockWithProbability
(
const
SSampleExecInfo
*
pInfo
);
bool
processBlockWithProbability
(
const
SSampleExecInfo
*
pInfo
)
{
...
...
source/libs/executor/src/timewindowoperator.c
浏览文件 @
3ef7d43d
...
...
@@ -4758,6 +4758,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
if
(
!
pInfo
->
pUpdated
)
{
pInfo
->
pUpdated
=
taosArrayInit
(
4
,
sizeof
(
SWinKey
));
}
if
(
!
pInfo
->
pUpdatedMap
)
{
_hash_fn_t
hashFn
=
taosGetDefaultHashFunction
(
TSDB_DATA_TYPE_BINARY
);
pInfo
->
pUpdatedMap
=
tSimpleHashInit
(
1024
,
hashFn
);
...
...
source/libs/stream/src/stream.c
浏览文件 @
3ef7d43d
...
...
@@ -92,22 +92,22 @@ int32_t streamSetupTrigger(SStreamTask* pTask) {
int32_t
streamSchedExec
(
SStreamTask
*
pTask
)
{
int8_t
schedStatus
=
atomic_val_compare_exchange_8
(
&
pTask
->
schedStatus
,
TASK_SCHED_STATUS__INACTIVE
,
TASK_SCHED_STATUS__WAITING
);
if
(
schedStatus
==
TASK_SCHED_STATUS__INACTIVE
)
{
SStreamTaskRunReq
*
pRunReq
=
rpcMallocCont
(
sizeof
(
SStreamTaskRunReq
));
if
(
pRunReq
==
NULL
)
{
atomic_store_8
(
&
pTask
->
schedStatus
,
TASK_SCHED_STATUS__INACTIVE
);
return
-
1
;
}
pRunReq
->
head
.
vgId
=
pTask
->
nodeId
;
pRunReq
->
streamId
=
pTask
->
streamId
;
pRunReq
->
taskId
=
pTask
->
taskId
;
SRpcMsg
msg
=
{
.
msgType
=
TDMT_STREAM_TASK_RUN
,
.
pCont
=
pRunReq
,
.
contLen
=
sizeof
(
SStreamTaskRunReq
),
};
SRpcMsg
msg
=
{
.
msgType
=
TDMT_STREAM_TASK_RUN
,
.
pCont
=
pRunReq
,
.
contLen
=
sizeof
(
SStreamTaskRunReq
)
};
tmsgPutToQueue
(
pTask
->
pMsgCb
,
STREAM_QUEUE
,
&
msg
);
}
return
0
;
}
...
...
@@ -275,7 +275,38 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S
return
0
;
}
// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) {
// //
// return 0;
// }
int32_t
streamTaskInput
(
SStreamTask
*
pTask
,
SStreamQueueItem
*
pItem
)
{
int8_t
type
=
pItem
->
type
;
if
(
type
==
STREAM_INPUT__DATA_SUBMIT
)
{
SStreamDataSubmit2
*
pSubmitClone
=
streamSubmitRefClone
((
SStreamDataSubmit2
*
)
pItem
);
if
(
pSubmitClone
==
NULL
)
{
qDebug
(
"task %d %p submit enqueue failed since out of memory"
,
pTask
->
taskId
,
pTask
);
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
atomic_store_8
(
&
pTask
->
inputStatus
,
TASK_INPUT_STATUS__FAILED
);
return
-
1
;
}
taosWriteQitem
(
pTask
->
inputQueue
->
queue
,
pSubmitClone
);
qDebug
(
"stream task:%d %p submit enqueue %p %p %p msgLen:%d ver:%"
PRId64
", total in queue:%d"
,
pTask
->
taskId
,
pTask
,
pItem
,
pSubmitClone
,
pSubmitClone
->
submit
.
msgStr
,
pSubmitClone
->
submit
.
msgLen
,
pSubmitClone
->
submit
.
ver
,
pTask
->
inputQueue
->
queue
->
numOfItems
);
}
else
if
(
type
==
STREAM_INPUT__DATA_BLOCK
||
type
==
STREAM_INPUT__DATA_RETRIEVE
||
type
==
STREAM_INPUT__REF_DATA_BLOCK
)
{
taosWriteQitem
(
pTask
->
inputQueue
->
queue
,
pItem
);
}
else
if
(
type
==
STREAM_INPUT__CHECKPOINT
)
{
taosWriteQitem
(
pTask
->
inputQueue
->
queue
,
pItem
);
}
else
if
(
type
==
STREAM_INPUT__GET_RES
)
{
taosWriteQitem
(
pTask
->
inputQueue
->
queue
,
pItem
);
}
if
(
type
!=
STREAM_INPUT__GET_RES
&&
type
!=
STREAM_INPUT__CHECKPOINT
&&
pTask
->
triggerParam
!=
0
)
{
atomic_val_compare_exchange_8
(
&
pTask
->
triggerStatus
,
TASK_TRIGGER_STATUS__INACTIVE
,
TASK_TRIGGER_STATUS__ACTIVE
);
}
#if 0
// TODO: back pressure
atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL);
#endif
return
0
;
}
\ No newline at end of file
source/libs/stream/src/streamData.c
浏览文件 @
3ef7d43d
...
...
@@ -68,16 +68,20 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock
SStreamDataSubmit2
*
streamDataSubmitNew
(
SPackedData
submit
)
{
SStreamDataSubmit2
*
pDataSubmit
=
(
SStreamDataSubmit2
*
)
taosAllocateQitem
(
sizeof
(
SStreamDataSubmit2
),
DEF_QITEM
,
0
);
if
(
pDataSubmit
==
NULL
)
return
NULL
;
if
(
pDataSubmit
==
NULL
)
{
return
NULL
;
}
pDataSubmit
->
dataRef
=
(
int32_t
*
)
taosMemoryMalloc
(
sizeof
(
int32_t
));
if
(
pDataSubmit
->
dataRef
==
NULL
)
goto
FAIL
;
if
(
pDataSubmit
->
dataRef
==
NULL
)
{
taosFreeQitem
(
pDataSubmit
);
return
NULL
;
}
pDataSubmit
->
submit
=
submit
;
*
pDataSubmit
->
dataRef
=
1
;
pDataSubmit
->
type
=
STREAM_INPUT__DATA_SUBMIT
;
return
pDataSubmit
;
FAIL:
taosFreeQitem
(
pDataSubmit
);
return
NULL
;
}
SStreamMergedSubmit2
*
streamMergedSubmitNew
()
{
...
...
source/libs/stream/src/streamExec.c
浏览文件 @
3ef7d43d
...
...
@@ -34,7 +34,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray*
}
else
if
(
pItem
->
type
==
STREAM_INPUT__DATA_SUBMIT
)
{
ASSERT
(
pTask
->
taskLevel
==
TASK_LEVEL__SOURCE
);
const
SStreamDataSubmit2
*
pSubmit
=
(
const
SStreamDataSubmit2
*
)
data
;
qDebug
(
"
task
%d %p set submit input %p %p %d %"
PRId64
,
pTask
->
taskId
,
pTask
,
pSubmit
,
pSubmit
->
submit
.
msgStr
,
qDebug
(
"
stream task:
%d %p set submit input %p %p %d %"
PRId64
,
pTask
->
taskId
,
pTask
,
pSubmit
,
pSubmit
->
submit
.
msgStr
,
pSubmit
->
submit
.
msgLen
,
pSubmit
->
submit
.
ver
);
qSetMultiStreamInput
(
exec
,
&
pSubmit
->
submit
,
1
,
STREAM_INPUT__DATA_SUBMIT
);
}
else
if
(
pItem
->
type
==
STREAM_INPUT__DATA_BLOCK
||
pItem
->
type
==
STREAM_INPUT__DATA_RETRIEVE
)
{
...
...
@@ -268,9 +268,10 @@ int32_t streamExecForAll(SStreamTask* pTask) {
SArray
*
pRes
=
taosArrayInit
(
0
,
sizeof
(
SSDataBlock
));
qDebug
(
"stream task
%d exec begin, msg batch: %d"
,
pTask
->
taskId
,
batchCnt
);
qDebug
(
"stream task
:
%d exec begin, msg batch: %d"
,
pTask
->
taskId
,
batchCnt
);
streamTaskExecImpl
(
pTask
,
input
,
pRes
);
qDebug
(
"stream task %d exec end"
,
pTask
->
taskId
);
qDebug
(
"stream task:%d exec end"
,
pTask
->
taskId
);
if
(
taosArrayGetSize
(
pRes
)
!=
0
)
{
SStreamDataBlock
*
qRes
=
taosAllocateQitem
(
sizeof
(
SStreamDataBlock
),
DEF_QITEM
,
0
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录