Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
efa99b87
T
TDengine
项目概览
taosdata
/
TDengine
大约 2 年 前同步成功
通知
1193
Star
22018
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
efa99b87
编写于
7月 31, 2023
作者:
K
kailixu
浏览文件
操作
浏览文件
下载
差异文件
Merge branch '3.0' into fix/TD-24872-3.0
上级
f70709be
3c187414
变更
29
显示空白变更内容
内联
并排
Showing
29 changed file
with
556 addition
and
358 deletion
+556
-358
docs/en/14-reference/03-connector/06-rust.mdx
docs/en/14-reference/03-connector/06-rust.mdx
+2
-1
docs/zh/08-connector/26-rust.mdx
docs/zh/08-connector/26-rust.mdx
+2
-1
include/libs/stream/tstream.h
include/libs/stream/tstream.h
+0
-1
source/client/src/clientTmq.c
source/client/src/clientTmq.c
+5
-5
source/common/src/tmsg.c
source/common/src/tmsg.c
+3
-0
source/dnode/mnode/impl/src/mndConsumer.c
source/dnode/mnode/impl/src/mndConsumer.c
+6
-1
source/dnode/mnode/impl/src/mndSubscribe.c
source/dnode/mnode/impl/src/mndSubscribe.c
+1
-1
source/dnode/snode/src/snode.c
source/dnode/snode/src/snode.c
+3
-3
source/dnode/vnode/src/tq/tq.c
source/dnode/vnode/src/tq/tq.c
+112
-106
source/dnode/vnode/src/tq/tqPush.c
source/dnode/vnode/src/tq/tqPush.c
+3
-3
source/dnode/vnode/src/tq/tqRestore.c
source/dnode/vnode/src/tq/tqRestore.c
+4
-4
source/dnode/vnode/src/tq/tqUtil.c
source/dnode/vnode/src/tq/tqUtil.c
+1
-0
source/dnode/vnode/src/tsdb/tsdbCache.c
source/dnode/vnode/src/tsdb/tsdbCache.c
+15
-107
source/dnode/vnode/src/vnd/vnodeOpen.c
source/dnode/vnode/src/vnd/vnodeOpen.c
+1
-1
source/libs/executor/src/executor.c
source/libs/executor/src/executor.c
+1
-0
source/libs/executor/src/scanoperator.c
source/libs/executor/src/scanoperator.c
+97
-92
source/libs/executor/src/timewindowoperator.c
source/libs/executor/src/timewindowoperator.c
+4
-9
source/libs/parser/inc/parUtil.h
source/libs/parser/inc/parUtil.h
+6
-0
source/libs/parser/src/parTranslater.c
source/libs/parser/src/parTranslater.c
+18
-16
source/libs/parser/src/parUtil.c
source/libs/parser/src/parUtil.c
+15
-0
source/libs/stream/src/stream.c
source/libs/stream/src/stream.c
+1
-1
source/libs/stream/src/streamExec.c
source/libs/stream/src/streamExec.c
+10
-1
source/libs/stream/src/streamRecover.c
source/libs/stream/src/streamRecover.c
+13
-1
source/util/CMakeLists.txt
source/util/CMakeLists.txt
+7
-0
source/util/src/tlog.c
source/util/src/tlog.c
+4
-0
tests/parallel_test/cases.task
tests/parallel_test/cases.task
+2
-2
tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
...st/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
+2
-2
tests/system-test/eco-system/manager/cmul.py
tests/system-test/eco-system/manager/cmul.py
+104
-0
tests/system-test/eco-system/manager/mul.py
tests/system-test/eco-system/manager/mul.py
+114
-0
未找到文件。
docs/en/14-reference/03-connector/06-rust.mdx
浏览文件 @
efa99b87
...
...
@@ -31,7 +31,8 @@ Websocket connections are supported on all platforms that can run Go.
| connector-rust version | TDengine version | major features |
| :----------------: | :--------------: | :--------------------------------------------------: |
| v0.8.12 | 3.0.5.0 or later | TMQ: Get consuming progress and seek offset to consume. |
| v0.9.2 | 3.0.7.0 or later | STMT: Get tag_fields and col_fields under ws. |
| v0.8.12 | 3.0.5.0 | TMQ: Get consuming progress and seek offset to consume. |
| v0.8.0 | 3.0.4.0 | Support schemaless insert. |
| v0.7.6 | 3.0.3.0 | Support req_id in query. |
| v0.6.0 | 3.0.0.0 | Base features. |
...
...
docs/zh/08-connector/26-rust.mdx
浏览文件 @
efa99b87
...
...
@@ -30,7 +30,8 @@ Websocket 连接支持所有能运行 Rust 的平台。
| Rust 连接器版本 | TDengine 版本 | 主要功能 |
| :----------------: | :--------------: | :--------------------------------------------------: |
| v0.8.12 | 3.0.5.0 or later | 消息订阅:获取消费进度及按照指定进度开始消费。 |
| v0.9.2 | 3.0.7.0 or later | STMT:ws 下获取 tag_fields、col_fields。 |
| v0.8.12 | 3.0.5.0 | 消息订阅:获取消费进度及按照指定进度开始消费。 |
| v0.8.0 | 3.0.4.0 | 支持无模式写入。 |
| v0.7.6 | 3.0.3.0 | 支持在请求中使用 req_id。 |
| v0.6.0 | 3.0.0.0 | 基础功能。 |
...
...
include/libs/stream/tstream.h
浏览文件 @
efa99b87
...
...
@@ -45,7 +45,6 @@ enum {
TASK_STATUS__FAIL
,
TASK_STATUS__STOP
,
TASK_STATUS__SCAN_HISTORY
,
// stream task scan history data by using tsdbread in the stream scanner
TASK_STATUS__SCAN_HISTORY_WAL
,
// scan history data in wal
TASK_STATUS__HALT
,
// pause, but not be manipulated by user command
TASK_STATUS__PAUSE
,
// pause
};
...
...
source/client/src/clientTmq.c
浏览文件 @
efa99b87
...
...
@@ -1859,8 +1859,8 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p
static
void
updateVgInfo
(
SMqClientVg
*
pVg
,
STqOffsetVal
*
reqOffset
,
STqOffsetVal
*
rspOffset
,
int64_t
sver
,
int64_t
ever
,
int64_t
consumerId
){
if
(
!
pVg
->
seekUpdated
)
{
tscDebug
(
"consumer:0x%"
PRIx64
" local offset is update, since seekupdate not set"
,
consumerId
);
if
(
reqOffset
->
type
!=
0
)
pVg
->
offsetInfo
.
beginOffset
=
*
reqOffset
;
if
(
rspOffset
->
type
!=
0
)
pVg
->
offsetInfo
.
endOffset
=
*
rspOffset
;
pVg
->
offsetInfo
.
beginOffset
=
*
reqOffset
;
pVg
->
offsetInfo
.
endOffset
=
*
rspOffset
;
}
else
{
tscDebug
(
"consumer:0x%"
PRIx64
" local offset is NOT update, since seekupdate is set"
,
consumerId
);
}
...
...
@@ -1948,7 +1948,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
}
taosWUnLockLatch
(
&
tmq
->
lock
);
}
else
{
tsc
Debug
(
"consumer:0x%"
PRIx64
" vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d"
,
tsc
Info
(
"consumer:0x%"
PRIx64
" vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d"
,
tmq
->
consumerId
,
pollRspWrapper
->
vgId
,
pDataRsp
->
head
.
epoch
,
consumerEpoch
);
pRspWrapper
=
tmqFreeRspWrapper
(
pRspWrapper
);
taosFreeQitem
(
pollRspWrapper
);
...
...
@@ -1979,7 +1979,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
taosWUnLockLatch
(
&
tmq
->
lock
);
return
pRsp
;
}
else
{
tsc
Debug
(
"consumer:0x%"
PRIx64
" vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d"
,
tsc
Info
(
"consumer:0x%"
PRIx64
" vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d"
,
tmq
->
consumerId
,
pollRspWrapper
->
vgId
,
pollRspWrapper
->
metaRsp
.
head
.
epoch
,
consumerEpoch
);
pRspWrapper
=
tmqFreeRspWrapper
(
pRspWrapper
);
taosFreeQitem
(
pollRspWrapper
);
...
...
@@ -2036,7 +2036,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) {
taosWUnLockLatch
(
&
tmq
->
lock
);
return
pRsp
;
}
else
{
tsc
Debug
(
"consumer:0x%"
PRIx64
" vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d"
,
tsc
Info
(
"consumer:0x%"
PRIx64
" vgId:%d msg discard since epoch mismatch: msg epoch %d, consumer epoch %d"
,
tmq
->
consumerId
,
pollRspWrapper
->
vgId
,
pollRspWrapper
->
taosxRsp
.
head
.
epoch
,
consumerEpoch
);
pRspWrapper
=
tmqFreeRspWrapper
(
pRspWrapper
);
taosFreeQitem
(
pollRspWrapper
);
...
...
source/common/src/tmsg.c
浏览文件 @
efa99b87
...
...
@@ -7227,6 +7227,9 @@ bool tOffsetEqual(const STqOffsetVal *pLeft, const STqOffsetVal *pRight) {
return
pLeft
->
uid
==
pRight
->
uid
&&
pLeft
->
ts
==
pRight
->
ts
;
}
else
if
(
pLeft
->
type
==
TMQ_OFFSET__SNAPSHOT_META
)
{
return
pLeft
->
uid
==
pRight
->
uid
;
}
else
{
uError
(
"offset type:%d"
,
pLeft
->
type
);
ASSERT
(
0
);
}
}
return
false
;
...
...
source/dnode/mnode/impl/src/mndConsumer.c
浏览文件 @
efa99b87
...
...
@@ -420,6 +420,7 @@ static int32_t mndProcessMqHbReq(SRpcMsg *pMsg) {
SMqSubscribeObj
*
pSub
=
mndAcquireSubscribe
(
pMnode
,
pConsumer
->
cgroup
,
data
->
topicName
);
if
(
pSub
==
NULL
){
ASSERT
(
0
);
continue
;
}
taosWLockLatch
(
&
pSub
->
lock
);
...
...
@@ -515,7 +516,10 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
char
*
topic
=
taosArrayGetP
(
pConsumer
->
currentTopics
,
i
);
SMqSubscribeObj
*
pSub
=
mndAcquireSubscribe
(
pMnode
,
pConsumer
->
cgroup
,
topic
);
// txn guarantees pSub is created
if
(
pSub
==
NULL
)
continue
;
if
(
pSub
==
NULL
)
{
ASSERT
(
0
);
continue
;
}
taosRLockLatch
(
&
pSub
->
lock
);
SMqSubTopicEp
topicEp
=
{
0
};
...
...
@@ -524,6 +528,7 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) {
// 2.1 fetch topic schema
SMqTopicObj
*
pTopic
=
mndAcquireTopic
(
pMnode
,
topic
);
if
(
pTopic
==
NULL
)
{
ASSERT
(
0
);
taosRUnLockLatch
(
&
pSub
->
lock
);
mndReleaseSubscribe
(
pMnode
,
pSub
);
continue
;
...
...
source/dnode/mnode/impl/src/mndSubscribe.c
浏览文件 @
efa99b87
...
...
@@ -1168,7 +1168,7 @@ static int32_t buildResult(SSDataBlock *pBlock, int32_t* numOfRows, int64_t cons
pColInfo
=
taosArrayGet
(
pBlock
->
pDataBlock
,
cols
++
);
colDataSetVal
(
pColInfo
,
*
numOfRows
,
(
const
char
*
)
consumerIdHex
,
consumerId
==
-
1
);
m
Debug
(
"mnd show subscriptions: topic %s, consumer:0x%"
PRIx64
" cgroup %s vgid %d"
,
varDataVal
(
topic
),
m
Info
(
"mnd show subscriptions: topic %s, consumer:0x%"
PRIx64
" cgroup %s vgid %d"
,
varDataVal
(
topic
),
consumerId
,
varDataVal
(
cgroup
),
pVgEp
->
vgId
);
// offset
...
...
source/dnode/snode/src/snode.c
浏览文件 @
efa99b87
...
...
@@ -346,9 +346,9 @@ int32_t sndProcessStreamTaskCheckReq(SSnode *pSnode, SRpcMsg *pMsg) {
rsp
.
status
=
streamTaskCheckStatus
(
pTask
);
streamMetaReleaseTask
(
pSnode
->
pMeta
,
pTask
);
qDebug
(
"s-task:%s recv task check req(reqId:0x%"
PRIx64
") task:0x%x (vgId:%d), status:%s, rsp status %d"
,
pTask
->
id
.
idStr
,
rsp
.
reqId
,
rsp
.
upstreamTaskId
,
rsp
.
upstreamNodeId
,
streamGetTaskStatusStr
(
pTask
->
status
.
taskStatus
)
,
rsp
.
status
);
const
char
*
pStatus
=
streamGetTaskStatusStr
(
pTask
->
status
.
taskStatus
);
qDebug
(
"s-task:%s status:%s, recv task check req(reqId:0x%"
PRIx64
") task:0x%x (vgId:%d), ready:%d"
,
pTask
->
id
.
idStr
,
pStatus
,
rsp
.
reqId
,
rsp
.
upstreamTaskId
,
rsp
.
upstreamNodeId
,
rsp
.
status
);
}
else
{
rsp
.
status
=
0
;
qDebug
(
"tq recv task check(taskId:0x%x not built yet) req(reqId:0x%"
PRIx64
...
...
source/dnode/vnode/src/tq/tq.c
浏览文件 @
efa99b87
...
...
@@ -146,6 +146,20 @@ void tqClose(STQ* pTq) {
return
;
}
void
*
pIter
=
taosHashIterate
(
pTq
->
pPushMgr
,
NULL
);
while
(
pIter
)
{
STqHandle
*
pHandle
=
*
(
STqHandle
**
)
pIter
;
int32_t
vgId
=
TD_VID
(
pTq
->
pVnode
);
if
(
pHandle
->
msg
!=
NULL
)
{
tqPushEmptyDataRsp
(
pHandle
,
vgId
);
rpcFreeCont
(
pHandle
->
msg
->
pCont
);
taosMemoryFree
(
pHandle
->
msg
);
pHandle
->
msg
=
NULL
;
}
pIter
=
taosHashIterate
(
pTq
->
pPushMgr
,
pIter
);
}
tqOffsetClose
(
pTq
->
pOffsetStore
);
taosHashCleanup
(
pTq
->
pHandle
);
taosHashCleanup
(
pTq
->
pPushMgr
);
...
...
@@ -278,6 +292,10 @@ int32_t tqPushEmptyDataRsp(STqHandle* pHandle, int32_t vgId) {
tqInitDataRsp
(
&
dataRsp
,
&
req
);
dataRsp
.
blockNum
=
0
;
dataRsp
.
rspOffset
=
dataRsp
.
reqOffset
;
char
buf
[
TSDB_OFFSET_LEN
]
=
{
0
};
tFormatOffset
(
buf
,
TSDB_OFFSET_LEN
,
&
dataRsp
.
reqOffset
);
tqInfo
(
"tqPushEmptyDataRsp to consumer:0x%"
PRIx64
" vgId:%d, offset:%s, reqId:0x%"
PRIx64
,
req
.
consumerId
,
vgId
,
buf
,
req
.
reqId
);
tqSendDataRsp
(
pHandle
,
pHandle
->
msg
,
&
req
,
&
dataRsp
,
TMQ_MSG_TYPE__POLL_DATA_RSP
,
vgId
);
tDeleteMqDataRsp
(
&
dataRsp
);
return
0
;
...
...
@@ -515,10 +533,11 @@ int32_t tqProcessPollPush(STQ* pTq, SRpcMsg* pMsg) {
while
(
pIter
)
{
STqHandle
*
pHandle
=
*
(
STqHandle
**
)
pIter
;
tq
Debug
(
"vgId:%d start set submit for pHandle:%p, consumer:0x%"
PRIx64
,
vgId
,
pHandle
,
pHandle
->
consumerId
);
tq
Info
(
"vgId:%d start set submit for pHandle:%p, consumer:0x%"
PRIx64
,
vgId
,
pHandle
,
pHandle
->
consumerId
);
if
(
ASSERT
(
pHandle
->
msg
!=
NULL
))
{
tqError
(
"pHandle->msg should not be null"
);
taosHashCancelIterate
(
pTq
->
pPushMgr
,
pIter
);
break
;
}
else
{
SRpcMsg
msg
=
{.
msgType
=
TDMT_VND_TMQ_CONSUME
,
.
pCont
=
pHandle
->
msg
->
pCont
,
.
contLen
=
pHandle
->
msg
->
contLen
,
.
info
=
pHandle
->
msg
->
info
};
...
...
@@ -849,13 +868,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
taosWLockLatch
(
&
pTq
->
lock
);
if
(
pHandle
->
consumerId
==
req
.
newConsumerId
)
{
// do nothing
tqInfo
(
"vgId:%d consumer:0x%"
PRIx64
" remains, no switch occurs, should not reach here"
,
req
.
vgId
,
req
.
newConsumerId
);
tqInfo
(
"vgId:%d no switch consumer:0x%"
PRIx64
" remains, because redo wal log"
,
req
.
vgId
,
req
.
newConsumerId
);
}
else
{
tqInfo
(
"vgId:%d switch consumer from Id:0x%"
PRIx64
" to Id:0x%"
PRIx64
,
req
.
vgId
,
pHandle
->
consumerId
,
req
.
newConsumerId
);
tqInfo
(
"vgId:%d switch consumer from Id:0x%"
PRIx64
" to Id:0x%"
PRIx64
,
req
.
vgId
,
pHandle
->
consumerId
,
req
.
newConsumerId
);
atomic_store_64
(
&
pHandle
->
consumerId
,
req
.
newConsumerId
);
}
// atomic_add_fetch_32(&pHandle->epoch, 1);
// kill executing task
...
...
@@ -871,9 +887,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg
// }
// remove if it has been register in the push manager, and return one empty block to consumer
tqUnregisterPushHandle
(
pTq
,
pHandle
);
taosWUnLockLatch
(
&
pTq
->
lock
);
ret
=
tqMetaSaveHandle
(
pTq
,
req
.
subKey
,
pHandle
);
}
taosWUnLockLatch
(
&
pTq
->
lock
);
}
end:
tDecoderClear
(
&
dc
);
...
...
@@ -1041,9 +1058,9 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) {
rsp
.
status
=
streamTaskCheckStatus
(
pTask
);
streamMetaReleaseTask
(
pTq
->
pStreamMeta
,
pTask
);
tqDebug
(
"s-task:%s recv task check req(reqId:0x%"
PRIx64
") task:0x%x (vgId:%d), status:%s, rsp status %d"
,
pTask
->
id
.
idStr
,
rsp
.
reqId
,
rsp
.
upstreamTaskId
,
rsp
.
upstreamNodeId
,
streamGetTaskStatusStr
(
pTask
->
status
.
taskStatus
)
,
rsp
.
status
);
const
char
*
pStatus
=
streamGetTaskStatusStr
(
pTask
->
status
.
taskStatus
);
tqDebug
(
"s-task:%s status:%s, recv task check req(reqId:0x%"
PRIx64
") task:0x%x (vgId:%d), ready:%d"
,
pTask
->
id
.
idStr
,
pStatus
,
rsp
.
reqId
,
rsp
.
upstreamTaskId
,
rsp
.
upstreamNodeId
,
rsp
.
status
);
}
else
{
rsp
.
status
=
0
;
tqDebug
(
"tq recv task check(taskId:0x%x not built yet) req(reqId:0x%"
PRIx64
") from task:0x%x (vgId:%d), rsp status %d"
,
...
...
@@ -1145,7 +1162,6 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
// 3. It's an fill history task, do nothing. wait for the main task to start it
SStreamTask
*
p
=
streamMetaAcquireTask
(
pStreamMeta
,
taskId
);
if
(
p
!=
NULL
)
{
// reset the downstreamReady flag.
p
->
status
.
downstreamReady
=
0
;
streamTaskCheckDownstreamTasks
(
p
);
}
...
...
@@ -1154,12 +1170,10 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms
}
int32_t
tqProcessTaskScanHistory
(
STQ
*
pTq
,
SRpcMsg
*
pMsg
)
{
int32_t
code
=
TSDB_CODE_SUCCESS
;
char
*
msg
=
pMsg
->
pCont
;
SStreamScanHistoryReq
*
pReq
=
(
SStreamScanHistoryReq
*
)
pMsg
->
pCont
;
SStreamMeta
*
pMeta
=
pTq
->
pStreamMeta
;
SStreamScanHistoryReq
*
pReq
=
(
SStreamScanHistoryReq
*
)
msg
;
int32_t
code
=
TSDB_CODE_SUCCESS
;
SStreamTask
*
pTask
=
streamMetaAcquireTask
(
pMeta
,
pReq
->
taskId
);
if
(
pTask
==
NULL
)
{
tqError
(
"vgId:%d failed to acquire stream task:0x%x during stream recover, task may have been destroyed"
,
...
...
@@ -1167,12 +1181,20 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
return
-
1
;
}
// do recovery step
1
// do recovery step1
const
char
*
id
=
pTask
->
id
.
idStr
;
const
char
*
pStatus
=
streamGetTaskStatusStr
(
pTask
->
status
.
taskStatus
);
tqDebug
(
"s-task:%s start
history data scan
stage(step 1), status:%s"
,
id
,
pStatus
);
tqDebug
(
"s-task:%s start
scan-history
stage(step 1), status:%s"
,
id
,
pStatus
);
int64_t
st
=
taosGetTimestampMs
();
if
(
pTask
->
tsInfo
.
step1Start
==
0
)
{
ASSERT
(
pTask
->
status
.
pauseAllowed
==
false
);
pTask
->
tsInfo
.
step1Start
=
taosGetTimestampMs
();
if
(
pTask
->
info
.
fillHistory
==
1
)
{
streamTaskEnablePause
(
pTask
);
}
}
else
{
tqDebug
(
"s-task:%s resume from paused, start ts:%"
PRId64
,
pTask
->
id
.
idStr
,
pTask
->
tsInfo
.
step1Start
);
}
// we have to continue retrying to successfully execute the scan history task.
int8_t
schedStatus
=
atomic_val_compare_exchange_8
(
&
pTask
->
status
.
schedStatus
,
TASK_SCHED_STATUS__INACTIVE
,
...
...
@@ -1185,31 +1207,21 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
return
0
;
}
ASSERT
(
pTask
->
status
.
pauseAllowed
==
false
);
if
(
pTask
->
info
.
fillHistory
==
1
)
{
streamTaskEnablePause
(
pTask
);
ASSERT
(
pTask
->
status
.
pauseAllowed
==
true
);
}
if
(
!
streamTaskRecoverScanStep1Finished
(
pTask
))
{
streamSourceScanHistoryData
(
pTask
);
}
// disable the pause when handling the step2 scan of tsdb data.
// the whole next procedure cann't be stopped.
// todo fix it: the following procedure should be executed completed and then shutdown when trying to close vnode.
if
(
pTask
->
info
.
fillHistory
==
1
)
{
streamTaskDisablePause
(
pTask
);
}
if
(
streamTaskShouldStop
(
&
pTask
->
status
)
||
streamTaskShouldPause
(
&
pTask
->
status
))
{
tqDebug
(
"s-task:%s is dropped or paused, abort recover in step1"
,
id
);
if
(
pTask
->
status
.
taskStatus
==
TASK_STATUS__PAUSE
)
{
double
el
=
(
taosGetTimestampMs
()
-
pTask
->
tsInfo
.
step1Start
)
/
1000
.
0
;
tqDebug
(
"s-task:%s is paused in the step1, elapsed time:%.2fs, sched-status:%d"
,
pTask
->
id
.
idStr
,
el
,
TASK_SCHED_STATUS__INACTIVE
);
atomic_store_8
(
&
pTask
->
status
.
schedStatus
,
TASK_SCHED_STATUS__INACTIVE
);
streamMetaReleaseTask
(
pMeta
,
pTask
);
return
0
;
}
double
el
=
(
taosGetTimestampMs
()
-
st
)
/
1000
.
0
;
// the following procedure should be executed, no matter status is stop/pause or not
double
el
=
(
taosGetTimestampMs
()
-
pTask
->
tsInfo
.
step1Start
)
/
1000
.
0
;
tqDebug
(
"s-task:%s scan-history stage(step 1) ended, elapsed time:%.2fs"
,
id
,
el
);
if
(
pTask
->
info
.
fillHistory
)
{
...
...
@@ -1217,25 +1229,24 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
SStreamTask
*
pStreamTask
=
NULL
;
bool
done
=
false
;
if
(
!
pReq
->
igUntreated
&&
!
streamTaskRecoverScanStep1Finished
(
pTask
))
{
// 1. stop the related stream task, get the current scan wal version of stream task, ver.
// 1. get the related stream task
pStreamTask
=
streamMetaAcquireTask
(
pMeta
,
pTask
->
streamTaskId
.
taskId
);
if
(
pStreamTask
==
NULL
)
{
// todo delete this task, if the related stream task is dropped
qError
(
"failed to find s-task:0x%x, it may have been destroyed, drop fill-history task:%s"
,
pTask
->
streamTaskId
.
taskId
,
pTask
->
id
.
idStr
);
pTask
->
status
.
taskStatus
=
TASK_STATUS__DROPPING
;
tqDebug
(
"s-task:%s fill-history task set status to be dropping"
,
id
);
streamMetaSaveTask
(
pMeta
,
pTask
);
streamMetaUnregisterTask
(
pMeta
,
pTask
->
id
.
taskId
);
streamMetaReleaseTask
(
pMeta
,
pTask
);
return
-
1
;
}
ASSERT
(
pStreamTask
->
info
.
taskLevel
==
TASK_LEVEL__SOURCE
);
// stream task in TASK_STATUS__SCAN_HISTORY can not be paused.
// wait for the
stream task get ready for scan history data
// 2. it cannot be paused, when the stream task in TASK_STATUS__SCAN_HISTORY status. Let's wait for the
//
stream task get ready for scan history data
while
(
pStreamTask
->
status
.
taskStatus
==
TASK_STATUS__SCAN_HISTORY
)
{
tqDebug
(
"s-task:%s level:%d related stream task:%s(status:%s) not ready for halt, wait for it and recheck in 100ms"
,
...
...
@@ -1245,6 +1256,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
// now we can stop the stream task execution
streamTaskHalt
(
pStreamTask
);
tqDebug
(
"s-task:%s level:%d sched-status:%d is halt by fill-history task:%s"
,
pStreamTask
->
id
.
idStr
,
pStreamTask
->
info
.
taskLevel
,
pStreamTask
->
status
.
schedStatus
,
id
);
...
...
@@ -1252,42 +1264,36 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
pRange
=
&
pTask
->
dataRange
.
range
;
int64_t
latestVer
=
walReaderGetCurrentVer
(
pStreamTask
->
exec
.
pWalReader
);
done
=
streamHistoryTaskSetVerRangeStep2
(
pTask
,
latestVer
);
}
if
(
done
)
{
pTask
->
tsInfo
.
step2Start
=
taosGetTimestampMs
();
streamTaskEndScanWAL
(
pTask
);
streamMetaReleaseTask
(
pMeta
,
pTask
);
}
else
{
if
(
!
streamTaskRecoverScanStep1Finished
(
pTask
))
{
STimeWindow
*
pWindow
=
&
pTask
->
dataRange
.
window
;
tqDebug
(
"s-task:%s level:%d verRange:%"
PRId64
" - %"
PRId64
" window:%"
PRId64
"-%"
PRId64
", do secondary scan-history from WAL after halt the related stream task:%s"
,
id
,
pTask
->
info
.
taskLevel
,
pRange
->
minVer
,
pRange
->
maxVer
,
pWindow
->
skey
,
pWindow
->
ekey
,
id
);
id
,
pTask
->
info
.
taskLevel
,
pRange
->
minVer
,
pRange
->
maxVer
,
pWindow
->
skey
,
pWindow
->
ekey
,
pStreamTask
->
id
.
idStr
);
ASSERT
(
pTask
->
status
.
schedStatus
==
TASK_SCHED_STATUS__WAITING
);
pTask
->
tsInfo
.
step2Start
=
taosGetTimestampMs
();
streamSetParamForStreamScannerStep2
(
pTask
,
pRange
,
pWindow
);
}
if
(
!
streamTaskRecoverScanStep2Finished
(
pTask
))
{
pTask
->
status
.
taskStatus
=
TASK_STATUS__SCAN_HISTORY_WAL
;
if
(
streamTaskShouldStop
(
&
pTask
->
status
)
||
streamTaskShouldPause
(
&
pTask
->
status
))
{
tqDebug
(
"s-task:%s is dropped or paused, abort recover in step1"
,
id
);
streamMetaReleaseTask
(
pMeta
,
pTask
);
return
0
;
}
int64_t
dstVer
=
pTask
->
dataRange
.
range
.
minVer
-
1
;
pTask
->
chkInfo
.
currentVer
=
dstVer
;
walReaderSetSkipToVersion
(
pTask
->
exec
.
pWalReader
,
dstVer
);
tqDebug
(
"s-task:%s wal reader start scan from WAL ver:%"
PRId64
", set sched-status:%d"
,
id
,
dstVer
,
TASK_SCHED_STATUS__INACTIVE
);
}
tqDebug
(
"s-task:%s wal reader start scan WAL verRange:%"
PRId64
"-%"
PRId64
", set sched-status:%d"
,
id
,
dstVer
,
pTask
->
dataRange
.
range
.
maxVer
,
TASK_SCHED_STATUS__INACTIVE
);
atomic_store_8
(
&
pTask
->
status
.
schedStatus
,
TASK_SCHED_STATUS__INACTIVE
);
// set the fill-history task to be normal
if
(
pTask
->
info
.
fillHistory
==
1
)
{
streamSetStatusNormal
(
pTask
);
}
// 4. 1) transfer the ownership of executor state, 2) update the scan data range for source task.
// 5. resume the related stream task.
streamMetaReleaseTask
(
pMeta
,
pTask
);
...
...
@@ -1304,7 +1310,7 @@ int32_t tqProcessTaskScanHistory(STQ* pTq, SRpcMsg* pMsg) {
if
(
pTask
->
historyTaskId
.
taskId
==
0
)
{
*
pWindow
=
(
STimeWindow
){
INT64_MIN
,
INT64_MAX
};
tqDebug
(
"s-task:%s scanhistory in stream time window completed, no related fill-history task, reset the time "
"s-task:%s scan
-
history in stream time window completed, no related fill-history task, reset the time "
"window:%"
PRId64
" - %"
PRId64
,
id
,
pWindow
->
skey
,
pWindow
->
ekey
);
qResetStreamInfoTimeWindow
(
pTask
->
exec
.
pExecutor
);
...
...
@@ -1500,7 +1506,7 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) {
if
(
pTask
!=
NULL
)
{
// even in halt status, the data in inputQ must be processed
int8_t
st
=
pTask
->
status
.
taskStatus
;
if
(
st
==
TASK_STATUS__NORMAL
||
st
==
TASK_STATUS__SCAN_HISTORY
||
st
==
TASK_STATUS__SCAN_HISTORY_WAL
)
{
if
(
st
==
TASK_STATUS__NORMAL
||
st
==
TASK_STATUS__SCAN_HISTORY
/* || st == TASK_STATUS__SCAN_HISTORY_WAL*/
)
{
tqDebug
(
"vgId:%d s-task:%s start to process block from inputQ, last chk point:%"
PRId64
,
vgId
,
pTask
->
id
.
idStr
,
pTask
->
chkInfo
.
version
);
streamProcessRunReq
(
pTask
);
...
...
@@ -1637,7 +1643,7 @@ int32_t tqProcessTaskResumeImpl(STQ* pTq, SStreamTask* pTask, int64_t sversion,
vgId
,
pTask
->
id
.
idStr
,
pTask
->
chkInfo
.
currentVer
,
sversion
,
pTask
->
status
.
schedStatus
);
}
if
(
level
==
TASK_LEVEL__SOURCE
&&
pTask
->
info
.
fillHistory
)
{
if
(
level
==
TASK_LEVEL__SOURCE
&&
pTask
->
info
.
fillHistory
&&
pTask
->
status
.
taskStatus
==
TASK_STATUS__SCAN_HISTORY
)
{
streamStartRecoverTask
(
pTask
,
igUntreated
);
}
else
if
(
level
==
TASK_LEVEL__SOURCE
&&
(
taosQueueItemSize
(
pTask
->
inputQueue
->
queue
)
==
0
))
{
tqStartStreamTasks
(
pTq
);
...
...
source/dnode/vnode/src/tq/tqPush.c
浏览文件 @
efa99b87
...
...
@@ -78,12 +78,12 @@ int32_t tqRegisterPushHandle(STQ* pTq, void* handle, SRpcMsg* pMsg) {
memcpy
(
pHandle
->
msg
->
pCont
,
pMsg
->
pCont
,
pMsg
->
contLen
);
pHandle
->
msg
->
contLen
=
pMsg
->
contLen
;
int32_t
ret
=
taosHashPut
(
pTq
->
pPushMgr
,
pHandle
->
subKey
,
strlen
(
pHandle
->
subKey
),
&
pHandle
,
POINTER_BYTES
);
tq
Debug
(
"vgId:%d data is over, ret:%d, consumerId:0x%"
PRIx64
", register to pHandle:%p, pCont:%p, len:%d"
,
vgId
,
ret
,
tq
Info
(
"vgId:%d data is over, ret:%d, consumerId:0x%"
PRIx64
", register to pHandle:%p, pCont:%p, len:%d"
,
vgId
,
ret
,
pHandle
->
consumerId
,
pHandle
,
pHandle
->
msg
->
pCont
,
pHandle
->
msg
->
contLen
);
return
0
;
}
int
32_t
tqUnregisterPushHandle
(
STQ
*
pTq
,
void
*
handle
)
{
int
tqUnregisterPushHandle
(
STQ
*
pTq
,
void
*
handle
)
{
STqHandle
*
pHandle
=
(
STqHandle
*
)
handle
;
int32_t
vgId
=
TD_VID
(
pTq
->
pVnode
);
...
...
@@ -91,7 +91,7 @@ int32_t tqUnregisterPushHandle(STQ* pTq, void *handle) {
return
0
;
}
int32_t
ret
=
taosHashRemove
(
pTq
->
pPushMgr
,
pHandle
->
subKey
,
strlen
(
pHandle
->
subKey
));
tq
Debug
(
"vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%"
PRIx64
,
vgId
,
pHandle
,
ret
,
pHandle
->
consumerId
);
tq
Info
(
"vgId:%d remove pHandle:%p,ret:%d consumer Id:0x%"
PRIx64
,
vgId
,
pHandle
,
ret
,
pHandle
->
consumerId
);
if
(
pHandle
->
msg
!=
NULL
)
{
// tqPushDataRsp(pHandle, vgId);
...
...
source/dnode/vnode/src/tq/tqRestore.c
浏览文件 @
efa99b87
...
...
@@ -211,7 +211,7 @@ int32_t doSetOffsetForWalReader(SStreamTask *pTask, int32_t vgId) {
static
void
checkForFillHistoryVerRange
(
SStreamTask
*
pTask
,
int64_t
ver
)
{
if
((
pTask
->
info
.
fillHistory
==
1
)
&&
ver
>
pTask
->
dataRange
.
range
.
maxVer
)
{
qWarn
(
"s-task:%s fill-history scan WAL, currentVer:%"
PRId64
"reach the maximum ver:%"
PRId64
qWarn
(
"s-task:%s fill-history scan WAL, currentVer:%"
PRId64
"
reach the maximum ver:%"
PRId64
", not scan wal anymore, set the transfer state flag"
,
pTask
->
id
.
idStr
,
ver
,
pTask
->
dataRange
.
range
.
maxVer
);
pTask
->
status
.
transferState
=
true
;
...
...
@@ -251,19 +251,19 @@ int32_t createStreamTaskRunReq(SStreamMeta* pStreamMeta, bool* pScanIdle) {
int32_t
status
=
pTask
->
status
.
taskStatus
;
// non-source or fill-history tasks don't need to response the WAL scan action.
if
(
pTask
->
info
.
taskLevel
!=
TASK_LEVEL__SOURCE
)
{
if
(
(
pTask
->
info
.
taskLevel
!=
TASK_LEVEL__SOURCE
)
||
(
pTask
->
status
.
downstreamReady
==
0
)
)
{
streamMetaReleaseTask
(
pStreamMeta
,
pTask
);
continue
;
}
if
(
status
!=
TASK_STATUS__NORMAL
&&
status
!=
TASK_STATUS__SCAN_HISTORY_WAL
)
{
if
(
status
!=
TASK_STATUS__NORMAL
)
{
tqDebug
(
"s-task:%s not ready for new submit block from wal, status:%s"
,
pTask
->
id
.
idStr
,
streamGetTaskStatusStr
(
status
));
streamMetaReleaseTask
(
pStreamMeta
,
pTask
);
continue
;
}
if
((
pTask
->
info
.
fillHistory
==
1
)
&&
pTask
->
status
.
transferState
)
{
ASSERT
(
status
==
TASK_STATUS__
SCAN_HISTORY_W
AL
);
ASSERT
(
status
==
TASK_STATUS__
NORM
AL
);
// the maximum version of data in the WAL has reached already, the step2 is done
tqDebug
(
"s-task:%s fill-history reach the maximum ver:%"
PRId64
", not scan wal anymore"
,
pTask
->
id
.
idStr
,
pTask
->
dataRange
.
range
.
maxVer
);
...
...
source/dnode/vnode/src/tq/tqUtil.c
浏览文件 @
efa99b87
...
...
@@ -317,6 +317,7 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ
// the offset value can not be monotonious increase??
offset
=
reqOffset
;
}
else
{
uError
(
"req offset type is 0"
);
return
TSDB_CODE_TMQ_INVALID_MSG
;
}
...
...
source/dnode/vnode/src/tsdb/tsdbCache.c
浏览文件 @
efa99b87
...
...
@@ -1028,55 +1028,7 @@ int32_t tsdbCacheGetBatch(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCache
return
code
;
}
/*
int32_t tsdbCacheGet(STsdb *pTsdb, tb_uid_t uid, SArray *pLastArray, SCacheRowsReader *pr, int8_t ltype) {
int32_t code = 0;
SLRUCache *pCache = pTsdb->lruCache;
SArray *pCidList = pr->pCidList;
int num_keys = TARRAY_SIZE(pCidList);
for (int i = 0; i < num_keys; ++i) {
SLastCol *pLastCol = NULL;
int16_t cid = *(int16_t *)taosArrayGet(pCidList, i);
SLastKey *key = &(SLastKey){.ltype = ltype, .uid = uid, .cid = cid};
LRUHandle *h = taosLRUCacheLookup(pCache, key, ROCKS_KEY_LEN);
if (!h) {
taosThreadMutexLock(&pTsdb->lruMutex);
h = taosLRUCacheLookup(pCache, key, ROCKS_KEY_LEN);
if (!h) {
pLastCol = tsdbCacheLoadCol(pTsdb, pr, pr->pSlotIds[i], uid, cid, ltype);
size_t charge = sizeof(*pLastCol);
if (IS_VAR_DATA_TYPE(pLastCol->colVal.type)) {
charge += pLastCol->colVal.value.nData;
}
LRUStatus status = taosLRUCacheInsert(pCache, key, ROCKS_KEY_LEN, pLastCol, charge, tsdbCacheDeleter, &h,
TAOS_LRU_PRIORITY_LOW, &pTsdb->flushState);
if (status != TAOS_LRU_STATUS_OK) {
code = -1;
}
}
taosThreadMutexUnlock(&pTsdb->lruMutex);
}
pLastCol = (SLastCol *)taosLRUCacheValue(pCache, h);
SLastCol lastCol = *pLastCol;
reallocVarData(&lastCol.colVal);
if (h) {
taosLRUCacheRelease(pCache, h, false);
}
taosArrayPush(pLastArray, &lastCol);
}
return code;
}
*/
int32_t
tsdbCacheDel
(
STsdb
*
pTsdb
,
tb_uid_t
suid
,
tb_uid_t
uid
,
TSKEY
sKey
,
TSKEY
eKey
)
{
int32_t
code
=
0
;
// fetch schema
...
...
@@ -1108,6 +1060,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
char
**
values_list
=
taosMemoryCalloc
(
num_keys
*
2
,
sizeof
(
char
*
));
size_t
*
values_list_sizes
=
taosMemoryCalloc
(
num_keys
*
2
,
sizeof
(
size_t
));
char
**
errs
=
taosMemoryCalloc
(
num_keys
*
2
,
sizeof
(
char
*
));
taosThreadMutexLock
(
&
pTsdb
->
lruMutex
);
taosThreadMutexLock
(
&
pTsdb
->
rCache
.
rMutex
);
rocksMayWrite
(
pTsdb
,
true
,
false
,
false
);
rocksdb_multi_get
(
pTsdb
->
rCache
.
db
,
pTsdb
->
rCache
.
readoptions
,
num_keys
*
2
,
(
const
char
*
const
*
)
keys_list
,
...
...
@@ -1137,7 +1090,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
rocksdb_free
(
values_list
[
i
]);
rocksdb_free
(
values_list
[
i
+
num_keys
]);
taosThreadMutexLock
(
&
pTsdb
->
lruMutex
);
//
taosThreadMutexLock(&pTsdb->lruMutex);
LRUHandle
*
h
=
taosLRUCacheLookup
(
pTsdb
->
lruCache
,
keys_list
[
i
],
klen
);
if
(
h
)
{
...
...
@@ -1159,7 +1112,7 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
}
taosLRUCacheErase
(
pTsdb
->
lruCache
,
keys_list
[
num_keys
+
i
],
klen
);
taosThreadMutexUnlock
(
&
pTsdb
->
lruMutex
);
//
taosThreadMutexUnlock(&pTsdb->lruMutex);
}
for
(
int
i
=
0
;
i
<
num_keys
;
++
i
)
{
taosMemoryFree
(
keys_list
[
i
]);
...
...
@@ -1171,6 +1124,8 @@ int32_t tsdbCacheDel(STsdb *pTsdb, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKE
rocksMayWrite
(
pTsdb
,
true
,
false
,
true
);
taosThreadMutexUnlock
(
&
pTsdb
->
lruMutex
);
_exit:
taosMemoryFree
(
pTSchema
);
...
...
@@ -1311,62 +1266,7 @@ int32_t tsdbCacheDeleteLast(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) {
return
code
;
}
/*
int32_t tsdbCacheDelete(SLRUCache *pCache, tb_uid_t uid, TSKEY eKey) {
int32_t code = 0;
char key[32] = {0};
int keyLen = 0;
// getTableCacheKey(uid, "lr", key, &keyLen);
getTableCacheKey(uid, 0, key, &keyLen);
LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen);
if (h) {
SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h);
bool invalidate = false;
int16_t nCol = taosArrayGetSize(pLast);
for (int16_t iCol = 0; iCol < nCol; ++iCol) {
SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol);
if (eKey >= tTsVal->ts) {
invalidate = true;
break;
}
}
if (invalidate) {
taosLRUCacheRelease(pCache, h, true);
} else {
taosLRUCacheRelease(pCache, h, false);
}
}
// getTableCacheKey(uid, "l", key, &keyLen);
getTableCacheKey(uid, 1, key, &keyLen);
h = taosLRUCacheLookup(pCache, key, keyLen);
if (h) {
SArray *pLast = (SArray *)taosLRUCacheValue(pCache, h);
bool invalidate = false;
int16_t nCol = taosArrayGetSize(pLast);
for (int16_t iCol = 0; iCol < nCol; ++iCol) {
SLastCol *tTsVal = (SLastCol *)taosArrayGet(pLast, iCol);
if (eKey >= tTsVal->ts) {
invalidate = true;
break;
}
}
if (invalidate) {
taosLRUCacheRelease(pCache, h, true);
} else {
taosLRUCacheRelease(pCache, h, false);
}
// void taosLRUCacheErase(SLRUCache * cache, const void *key, size_t keyLen);
}
return code;
}
*/
int32_t
tsdbCacheInsertLastrow
(
SLRUCache
*
pCache
,
STsdb
*
pTsdb
,
tb_uid_t
uid
,
TSDBROW
*
row
,
bool
dup
)
{
int32_t
code
=
0
;
STSRow
*
cacheRow
=
NULL
;
...
...
@@ -1767,6 +1667,10 @@ static int32_t loadTombFromBlk(const TTombBlkArray *pTombBlkArray, SCacheRowsRea
}
if
(
record
.
version
<=
pReader
->
info
.
verRange
.
maxVer
)
{
/*
tsdbError("tomb xx load/cache: vgId:%d fid:%d commit %" PRId64 "~%" PRId64 "~%" PRId64 " tomb records",
TD_VID(pReader->pTsdb->pVnode), pReader->pCurFileSet->fid, record.skey, record.ekey, uid);
*/
SDelData
delData
=
{.
version
=
record
.
version
,
.
sKey
=
record
.
skey
,
.
eKey
=
record
.
ekey
};
taosArrayPush
(
pInfo
->
pTombData
,
&
delData
);
}
...
...
@@ -1977,9 +1881,9 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
goto
_err
;
}
loadDataTomb
(
state
->
pr
,
state
->
pr
->
pFileReader
);
state
->
pr
->
pCurFileSet
=
state
->
pFileSet
;
loadDataTomb
(
state
->
pr
,
state
->
pr
->
pFileReader
);
}
if
(
!
state
->
pIndexList
)
{
...
...
@@ -2017,6 +1921,10 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie
state
->
iBrinIndex
=
indexSize
;
}
if
(
state
->
pFileSet
!=
state
->
pr
->
pCurFileSet
)
{
state
->
pr
->
pCurFileSet
=
state
->
pFileSet
;
}
code
=
lastIterOpen
(
&
state
->
lastIter
,
state
->
pFileSet
,
state
->
pTsdb
,
state
->
pTSchema
,
state
->
suid
,
state
->
uid
,
state
->
pr
,
state
->
lastTs
,
aCols
,
nCols
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
...
...
source/dnode/vnode/src/vnd/vnodeOpen.c
浏览文件 @
efa99b87
...
...
@@ -476,8 +476,8 @@ void vnodeClose(SVnode *pVnode) {
tsem_wait
(
&
pVnode
->
canCommit
);
vnodeSyncClose
(
pVnode
);
vnodeQueryClose
(
pVnode
);
walClose
(
pVnode
->
pWal
);
tqClose
(
pVnode
->
pTq
);
walClose
(
pVnode
->
pWal
);
if
(
pVnode
->
pTsdb
)
tsdbClose
(
&
pVnode
->
pTsdb
);
smaClose
(
pVnode
->
pSma
);
if
(
pVnode
->
pMeta
)
metaClose
(
&
pVnode
->
pMeta
);
...
...
source/libs/executor/src/executor.c
浏览文件 @
efa99b87
...
...
@@ -341,6 +341,7 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, SReadHandle* readers, int32_t v
return
NULL
;
}
qResetStreamInfoTimeWindow
(
pTaskInfo
);
return
pTaskInfo
;
}
...
...
source/libs/executor/src/scanoperator.c
浏览文件 @
efa99b87
...
...
@@ -1550,10 +1550,86 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock
}
}
static
int32_t
setBlockIntoRes
(
SStreamScanInfo
*
pInfo
,
const
SSDataBlock
*
pBlock
,
bool
filter
)
{
static
void
doBlockDataWindowFilter
(
SSDataBlock
*
pBlock
,
int32_t
tsIndex
,
STimeWindow
*
pWindow
,
const
char
*
id
)
{
if
(
pWindow
->
skey
!=
INT64_MIN
||
pWindow
->
ekey
!=
INT64_MAX
)
{
bool
*
p
=
taosMemoryCalloc
(
pBlock
->
info
.
rows
,
sizeof
(
bool
));
bool
hasUnqualified
=
false
;
SColumnInfoData
*
pCol
=
taosArrayGet
(
pBlock
->
pDataBlock
,
tsIndex
);
if
(
pWindow
->
skey
!=
INT64_MIN
)
{
qDebug
(
"%s filter for additional history window, skey:%"
PRId64
,
id
,
pWindow
->
skey
);
ASSERT
(
pCol
->
pData
!=
NULL
);
for
(
int32_t
i
=
0
;
i
<
pBlock
->
info
.
rows
;
++
i
)
{
int64_t
*
ts
=
(
int64_t
*
)
colDataGetData
(
pCol
,
i
);
p
[
i
]
=
(
*
ts
>=
pWindow
->
skey
);
if
(
!
p
[
i
])
{
hasUnqualified
=
true
;
}
}
}
else
if
(
pWindow
->
ekey
!=
INT64_MAX
)
{
qDebug
(
"%s filter for additional history window, ekey:%"
PRId64
,
id
,
pWindow
->
ekey
);
for
(
int32_t
i
=
0
;
i
<
pBlock
->
info
.
rows
;
++
i
)
{
int64_t
*
ts
=
(
int64_t
*
)
colDataGetData
(
pCol
,
i
);
p
[
i
]
=
(
*
ts
<=
pWindow
->
ekey
);
if
(
!
p
[
i
])
{
hasUnqualified
=
true
;
}
}
}
if
(
hasUnqualified
)
{
trimDataBlock
(
pBlock
,
pBlock
->
info
.
rows
,
p
);
}
taosMemoryFree
(
p
);
}
}
// re-build the delete block, ONLY according to the split timestamp
static
void
rebuildDeleteBlockData
(
SSDataBlock
*
pBlock
,
int64_t
skey
,
const
char
*
id
)
{
if
(
skey
==
INT64_MIN
)
{
return
;
}
int32_t
numOfRows
=
pBlock
->
info
.
rows
;
bool
*
p
=
taosMemoryCalloc
(
numOfRows
,
sizeof
(
bool
));
bool
hasUnqualified
=
false
;
SColumnInfoData
*
pSrcStartCol
=
taosArrayGet
(
pBlock
->
pDataBlock
,
START_TS_COLUMN_INDEX
);
uint64_t
*
tsStartCol
=
(
uint64_t
*
)
pSrcStartCol
->
pData
;
SColumnInfoData
*
pSrcEndCol
=
taosArrayGet
(
pBlock
->
pDataBlock
,
END_TS_COLUMN_INDEX
);
uint64_t
*
tsEndCol
=
(
uint64_t
*
)
pSrcEndCol
->
pData
;
for
(
int32_t
i
=
0
;
i
<
numOfRows
;
i
++
)
{
if
(
tsStartCol
[
i
]
<
skey
)
{
tsStartCol
[
i
]
=
skey
;
}
if
(
tsEndCol
[
i
]
>=
skey
)
{
p
[
i
]
=
true
;
}
else
{
// this row should be removed, since it is not in this query time window, which is [skey, INT64_MAX]
hasUnqualified
=
true
;
}
}
if
(
hasUnqualified
)
{
trimDataBlock
(
pBlock
,
pBlock
->
info
.
rows
,
p
);
}
qDebug
(
"%s re-build delete datablock, start key revised to:%"
PRId64
", rows:%"
PRId64
,
id
,
skey
,
pBlock
->
info
.
rows
);
taosMemoryFree
(
p
);
}
static
int32_t
setBlockIntoRes
(
SStreamScanInfo
*
pInfo
,
const
SSDataBlock
*
pBlock
,
STimeWindow
*
pTimeWindow
,
bool
filter
)
{
SDataBlockInfo
*
pBlockInfo
=
&
pInfo
->
pRes
->
info
;
SOperatorInfo
*
pOperator
=
pInfo
->
pStreamScanOp
;
SExecTaskInfo
*
pTaskInfo
=
pOperator
->
pTaskInfo
;
const
char
*
id
=
GET_TASKID
(
pTaskInfo
);
blockDataEnsureCapacity
(
pInfo
->
pRes
,
pBlock
->
info
.
rows
);
...
...
@@ -1593,7 +1669,7 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
// currently only the tbname pseudo column
if
(
pInfo
->
numOfPseudoExpr
>
0
)
{
int32_t
code
=
addTagPseudoColumnData
(
&
pInfo
->
readHandle
,
pInfo
->
pPseudoExpr
,
pInfo
->
numOfPseudoExpr
,
pInfo
->
pRes
,
pBlockInfo
->
rows
,
GET_TASKID
(
pTaskInfo
)
,
&
pTableScanInfo
->
base
.
metaCache
);
pBlockInfo
->
rows
,
id
,
&
pTableScanInfo
->
base
.
metaCache
);
// ignore the table not exists error, since this table may have been dropped during the scan procedure.
if
(
code
!=
TSDB_CODE_SUCCESS
&&
code
!=
TSDB_CODE_PAR_TABLE_NOT_EXIST
)
{
blockDataFreeRes
((
SSDataBlock
*
)
pBlock
);
...
...
@@ -1608,8 +1684,14 @@ static int32_t setBlockIntoRes(SStreamScanInfo* pInfo, const SSDataBlock* pBlock
doFilter
(
pInfo
->
pRes
,
pOperator
->
exprSupp
.
pFilterInfo
,
NULL
);
}
// filter the block extracted from WAL files, according to the time window apply additional time window filter
doBlockDataWindowFilter
(
pInfo
->
pRes
,
pInfo
->
primaryTsIndex
,
pTimeWindow
,
id
);
pInfo
->
pRes
->
info
.
dataLoad
=
1
;
blockDataUpdateTsWindow
(
pInfo
->
pRes
,
pInfo
->
primaryTsIndex
);
if
(
pInfo
->
pRes
->
info
.
rows
==
0
)
{
return
0
;
}
calBlockTbName
(
pInfo
,
pInfo
->
pRes
);
return
0
;
...
...
@@ -1666,7 +1748,8 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) {
qDebug
(
"doQueueScan get data from log %"
PRId64
" rows, version:%"
PRId64
,
pRes
->
info
.
rows
,
pTaskInfo
->
streamInfo
.
currentOffset
.
version
);
blockDataCleanup
(
pInfo
->
pRes
);
setBlockIntoRes
(
pInfo
,
pRes
,
true
);
STimeWindow
defaultWindow
=
{.
skey
=
INT64_MIN
,
.
ekey
=
INT64_MAX
};
setBlockIntoRes
(
pInfo
,
pRes
,
&
defaultWindow
,
true
);
if
(
pInfo
->
pRes
->
info
.
rows
>
0
)
{
return
pInfo
->
pRes
;
}
...
...
@@ -1775,80 +1858,6 @@ void streamScanOperatorDecode(void* pBuff, int32_t len, SStreamScanInfo* pInfo)
}
}
static
void
doBlockDataWindowFilter
(
SSDataBlock
*
pBlock
,
int32_t
tsIndex
,
STimeWindow
*
pWindow
,
const
char
*
id
)
{
if
(
pWindow
->
skey
!=
INT64_MIN
||
pWindow
->
ekey
!=
INT64_MAX
)
{
bool
*
p
=
taosMemoryCalloc
(
pBlock
->
info
.
rows
,
sizeof
(
bool
));
bool
hasUnqualified
=
false
;
SColumnInfoData
*
pCol
=
taosArrayGet
(
pBlock
->
pDataBlock
,
tsIndex
);
if
(
pWindow
->
skey
!=
INT64_MIN
)
{
qDebug
(
"%s filter for additional history window, skey:%"
PRId64
,
id
,
pWindow
->
skey
);
for
(
int32_t
i
=
0
;
i
<
pBlock
->
info
.
rows
;
++
i
)
{
int64_t
*
ts
=
(
int64_t
*
)
colDataGetData
(
pCol
,
i
);
p
[
i
]
=
(
*
ts
>=
pWindow
->
skey
);
if
(
!
p
[
i
])
{
hasUnqualified
=
true
;
}
}
}
else
if
(
pWindow
->
ekey
!=
INT64_MAX
)
{
qDebug
(
"%s filter for additional history window, ekey:%"
PRId64
,
id
,
pWindow
->
ekey
);
for
(
int32_t
i
=
0
;
i
<
pBlock
->
info
.
rows
;
++
i
)
{
int64_t
*
ts
=
(
int64_t
*
)
colDataGetData
(
pCol
,
i
);
p
[
i
]
=
(
*
ts
<=
pWindow
->
ekey
);
if
(
!
p
[
i
])
{
hasUnqualified
=
true
;
}
}
}
if
(
hasUnqualified
)
{
trimDataBlock
(
pBlock
,
pBlock
->
info
.
rows
,
p
);
}
taosMemoryFree
(
p
);
}
}
// re-build the delete block, ONLY according to the split timestamp
static
void
rebuildDeleteBlockData
(
SSDataBlock
*
pBlock
,
int64_t
skey
,
const
char
*
id
)
{
if
(
skey
==
INT64_MIN
)
{
return
;
}
int32_t
numOfRows
=
pBlock
->
info
.
rows
;
bool
*
p
=
taosMemoryCalloc
(
numOfRows
,
sizeof
(
bool
));
bool
hasUnqualified
=
false
;
SColumnInfoData
*
pSrcStartCol
=
taosArrayGet
(
pBlock
->
pDataBlock
,
START_TS_COLUMN_INDEX
);
uint64_t
*
tsStartCol
=
(
uint64_t
*
)
pSrcStartCol
->
pData
;
SColumnInfoData
*
pSrcEndCol
=
taosArrayGet
(
pBlock
->
pDataBlock
,
END_TS_COLUMN_INDEX
);
uint64_t
*
tsEndCol
=
(
uint64_t
*
)
pSrcEndCol
->
pData
;
for
(
int32_t
i
=
0
;
i
<
numOfRows
;
i
++
)
{
if
(
tsStartCol
[
i
]
<
skey
)
{
tsStartCol
[
i
]
=
skey
;
}
if
(
tsEndCol
[
i
]
>=
skey
)
{
p
[
i
]
=
true
;
}
else
{
// this row should be removed, since it is not in this query time window, which is [skey, INT64_MAX]
hasUnqualified
=
true
;
}
}
if
(
hasUnqualified
)
{
trimDataBlock
(
pBlock
,
pBlock
->
info
.
rows
,
p
);
}
qDebug
(
"%s re-build delete datablock, start key revised to:%"
PRId64
", rows:%"
PRId64
,
id
,
skey
,
pBlock
->
info
.
rows
);
taosMemoryFree
(
p
);
}
static
SSDataBlock
*
doStreamScan
(
SOperatorInfo
*
pOperator
)
{
// NOTE: this operator does never check if current status is done or not
SExecTaskInfo
*
pTaskInfo
=
pOperator
->
pTaskInfo
;
...
...
@@ -2121,8 +2130,7 @@ FETCH_NEXT_BLOCK:
return
pInfo
->
pUpdateRes
;
}
SSDataBlock
*
pBlock
=
pInfo
->
pRes
;
SDataBlockInfo
*
pBlockInfo
=
&
pBlock
->
info
;
SDataBlockInfo
*
pBlockInfo
=
&
pInfo
->
pRes
->
info
;
int32_t
totalBlocks
=
taosArrayGetSize
(
pInfo
->
pBlockLists
);
NEXT_SUBMIT_BLK:
...
...
@@ -2146,21 +2154,23 @@ FETCH_NEXT_BLOCK:
}
}
blockDataCleanup
(
p
Block
);
blockDataCleanup
(
p
Info
->
pRes
);
while
(
pAPI
->
tqReaderFn
.
tqNextBlockImpl
(
pInfo
->
tqReader
,
id
))
{
SSDataBlock
*
pRes
=
NULL
;
int32_t
code
=
pAPI
->
tqReaderFn
.
tqRetrieveBlock
(
pInfo
->
tqReader
,
&
pRes
,
id
);
qDebug
(
"retrieve data from submit completed code:%s, rows:%"
PRId64
" %s"
,
tstrerror
(
code
),
pRes
->
info
.
rows
,
id
);
qDebug
(
"retrieve data from submit completed code:%s rows:%"
PRId64
" %s"
,
tstrerror
(
code
),
pRes
->
info
.
rows
,
id
);
if
(
code
!=
TSDB_CODE_SUCCESS
||
pRes
->
info
.
rows
==
0
)
{
qDebug
(
"retrieve data failed, try next block in submit block, %s"
,
id
);
continue
;
}
setBlockIntoRes
(
pInfo
,
pRes
,
false
);
setBlockIntoRes
(
pInfo
,
pRes
,
&
pStreamInfo
->
fillHistoryWindow
,
false
);
if
(
pInfo
->
pRes
->
info
.
rows
==
0
)
{
continue
;
}
if
(
pInfo
->
pCreateTbRes
->
info
.
rows
>
0
)
{
pInfo
->
scanMode
=
STREAM_SCAN_FROM_RES
;
...
...
@@ -2168,13 +2178,8 @@ FETCH_NEXT_BLOCK:
return
pInfo
->
pCreateTbRes
;
}
// apply additional time window filter
doBlockDataWindowFilter
(
pBlock
,
pInfo
->
primaryTsIndex
,
&
pStreamInfo
->
fillHistoryWindow
,
id
);
pBlock
->
info
.
dataLoad
=
1
;
blockDataUpdateTsWindow
(
pBlock
,
pInfo
->
primaryTsIndex
);
doCheckUpdate
(
pInfo
,
pBlockInfo
->
window
.
ekey
,
pBlock
);
doFilter
(
pBlock
,
pOperator
->
exprSupp
.
pFilterInfo
,
NULL
);
doCheckUpdate
(
pInfo
,
pBlockInfo
->
window
.
ekey
,
pInfo
->
pRes
);
doFilter
(
pInfo
->
pRes
,
pOperator
->
exprSupp
.
pFilterInfo
,
NULL
);
int64_t
numOfUpdateRes
=
pInfo
->
pUpdateDataRes
->
info
.
rows
;
qDebug
(
"%s %"
PRId64
" rows in datablock, update res:%"
PRId64
,
id
,
pBlockInfo
->
rows
,
numOfUpdateRes
);
...
...
@@ -2196,7 +2201,7 @@ FETCH_NEXT_BLOCK:
qDebug
(
"stream scan completed, and return source rows:%"
PRId64
", %s"
,
pBlockInfo
->
rows
,
id
);
if
(
pBlockInfo
->
rows
>
0
)
{
return
p
Block
;
return
p
Info
->
pRes
;
}
if
(
pInfo
->
pUpdateDataRes
->
info
.
rows
>
0
)
{
...
...
@@ -2587,7 +2592,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys
pInfo
->
igCheckUpdate
=
pTableScanNode
->
igCheckUpdate
;
pInfo
->
igExpired
=
pTableScanNode
->
igExpired
;
pInfo
->
twAggSup
.
maxTs
=
INT64_MIN
;
pInfo
->
pState
=
NULL
;
pInfo
->
pState
=
pTaskInfo
->
streamInfo
.
pState
;
pInfo
->
stateStore
=
pTaskInfo
->
storageAPI
.
stateStore
;
pInfo
->
readerFn
=
pTaskInfo
->
storageAPI
.
tqReaderFn
;
...
...
source/libs/executor/src/timewindowoperator.c
浏览文件 @
efa99b87
...
...
@@ -3736,7 +3736,6 @@ void streamSessionReloadState(SOperatorInfo* pOperator) {
setSessionOutputBuf
(
pAggSup
,
pSeKeyBuf
[
i
].
win
.
skey
,
pSeKeyBuf
[
i
].
win
.
ekey
,
pSeKeyBuf
[
i
].
groupId
,
&
winInfo
);
int32_t
winNum
=
compactSessionWindow
(
pOperator
,
&
winInfo
,
pInfo
->
pStUpdated
,
pInfo
->
pStDeleted
,
true
);
if
(
winNum
>
0
)
{
saveSessionOutputBuf
(
pAggSup
,
&
winInfo
);
if
(
pInfo
->
twAggSup
.
calTrigger
==
STREAM_TRIGGER_AT_ONCE
)
{
saveResult
(
winInfo
,
pInfo
->
pStUpdated
);
}
else
if
(
pInfo
->
twAggSup
.
calTrigger
==
STREAM_TRIGGER_WINDOW_CLOSE
)
{
...
...
@@ -3747,9 +3746,8 @@ void streamSessionReloadState(SOperatorInfo* pOperator) {
getSessionHashKey
(
&
winInfo
.
sessionWin
,
&
key
);
tSimpleHashPut
(
pAggSup
->
pResultRows
,
&
key
,
sizeof
(
SSessionKey
),
&
winInfo
,
sizeof
(
SResultWindowInfo
));
}
}
else
{
releaseOutputBuf
(
pAggSup
->
pState
,
NULL
,
(
SResultRow
*
)
winInfo
.
pOutputBuf
,
&
pAggSup
->
stateStore
);
}
saveSessionOutputBuf
(
pAggSup
,
&
winInfo
);
}
taosMemoryFree
(
pBuf
);
...
...
@@ -4398,7 +4396,6 @@ void streamStateReloadState(SOperatorInfo* pOperator) {
setStateOutputBuf
(
pAggSup
,
pSeKeyBuf
[
i
].
win
.
skey
,
pSeKeyBuf
[
i
].
groupId
,
NULL
,
&
curInfo
,
&
nextInfo
);
if
(
compareStateKey
(
curInfo
.
pStateKey
,
nextInfo
.
pStateKey
))
{
compactStateWindow
(
pOperator
,
&
curInfo
.
winInfo
,
&
nextInfo
.
winInfo
,
pInfo
->
pSeUpdated
,
pInfo
->
pSeUpdated
);
saveSessionOutputBuf
(
pAggSup
,
&
curInfo
.
winInfo
);
if
(
pInfo
->
twAggSup
.
calTrigger
==
STREAM_TRIGGER_AT_ONCE
)
{
saveResult
(
curInfo
.
winInfo
,
pInfo
->
pSeUpdated
);
}
else
if
(
pInfo
->
twAggSup
.
calTrigger
==
STREAM_TRIGGER_WINDOW_CLOSE
)
{
...
...
@@ -4409,14 +4406,12 @@ void streamStateReloadState(SOperatorInfo* pOperator) {
getSessionHashKey
(
&
curInfo
.
winInfo
.
sessionWin
,
&
key
);
tSimpleHashPut
(
pAggSup
->
pResultRows
,
&
key
,
sizeof
(
SSessionKey
),
&
curInfo
.
winInfo
,
sizeof
(
SResultWindowInfo
));
}
}
else
if
(
IS_VALID_SESSION_WIN
(
nextInfo
.
winInfo
))
{
releaseOutputBuf
(
pAggSup
->
pState
,
NULL
,
(
SResultRow
*
)
nextInfo
.
winInfo
.
pOutputBuf
,
&
pAggSup
->
pSessionAPI
->
stateStore
);
}
if
(
IS_VALID_SESSION_WIN
(
curInfo
.
winInfo
))
{
releaseOutputBuf
(
pAggSup
->
pState
,
NULL
,
(
SResultRow
*
)
curInfo
.
winInfo
.
pOutputBuf
,
&
pAggSup
->
pSessionAPI
->
stateStore
);
}
if
(
IS_VALID_SESSION_WIN
(
nextInfo
.
winInfo
))
{
releaseOutputBuf
(
pAggSup
->
pState
,
NULL
,
(
SResultRow
*
)
nextInfo
.
winInfo
.
pOutputBuf
,
&
pAggSup
->
pSessionAPI
->
stateStore
);
saveSessionOutputBuf
(
pAggSup
,
&
curInfo
.
winInfo
);
}
}
taosMemoryFree
(
pBuf
);
...
...
source/libs/parser/inc/parUtil.h
浏览文件 @
efa99b87
...
...
@@ -118,6 +118,12 @@ int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes);
void
destoryParseMetaCache
(
SParseMetaCache
*
pMetaCache
,
bool
request
);
SNode
*
createSelectStmtImpl
(
bool
isDistinct
,
SNodeList
*
pProjectionList
,
SNode
*
pTable
);
/**
* @brief return a - b with overflow check
* @retval val range between [INT64_MIN, INT64_MAX]
*/
int64_t
int64SafeSub
(
int64_t
a
,
int64_t
b
);
#ifdef __cplusplus
}
#endif
...
...
source/libs/parser/src/parTranslater.c
浏览文件 @
efa99b87
...
...
@@ -3296,9 +3296,11 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode*
if
(
NULL
==
pInterval
)
{
return
TSDB_CODE_SUCCESS
;
}
int64_t
timeRange
=
TABS
(
pFill
->
timeRange
.
skey
-
pFill
->
timeRange
.
ekey
);
int64_t
timeRange
=
0
;
int64_t
intervalRange
=
0
;
if
(
!
pCxt
->
createStream
)
{
int64_t
res
=
int64SafeSub
(
pFill
->
timeRange
.
skey
,
pFill
->
timeRange
.
ekey
);
timeRange
=
res
<
0
?
res
==
INT64_MIN
?
INT64_MAX
:
-
res
:
res
;
if
(
IS_CALENDAR_TIME_DURATION
(
pInterval
->
unit
))
{
int64_t
f
=
1
;
if
(
pInterval
->
unit
==
'n'
)
{
...
...
@@ -3310,10 +3312,10 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode*
}
else
{
intervalRange
=
pInterval
->
datum
.
i
;
}
if
((
timeRange
/
intervalRange
)
>=
MAX_INTERVAL_TIME_WINDOW
)
{
return
generateSyntaxErrMsg
(
&
pCxt
->
msgBuf
,
TSDB_CODE_PAR_INVALID_FILL_TIME_RANGE
);
}
}
return
TSDB_CODE_SUCCESS
;
}
...
...
source/libs/parser/src/parUtil.c
浏览文件 @
efa99b87
...
...
@@ -1142,3 +1142,18 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache, bool request) {
taosHashCleanup
(
pMetaCache
->
pTableIndex
);
taosHashCleanup
(
pMetaCache
->
pTableCfg
);
}
int64_t
int64SafeSub
(
int64_t
a
,
int64_t
b
)
{
int64_t
res
=
(
uint64_t
)
a
-
(
uint64_t
)
b
;
if
(
a
>=
0
&&
b
<
0
)
{
if
((
uint64_t
)
res
>
(
uint64_t
)
INT64_MAX
)
{
// overflow
res
=
INT64_MAX
;
}
}
else
if
(
a
<
0
&&
b
>
0
&&
res
>=
0
)
{
// underflow
res
=
INT64_MIN
;
}
return
res
;
}
source/libs/stream/src/stream.c
浏览文件 @
efa99b87
...
...
@@ -379,7 +379,7 @@ int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) {
return
-
1
;
}
qDebug
(
"s-task:%s
data block enqueue, current(blocks:%d, size:%.2fMiB)
"
,
pTask
->
id
.
idStr
,
total
,
size
);
qDebug
(
"s-task:%s
blockdata enqueue, total in queue:%d, size:%.2fMiB
"
,
pTask
->
id
.
idStr
,
total
,
size
);
int32_t
code
=
taosWriteQitem
(
pTask
->
inputQueue
->
queue
,
pItem
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
destroyStreamDataBlock
((
SStreamDataBlock
*
)
pItem
);
...
...
source/libs/stream/src/streamExec.c
浏览文件 @
efa99b87
...
...
@@ -172,6 +172,12 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) {
bool
finished
=
false
;
while
(
1
)
{
if
(
streamTaskShouldPause
(
&
pTask
->
status
))
{
double
el
=
(
taosGetTimestampMs
()
-
pTask
->
tsInfo
.
step1Start
)
/
1000
.
0
;
qDebug
(
"s-task:%s paused from the scan-history task, elapsed time:%.2fsec"
,
pTask
->
id
.
idStr
,
el
);
return
0
;
}
SArray
*
pRes
=
taosArrayInit
(
0
,
sizeof
(
SSDataBlock
));
if
(
pRes
==
NULL
)
{
terrno
=
TSDB_CODE_OUT_OF_MEMORY
;
...
...
@@ -404,6 +410,8 @@ static int32_t streamTransferStateToStreamTask(SStreamTask* pTask) {
streamTaskReleaseState
(
pTask
);
streamTaskReloadState
(
pStreamTask
);
// clear the link between fill-history task and stream task info
pStreamTask
->
historyTaskId
.
taskId
=
0
;
streamTaskResumeFromHalt
(
pStreamTask
);
qDebug
(
"s-task:%s fill-history task set status to be dropping, save the state into disk"
,
pTask
->
id
.
idStr
);
...
...
@@ -414,6 +422,7 @@ static int32_t streamTransferStateToStreamTask(SStreamTask* pTask) {
// save to disk
taosWLockLatch
(
&
pMeta
->
lock
);
streamMetaSaveTask
(
pMeta
,
pStreamTask
);
if
(
streamMetaCommit
(
pMeta
)
<
0
)
{
// persist to disk
...
...
@@ -615,7 +624,7 @@ int32_t streamTryExec(SStreamTask* pTask) {
// todo the task should be commit here
if
(
taosQueueEmpty
(
pTask
->
inputQueue
->
queue
))
{
// fill-history WAL scan has completed
if
(
pTask
->
status
.
taskStatus
==
TASK_STATUS__SCAN_HISTORY_WAL
&&
pTask
->
status
.
transferState
==
true
)
{
if
(
pTask
->
info
.
taskLevel
==
TASK_LEVEL__SOURCE
&&
pTask
->
status
.
transferState
==
true
)
{
streamTaskRecoverSetAllStepFinished
(
pTask
);
streamTaskEndScanWAL
(
pTask
);
}
else
{
...
...
source/libs/stream/src/streamRecover.c
浏览文件 @
efa99b87
...
...
@@ -85,6 +85,7 @@ int32_t streamTaskLaunchScanHistory(SStreamTask* pTask) {
if
(
pTask
->
info
.
fillHistory
)
{
streamSetParamForScanHistory
(
pTask
);
}
streamTaskEnablePause
(
pTask
);
streamTaskScanHistoryPrepare
(
pTask
);
}
else
if
(
pTask
->
info
.
taskLevel
==
TASK_LEVEL__SINK
)
{
qDebug
(
"s-task:%s sink task do nothing to handle scan-history"
,
pTask
->
id
.
idStr
);
...
...
@@ -839,7 +840,7 @@ void streamTaskPause(SStreamTask* pTask) {
return
;
}
while
(
!
pTask
->
status
.
pauseAllowed
||
(
pTask
->
status
.
taskStatus
==
TASK_STATUS__HALT
))
{
while
(
!
pTask
->
status
.
pauseAllowed
||
(
pTask
->
status
.
taskStatus
==
TASK_STATUS__HALT
))
{
status
=
pTask
->
status
.
taskStatus
;
if
(
status
==
TASK_STATUS__DROPPING
)
{
qDebug
(
"vgId:%d s-task:%s task already dropped, do nothing"
,
pMeta
->
vgId
,
pTask
->
id
.
idStr
);
...
...
@@ -856,8 +857,19 @@ void streamTaskPause(SStreamTask* pTask) {
taosMsleep
(
100
);
}
// todo: use the lock of the task.
taosWLockLatch
(
&
pMeta
->
lock
);
status
=
pTask
->
status
.
taskStatus
;
if
(
status
==
TASK_STATUS__DROPPING
||
status
==
TASK_STATUS__STOP
)
{
taosWUnLockLatch
(
&
pMeta
->
lock
);
qDebug
(
"vgId:%d s-task:%s task already dropped/stopped/paused, do nothing"
,
pMeta
->
vgId
,
pTask
->
id
.
idStr
);
return
;
}
atomic_store_8
(
&
pTask
->
status
.
keepTaskStatus
,
pTask
->
status
.
taskStatus
);
atomic_store_8
(
&
pTask
->
status
.
taskStatus
,
TASK_STATUS__PAUSE
);
taosWUnLockLatch
(
&
pMeta
->
lock
);
int64_t
el
=
taosGetTimestampMs
()
-
st
;
qDebug
(
"vgId:%d s-task:%s set pause flag, prev:%s, elapsed time:%dms"
,
pMeta
->
vgId
,
pTask
->
id
.
idStr
,
...
...
source/util/CMakeLists.txt
浏览文件 @
efa99b87
...
...
@@ -5,6 +5,13 @@ if (DEFINED GRANT_CFG_INCLUDE_DIR)
add_definitions
(
-DGRANTS_CFG
)
endif
()
IF
(
${
ASSERT_NOT_CORE
}
)
ADD_DEFINITIONS
(
-DASSERT_NOT_CORE
)
MESSAGE
(
STATUS
"disable assert core"
)
ELSE
()
MESSAGE
(
STATUS
"enable assert core"
)
ENDIF
(
${
ASSERT_NOT_CORE
}
)
target_include_directories
(
util
PUBLIC
"
${
TD_SOURCE_DIR
}
/include/util"
...
...
source/util/src/tlog.c
浏览文件 @
efa99b87
...
...
@@ -76,7 +76,11 @@ static int32_t tsDaylightActive; /* Currently in daylight saving time. */
bool
tsLogEmbedded
=
0
;
bool
tsAsyncLog
=
true
;
#ifdef ASSERT_NOT_CORE
bool
tsAssert
=
false
;
#else
bool
tsAssert
=
true
;
#endif
int32_t
tsNumOfLogLines
=
10000000
;
int32_t
tsLogKeepDays
=
0
;
LogFp
tsLogFp
=
NULL
;
...
...
tests/parallel_test/cases.task
浏览文件 @
efa99b87
...
...
@@ -105,7 +105,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py
#
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py
,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDnodeRestart.py
...
...
@@ -450,7 +450,7 @@
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3
#,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -N 6 -M 3 -n 3
,,n,system-test,python3 ./test.py -f 6-cluster/manually-test/6dnode3mnodeInsertLessDataAlterRep3to1to3.py -N 6 -M 3
,,n,system-test,python ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1
#
,,n,system-test,python ./test.py -f 6-cluster/5dnode3mnodeRoll.py -N 3 -C 1
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6
,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 -n 3
#,,y,system-test,./pytest.sh python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5
...
...
tests/system-test/7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py
浏览文件 @
efa99b87
...
...
@@ -222,9 +222,9 @@ class TDTestCase:
actConsumeTotalRows
=
resultList
[
0
]
if
not
(
actConsumeTotalRows
>
0
and
actConsumeTotalRows
<
totalRowsInserted
):
if
not
(
actConsumeTotalRows
>
=
0
and
actConsumeTotalRows
<=
totalRowsInserted
):
tdLog
.
info
(
"act consume rows: %d"
%
(
actConsumeTotalRows
))
tdLog
.
info
(
"and second consume rows should be between
0 and %d
"
%
(
totalRowsInserted
))
tdLog
.
info
(
"and second consume rows should be between
[0 and %d]
"
%
(
totalRowsInserted
))
tdLog
.
exit
(
"%d tmq consume rows error!"
%
consumerId
)
time
.
sleep
(
10
)
...
...
tests/system-test/eco-system/manager/cmul.py
0 → 100644
浏览文件 @
efa99b87
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
#
# The option for wal_retetion_period and wal_retention_size is work well
#
import
taos
from
taos.tmq
import
Consumer
import
os
import
sys
import
threading
import
json
import
time
import
random
from
datetime
import
date
from
datetime
import
datetime
from
datetime
import
timedelta
from
os
import
path
topicName
=
"topic"
topicNum
=
100
# consume topic
def
consume_topic
(
topic_name
,
group
,
consume_cnt
,
index
,
wait
):
consumer
=
Consumer
(
{
"group.id"
:
group
,
"td.connect.user"
:
"root"
,
"td.connect.pass"
:
"taosdata"
,
"enable.auto.commit"
:
"true"
,
}
)
print
(
f
"start consumer topic:
{
topic_name
}
group=
{
group
}
index=
{
index
}
..."
)
consumer
.
subscribe
([
topic_name
])
cnt
=
0
try
:
while
True
and
cnt
<
consume_cnt
:
res
=
consumer
.
poll
(
1
)
if
not
res
:
if
wait
:
continue
else
:
break
err
=
res
.
error
()
if
err
is
not
None
:
raise
err
val
=
res
.
value
()
cnt
+=
1
print
(
f
" consume
{
cnt
}
"
)
for
block
in
val
:
datas
=
block
.
fetchall
()
data
=
datas
[
0
][:
50
]
print
(
f
"
{
topic_name
}
_
{
group
}
_
{
index
}
{
cnt
}
{
data
}
"
)
finally
:
consumer
.
unsubscribe
()
consumer
.
close
()
def
consumerThread
(
index
):
global
topicName
,
topicNum
print
(
f
' thread
{
index
}
start...'
)
while
True
:
idx
=
random
.
randint
(
0
,
topicNum
-
1
)
name
=
f
"
{
topicName
}{
idx
}
"
group
=
f
"group_
{
index
}
_
{
idx
}
"
consume_topic
(
name
,
group
,
100
,
index
,
True
)
if
__name__
==
"__main__"
:
print
(
sys
.
argv
)
threadCnt
=
10
if
len
(
sys
.
argv
)
==
1
:
threadCnt
=
int
(
sys
.
argv
[
1
])
threads
=
[]
print
(
f
'consumer with
{
threadCnt
}
threads...'
)
for
i
in
range
(
threadCnt
):
x
=
threading
.
Thread
(
target
=
consumerThread
,
args
=
(
i
,))
x
.
start
()
threads
.
append
(
x
)
# wait
for
i
,
thread
in
enumerate
(
threads
):
thread
.
join
()
print
(
f
'join thread
{
i
}
end.'
)
tests/system-test/eco-system/manager/mul.py
0 → 100644
浏览文件 @
efa99b87
###################################################################
# Copyright (c) 2016 by TAOS Technologies, Inc.
# All rights reserved.
#
# This file is proprietary and confidential to TAOS Technologies.
# No part of this file may be reproduced, stored, transmitted,
# disclosed or used in any form or by any means other than as
# expressly provided by the written permission from Jianhui Tao
#
###################################################################
# -*- coding: utf-8 -*-
import
os
import
sys
import
random
import
time
from
util.log
import
*
from
util.cases
import
*
from
util.sql
import
*
from
util.common
import
*
from
util.sqlset
import
*
class
TDTestCase
:
def
init
(
self
,
conn
,
logSql
,
replicaVar
=
1
):
self
.
replicaVar
=
int
(
replicaVar
)
tdLog
.
debug
(
"start to execute %s"
%
__file__
)
tdSql
.
init
(
conn
.
cursor
())
self
.
setsql
=
TDSetSql
()
# prepareEnv
def
prepareEnv
(
self
):
self
.
dbName
=
"mullevel"
self
.
stbName
=
"meters"
self
.
topicName
=
"topic"
self
.
topicNum
=
100
self
.
loop
=
50000
sql
=
f
"use
{
self
.
dbName
}
"
tdSql
.
execute
(
sql
)
# generate topic sql
self
.
sqls
=
[
f
"select * from
{
self
.
stbName
}
"
,
f
"select * from
{
self
.
stbName
}
where ui < 200"
,
f
"select * from
{
self
.
stbName
}
where fc > 20.1"
,
f
"select * from
{
self
.
stbName
}
where nch like '%%a%%'"
,
f
"select * from
{
self
.
stbName
}
where fc > 20.1"
,
f
"select lower(bin) from
{
self
.
stbName
}
where length(bin) < 10;"
,
f
"select upper(bin) from
{
self
.
stbName
}
where length(nch) > 10;"
,
f
"select upper(bin) from
{
self
.
stbName
}
where ti > 10 or ic < 40;"
,
f
"select * from
{
self
.
stbName
}
where ic < 100 "
]
# prepareEnv
def
createTopics
(
self
):
for
i
in
range
(
self
.
topicNum
):
topicName
=
f
"
{
self
.
topicName
}{
i
}
"
sql
=
random
.
choice
(
self
.
sqls
)
createSql
=
f
"create topic if not exists
{
topicName
}
as
{
sql
}
"
try
:
tdSql
.
execute
(
createSql
,
3
,
True
)
except
:
tdLog
.
info
(
f
" create topic
{
topicName
}
failed."
)
# random del topic
def
managerTopics
(
self
):
for
i
in
range
(
self
.
loop
):
tdLog
.
info
(
f
"start modify loop=
{
i
}
"
)
idx
=
random
.
randint
(
0
,
self
.
topicNum
-
1
)
# delete
topicName
=
f
"
{
self
.
topicName
}{
idx
}
"
sql
=
f
"drop topic if exist
{
topicName
}
"
try
:
tdSql
.
execute
(
sql
,
3
,
True
)
except
:
tdLog
.
info
(
f
" drop topic
{
topicName
}
failed."
)
# create topic
sql
=
random
.
choice
(
self
.
sqls
)
createSql
=
f
"create topic if not exists
{
topicName
}
as
{
sql
}
"
try
:
tdSql
.
execute
(
createSql
,
3
,
True
)
except
:
tdLog
.
info
(
f
" create topic
{
topicName
}
failed."
)
seconds
=
[
0.1
,
0.5
,
3
,
2.5
,
1.5
,
0.4
,
5.2
,
2.6
,
0.4
,
0.2
]
time
.
sleep
(
random
.
choice
(
seconds
))
# run
def
run
(
self
):
# prepare env
self
.
prepareEnv
()
# create topic
self
.
createTopics
()
# modify topic
self
.
managerTopics
()
def
stop
(
self
):
tdSql
.
close
()
tdLog
.
success
(
"%s successfully executed"
%
__file__
)
tdCases
.
addWindows
(
__file__
,
TDTestCase
())
tdCases
.
addLinux
(
__file__
,
TDTestCase
())
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录