Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
50487255
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1181
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
50487255
编写于
5月 26, 2023
作者:
wmmhello
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
fix:put poll to push manager if wal not exist when offset is latest
上级
4682e9c9
变更
3
显示空白变更内容
内联
并排
Showing
3 changed file
with
16 addition
and
49 deletion
+16
-49
source/dnode/vnode/src/inc/tq.h
source/dnode/vnode/src/inc/tq.h
+0
-1
source/dnode/vnode/src/tq/tqUtil.c
source/dnode/vnode/src/tq/tqUtil.c
+13
-48
source/libs/executor/src/executor.c
source/libs/executor/src/executor.c
+3
-0
未找到文件。
source/dnode/vnode/src/inc/tq.h
浏览文件 @
50487255
...
...
@@ -106,7 +106,6 @@ typedef struct {
// STqPushHandle pushHandle; // push
STqExecHandle
execHandle
;
// exec
SRpcMsg
*
msg
;
int32_t
noDataPollCnt
;
tq_handle_status
status
;
}
STqHandle
;
...
...
source/dnode/vnode/src/tq/tqUtil.c
浏览文件 @
50487255
...
...
@@ -16,7 +16,6 @@
#include "tq.h"
#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0)
#define NO_POLL_CNT 5
static
int32_t
tqSendMetaPollRsp
(
STQ
*
pTq
,
const
SRpcMsg
*
pMsg
,
const
SMqPollReq
*
pReq
,
const
SMqMetaRsp
*
pRsp
);
...
...
@@ -88,15 +87,12 @@ static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) {
return
0
;
}
static
int32_t
extractResetOffsetVal
(
STqOffsetVal
*
pOffsetVal
,
STQ
*
pTq
,
STqHandle
*
pHandle
,
const
SMqPollReq
*
pRequest
,
SRpcMsg
*
pMsg
,
bool
*
pBlockReturned
)
{
static
int32_t
extractResetOffsetVal
(
STqOffsetVal
*
pOffsetVal
,
STQ
*
pTq
,
STqHandle
*
pHandle
,
const
SMqPollReq
*
pRequest
)
{
uint64_t
consumerId
=
pRequest
->
consumerId
;
STqOffsetVal
reqOffset
=
pRequest
->
reqOffset
;
STqOffset
*
pOffset
=
tqOffsetRead
(
pTq
->
pOffsetStore
,
pRequest
->
subKey
);
int32_t
vgId
=
TD_VID
(
pTq
->
pVnode
);
*
pBlockReturned
=
false
;
// In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value.
if
(
pOffset
!=
NULL
)
{
*
pOffsetVal
=
pOffset
->
val
;
...
...
@@ -129,28 +125,8 @@ static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHand
tqOffsetResetToLog
(
pOffsetVal
,
pHandle
->
pRef
->
refVer
-
1
);
}
}
else
if
(
reqOffset
.
type
==
TMQ_OFFSET__RESET_LATEST
)
{
if
(
pHandle
->
execHandle
.
subType
==
TOPIC_SUB_TYPE__COLUMN
)
{
SMqDataRsp
dataRsp
=
{
0
};
tqInitDataRsp
(
&
dataRsp
,
pRequest
,
pHandle
->
execHandle
.
subType
);
tqOffsetResetToLog
(
&
dataRsp
.
rspOffset
,
walGetLastVer
(
pTq
->
pVnode
->
pWal
));
tqDebug
(
"tmq poll: consumer:0x%"
PRIx64
", subkey %s, vgId:%d, (latest) offset reset to %"
PRId64
,
consumerId
,
pHandle
->
subKey
,
vgId
,
dataRsp
.
rspOffset
.
version
);
int32_t
code
=
tqSendDataRsp
(
pTq
,
pMsg
,
pRequest
,
&
dataRsp
,
TMQ_MSG_TYPE__POLL_RSP
);
tDeleteSMqDataRsp
(
&
dataRsp
);
*
pBlockReturned
=
true
;
return
code
;
}
else
{
STaosxRsp
taosxRsp
=
{
0
};
tqInitTaosxRsp
(
&
taosxRsp
,
pRequest
);
tqOffsetResetToLog
(
&
taosxRsp
.
rspOffset
,
walGetLastVer
(
pTq
->
pVnode
->
pWal
));
int32_t
code
=
tqSendDataRsp
(
pTq
,
pMsg
,
pRequest
,
(
SMqDataRsp
*
)
&
taosxRsp
,
TMQ_MSG_TYPE__TAOSX_RSP
);
tDeleteSTaosxRsp
(
&
taosxRsp
);
*
pBlockReturned
=
true
;
return
code
;
}
// offset set to previous version when init
tqOffsetResetToLog
(
pOffsetVal
,
walGetLastVer
(
pTq
->
pVnode
->
pWal
));
}
else
if
(
reqOffset
.
type
==
TMQ_OFFSET__RESET_NONE
)
{
tqError
(
"tmq poll: subkey:%s, no offset committed for consumer:0x%"
PRIx64
" in vg %d, subkey %s, reset none failed"
,
pHandle
->
subKey
,
consumerId
,
vgId
,
pRequest
->
subKey
);
...
...
@@ -173,15 +149,13 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
qSetTaskId
(
pHandle
->
execHandle
.
task
,
consumerId
,
pRequest
->
reqId
);
code
=
tqScanData
(
pTq
,
pHandle
,
&
dataRsp
,
pOffset
);
if
(
code
!=
0
)
{
if
(
code
!=
0
&&
terrno
!=
TSDB_CODE_WAL_LOG_NOT_EXIST
)
{
goto
end
;
}
// till now, all data has been transferred to consumer, new data needs to push client once arrived.
if
(
dataRsp
.
blockNum
==
0
&&
dataRsp
.
reqOffset
.
type
==
TMQ_OFFSET__LOG
&&
dataRsp
.
reqOffset
.
version
==
dataRsp
.
rspOffset
.
version
&&
pHandle
->
consumerId
==
pRequest
->
consumerId
)
{
if
(
pHandle
->
noDataPollCnt
>=
NO_POLL_CNT
){
// send poll result to client if no data 5 times to avoid lost data
pHandle
->
noDataPollCnt
=
0
;
if
(
terrno
==
TSDB_CODE_WAL_LOG_NOT_EXIST
||
(
dataRsp
.
blockNum
==
0
&&
dataRsp
.
reqOffset
.
type
==
TMQ_OFFSET__LOG
&&
dataRsp
.
reqOffset
.
version
==
dataRsp
.
rspOffset
.
version
&&
pHandle
->
consumerId
==
pRequest
->
consumerId
))
{
// lock
taosWLockLatch
(
&
pTq
->
lock
);
code
=
tqRegisterPushHandle
(
pTq
,
pHandle
,
pMsg
);
...
...
@@ -189,10 +163,6 @@ static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle,
tDeleteSMqDataRsp
(
&
dataRsp
);
return
code
;
}
else
{
pHandle
->
noDataPollCnt
++
;
}
}
// NOTE: this pHandle->consumerId may have been changed already.
code
=
tqSendDataRsp
(
pTq
,
pMsg
,
pRequest
,
(
SMqDataRsp
*
)
&
dataRsp
,
TMQ_MSG_TYPE__POLL_RSP
);
...
...
@@ -326,16 +296,11 @@ int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequ
// 1. reset the offset if needed
if
(
IS_OFFSET_RESET_TYPE
(
reqOffset
.
type
))
{
// handle the reset offset cases, according to the consumer's choice.
bool
blockReturned
=
false
;
code
=
extractResetOffsetVal
(
&
offset
,
pTq
,
pHandle
,
pRequest
,
pMsg
,
&
blockReturned
);
code
=
extractResetOffsetVal
(
&
offset
,
pTq
,
pHandle
,
pRequest
);
if
(
code
!=
0
)
{
return
code
;
}
// empty block returned, quit
if
(
blockReturned
)
{
return
0
;
}
}
else
{
// use the consumer specified offset
// the offset value can not be monotonious increase??
offset
=
reqOffset
;
...
...
source/libs/executor/src/executor.c
浏览文件 @
50487255
...
...
@@ -1080,6 +1080,9 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
const
char
*
id
=
GET_TASKID
(
pTaskInfo
);
// if pOffset equal to current offset, means continue consume
char
buf
[
80
]
=
{
0
};
tFormatOffset
(
buf
,
80
,
&
pTaskInfo
->
streamInfo
.
currentOffset
);
qDebug
(
"currentOffset:%s"
,
buf
);
if
(
tOffsetEqual
(
pOffset
,
&
pTaskInfo
->
streamInfo
.
currentOffset
))
{
return
0
;
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录