Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
cc80848b
T
TDengine
项目概览
taosdata
/
TDengine
1 年多 前同步成功
通知
1185
Star
22016
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
cc80848b
编写于
1月 20, 2022
作者:
L
Liu Jicong
提交者:
GitHub
1月 20, 2022
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #9935 from taosdata/feature/tq
refine tqRead interface
上级
beffd609
6a3d98cd
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
81 addition
and
58 deletion
+81
-58
include/common/tmsg.h
include/common/tmsg.h
+3
-3
source/dnode/mnode/impl/src/mndSubscribe.c
source/dnode/mnode/impl/src/mndSubscribe.c
+2
-2
source/dnode/vnode/inc/tq.h
source/dnode/vnode/inc/tq.h
+7
-4
source/dnode/vnode/src/tq/tq.c
source/dnode/vnode/src/tq/tq.c
+62
-10
source/dnode/vnode/src/vnd/vnodeQuery.c
source/dnode/vnode/src/vnd/vnodeQuery.c
+1
-1
source/dnode/vnode/src/vnd/vnodeWrite.c
source/dnode/vnode/src/vnd/vnodeWrite.c
+6
-38
未找到文件。
include/common/tmsg.h
浏览文件 @
cc80848b
...
...
@@ -1520,7 +1520,7 @@ typedef struct SMqSetCVgReq {
int32_t
vgId
;
int64_t
consumerId
;
char
topicName
[
TSDB_TOPIC_FNAME_LEN
];
char
c
G
roup
[
TSDB_CONSUMER_GROUP_LEN
];
char
c
g
roup
[
TSDB_CONSUMER_GROUP_LEN
];
char
*
sql
;
char
*
logicalPlan
;
char
*
physicalPlan
;
...
...
@@ -1532,7 +1532,7 @@ static FORCE_INLINE int32_t tEncodeSMqSetCVgReq(void** buf, const SMqSetCVgReq*
tlen
+=
taosEncodeFixedI32
(
buf
,
pReq
->
vgId
);
tlen
+=
taosEncodeFixedI64
(
buf
,
pReq
->
consumerId
);
tlen
+=
taosEncodeString
(
buf
,
pReq
->
topicName
);
tlen
+=
taosEncodeString
(
buf
,
pReq
->
c
G
roup
);
tlen
+=
taosEncodeString
(
buf
,
pReq
->
c
g
roup
);
tlen
+=
taosEncodeString
(
buf
,
pReq
->
sql
);
tlen
+=
taosEncodeString
(
buf
,
pReq
->
logicalPlan
);
tlen
+=
taosEncodeString
(
buf
,
pReq
->
physicalPlan
);
...
...
@@ -1543,7 +1543,7 @@ static FORCE_INLINE void* tDecodeSMqSetCVgReq(void* buf, SMqSetCVgReq* pReq) {
buf
=
taosDecodeFixedI32
(
buf
,
&
pReq
->
vgId
);
buf
=
taosDecodeFixedI64
(
buf
,
&
pReq
->
consumerId
);
buf
=
taosDecodeStringTo
(
buf
,
pReq
->
topicName
);
buf
=
taosDecodeStringTo
(
buf
,
pReq
->
c
G
roup
);
buf
=
taosDecodeStringTo
(
buf
,
pReq
->
c
g
roup
);
buf
=
taosDecodeString
(
buf
,
&
pReq
->
sql
);
buf
=
taosDecodeString
(
buf
,
&
pReq
->
logicalPlan
);
buf
=
taosDecodeString
(
buf
,
&
pReq
->
physicalPlan
);
...
...
source/dnode/mnode/impl/src/mndSubscribe.c
浏览文件 @
cc80848b
...
...
@@ -100,7 +100,7 @@ static int32_t mndProcessMqTimerMsg(SMnodeMsg *pMsg) {
.
vgId
=
pCEp
->
vgId
,
.
consumerId
=
consumerId
,
};
strcpy
(
req
.
c
G
roup
,
cgroup
);
strcpy
(
req
.
c
g
roup
,
cgroup
);
strcpy
(
req
.
topicName
,
topic
);
strcpy
(
req
.
sql
,
pTopic
->
sql
);
strcpy
(
req
.
logicalPlan
,
pTopic
->
logicalPlan
);
...
...
@@ -168,7 +168,7 @@ static int mndBuildMqSetConsumerVgReq(SMnode *pMnode, STrans *pTrans, SMqConsume
.
vgId
=
vgId
,
.
consumerId
=
pConsumer
->
consumerId
,
};
strcpy
(
req
.
c
G
roup
,
pConsumer
->
cgroup
);
strcpy
(
req
.
c
g
roup
,
pConsumer
->
cgroup
);
strcpy
(
req
.
topicName
,
pTopic
->
name
);
strcpy
(
req
.
sql
,
pTopic
->
sql
);
strcpy
(
req
.
logicalPlan
,
pTopic
->
logicalPlan
);
...
...
source/dnode/vnode/inc/tq.h
浏览文件 @
cc80848b
...
...
@@ -297,7 +297,7 @@ typedef struct STQ {
STqCfg
*
tqConfig
;
STqMemRef
tqMemRef
;
STqMetaStore
*
tqMeta
;
SWal
*
pWal
;
SWal
*
pWal
;
}
STQ
;
typedef
struct
STqMgmt
{
...
...
@@ -331,7 +331,8 @@ int tqRegisterContext(STqGroup*, void* ahandle);
int tqSendLaunchQuery(STqMsgItem*, int64_t offset);
#endif
int32_t
tqProcessConsume
(
STQ
*
pTq
,
SRpcMsg
*
pMsg
,
SRpcMsg
**
ppRsp
);
int32_t
tqProcessConsumeReq
(
STQ
*
pTq
,
SRpcMsg
*
pMsg
,
SRpcMsg
**
ppRsp
);
int32_t
tqProcessSetConnReq
(
STQ
*
pTq
,
SMqSetCVgReq
*
pReq
);
typedef
struct
STqReadHandle
{
int64_t
ver
;
...
...
@@ -340,13 +341,15 @@ typedef struct STqReadHandle {
SSubmitMsgIter
msgIter
;
SSubmitBlkIter
blkIter
;
SMeta
*
pMeta
;
SArray
*
pColumnIdList
;
}
STqReadHandle
;
STqReadHandle
*
tqInitSubmitMsgScanner
(
SMeta
*
pMeta
,
SSubmitMsg
*
pMsg
);
STqReadHandle
*
tqInitSubmitMsgScanner
(
SMeta
*
pMeta
,
SArray
*
pColumnIdList
);
void
tqReadHandleSetMsg
(
STqReadHandle
*
pHandle
,
SSubmitMsg
*
pMsg
,
int64_t
ver
);
bool
tqNextDataBlock
(
STqReadHandle
*
pHandle
);
int
tqRetrieveDataBlockInfo
(
STqReadHandle
*
pHandle
,
SDataBlockInfo
*
pBlockInfo
);
// return SArray<SColumnInfoData>
SArray
*
tqRetrieveDataBlock
(
STqReadHandle
*
pHandle
,
SArray
*
pColumnIdList
);
SArray
*
tqRetrieveDataBlock
(
STqReadHandle
*
pHandle
);
#ifdef __cplusplus
}
...
...
source/dnode/vnode/src/tq/tq.c
浏览文件 @
cc80848b
...
...
@@ -606,7 +606,7 @@ int tqItemSSize() {
return
0
;
}
int32_t
tqProcessConsume
(
STQ
*
pTq
,
SRpcMsg
*
pMsg
,
SRpcMsg
**
ppRsp
)
{
int32_t
tqProcessConsume
Req
(
STQ
*
pTq
,
SRpcMsg
*
pMsg
,
SRpcMsg
**
ppRsp
)
{
SMqCVConsumeReq
*
pReq
=
pMsg
->
pCont
;
int64_t
reqId
=
pReq
->
reqId
;
int64_t
consumerId
=
pReq
->
consumerId
;
...
...
@@ -623,6 +623,7 @@ int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) {
int8_t
old
=
atomic_val_compare_exchange_8
(
&
pHandle
->
buffer
.
output
[
pos
].
status
,
0
,
1
);
if
(
old
==
1
)
{
// do nothing
continue
;
}
if
(
walReadWithHandle
(
pHandle
->
pReadhandle
,
offset
)
<
0
)
{
// TODO
...
...
@@ -635,7 +636,17 @@ int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) {
SSubQueryMsg
*
pQueryMsg
=
pHandle
->
buffer
.
output
[
pos
].
pMsg
;
// TODO: launch query and get output data
void
*
outputData
;
pHandle
->
buffer
.
output
[
pos
].
dst
=
outputData
;
if
(
pHandle
->
buffer
.
firstOffset
==
-
1
||
pReq
->
offset
<
pHandle
->
buffer
.
firstOffset
)
{
pHandle
->
buffer
.
firstOffset
=
pReq
->
offset
;
}
if
(
pHandle
->
buffer
.
lastOffset
==
-
1
||
pReq
->
offset
>
pHandle
->
buffer
.
lastOffset
)
{
pHandle
->
buffer
.
lastOffset
=
pReq
->
offset
;
}
atomic_store_8
(
&
pHandle
->
buffer
.
output
[
pos
].
status
,
1
);
// put output into rsp
...
...
@@ -647,18 +658,64 @@ int32_t tqProcessConsume(STQ* pTq, SRpcMsg* pMsg, SRpcMsg** ppRsp) {
return
0
;
}
STqReadHandle
*
tqInitSubmitMsgScanner
(
SMeta
*
pMeta
,
SSubmitMsg
*
pMsg
)
{
int32_t
tqProcessSetConnReq
(
STQ
*
pTq
,
SMqSetCVgReq
*
pReq
)
{
STqConsumerHandle
*
pConsumer
=
calloc
(
sizeof
(
STqConsumerHandle
),
1
);
if
(
pConsumer
==
NULL
)
{
return
-
1
;
}
STqTopicHandle
*
pTopic
=
calloc
(
sizeof
(
STqTopicHandle
),
1
);
if
(
pTopic
==
NULL
)
{
free
(
pConsumer
);
return
-
1
;
}
strcpy
(
pTopic
->
topicName
,
pReq
->
topicName
);
strcpy
(
pTopic
->
cgroup
,
pReq
->
cgroup
);
strcpy
(
pTopic
->
sql
,
pReq
->
sql
);
strcpy
(
pTopic
->
logicalPlan
,
pReq
->
logicalPlan
);
strcpy
(
pTopic
->
physicalPlan
,
pReq
->
physicalPlan
);
SArray
*
pArray
;
//TODO: deserialize to SQueryDag
SQueryDag
*
pDag
;
// convert to task
if
(
schedulerConvertDagToTaskList
(
pDag
,
&
pArray
)
<
0
)
{
// TODO: handle error
}
ASSERT
(
taosArrayGetSize
(
pArray
)
==
0
);
STaskInfo
*
pInfo
=
taosArrayGet
(
pArray
,
0
);
SArray
*
pTasks
;
schedulerCopyTask
(
pInfo
,
&
pTasks
,
TQ_BUFFER_SIZE
);
pTopic
->
buffer
.
firstOffset
=
-
1
;
pTopic
->
buffer
.
lastOffset
=
-
1
;
for
(
int
i
=
0
;
i
<
TQ_BUFFER_SIZE
;
i
++
)
{
SSubQueryMsg
*
pMsg
=
taosArrayGet
(
pTasks
,
i
);
pTopic
->
buffer
.
output
[
i
].
pMsg
=
pMsg
;
pTopic
->
buffer
.
output
[
i
].
status
=
0
;
}
pTopic
->
pReadhandle
=
walOpenReadHandle
(
pTq
->
pWal
);
// write mq meta
return
0
;
}
STqReadHandle
*
tqInitSubmitMsgScanner
(
SMeta
*
pMeta
,
SArray
*
pColumnIdList
)
{
STqReadHandle
*
pReadHandle
=
malloc
(
sizeof
(
STqReadHandle
));
if
(
pReadHandle
==
NULL
)
{
return
NULL
;
}
pReadHandle
->
pMeta
=
pMeta
;
pReadHandle
->
pMsg
=
pMsg
;
tInitSubmitMsgIter
(
pMsg
,
&
pReadHandle
->
msgIter
);
pReadHandle
->
pMsg
=
NULL
;
pReadHandle
->
ver
=
-
1
;
pReadHandle
->
pColumnIdList
=
pColumnIdList
;
return
NULL
;
}
void
tqReadHandleSetMsg
(
STqReadHandle
*
pReadHandle
,
SSubmitMsg
*
pMsg
,
int64_t
ver
)
{
pReadHandle
->
pMsg
=
pMsg
;
tInitSubmitMsgIter
(
pMsg
,
&
pReadHandle
->
msgIter
);
pReadHandle
->
ver
=
ver
;
memset
(
&
pReadHandle
->
blkIter
,
0
,
sizeof
(
SSubmitBlkIter
));
}
bool
tqNextDataBlock
(
STqReadHandle
*
pHandle
)
{
if
(
tGetSubmitMsgNext
(
&
pHandle
->
msgIter
,
&
pHandle
->
pBlock
)
<
0
)
{
return
false
;
...
...
@@ -676,7 +733,7 @@ int tqRetrieveDataBlockInfo(STqReadHandle* pHandle, SDataBlockInfo* pBlockInfo)
// TODO: filter out unused column
return
0
;
}
SArray
*
tqRetrieveDataBlock
(
STqReadHandle
*
pHandle
,
SArray
*
pColumnIdList
)
{
SArray
*
tqRetrieveDataBlock
(
STqReadHandle
*
pHandle
)
{
int32_t
sversion
=
pHandle
->
pBlock
->
sversion
;
SSchemaWrapper
*
pSchemaWrapper
=
metaGetTableSchema
(
pHandle
->
pMeta
,
pHandle
->
pBlock
->
uid
,
sversion
,
true
);
STSchema
*
pTschema
=
metaGetTbTSchema
(
pHandle
->
pMeta
,
pHandle
->
pBlock
->
uid
,
sversion
);
...
...
@@ -691,11 +748,6 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle, SArray* pColumnIdList) {
return
NULL
;
}
for
(
int
i
=
0
;
i
<
pTschema
->
numOfCols
;
i
++
)
{
// TODO: filter out unused column
taosArrayPush
(
pColumnIdList
,
&
(
schemaColAt
(
pTschema
,
i
)
->
colId
));
}
SMemRow
row
;
int32_t
kvIdx
;
while
((
row
=
tGetSubmitBlkNext
(
&
pHandle
->
blkIter
))
!=
NULL
)
{
...
...
source/dnode/vnode/src/vnd/vnodeQuery.c
浏览文件 @
cc80848b
...
...
@@ -58,7 +58,7 @@ int vnodeProcessFetchReq(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
case
TDMT_VND_TABLE_META
:
return
vnodeGetTableMeta
(
pVnode
,
pMsg
,
pRsp
);
case
TDMT_VND_CONSUME
:
return
tqProcessConsume
(
pVnode
->
pTq
,
pMsg
,
pRsp
);
return
tqProcessConsume
Req
(
pVnode
->
pTq
,
pMsg
,
pRsp
);
default:
vError
(
"unknown msg type:%d in fetch queue"
,
pMsg
->
msgType
);
return
TSDB_CODE_VND_APP_ERROR
;
...
...
source/dnode/vnode/src/vnd/vnodeWrite.c
浏览文件 @
cc80848b
...
...
@@ -13,8 +13,8 @@
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "vnd.h"
#include "tq.h"
#include "vnd.h"
int
vnodeProcessNoWalWMsgs
(
SVnode
*
pVnode
,
SRpcMsg
*
pMsg
)
{
switch
(
pMsg
->
msgType
)
{
...
...
@@ -34,7 +34,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
pMsg
=
*
(
SRpcMsg
**
)
taosArrayGet
(
pMsgs
,
i
);
// ser request version
void
*
pBuf
=
POINTER_SHIFT
(
pMsg
->
pCont
,
sizeof
(
SMsgHead
));
void
*
pBuf
=
POINTER_SHIFT
(
pMsg
->
pCont
,
sizeof
(
SMsgHead
));
int64_t
ver
=
pVnode
->
state
.
processed
++
;
taosEncodeFixedU64
(
&
pBuf
,
ver
);
...
...
@@ -53,7 +53,7 @@ int vnodeProcessWMsgs(SVnode *pVnode, SArray *pMsgs) {
int
vnodeApplyWMsg
(
SVnode
*
pVnode
,
SRpcMsg
*
pMsg
,
SRpcMsg
**
pRsp
)
{
SVCreateTbReq
vCreateTbReq
;
SVCreateTbBatchReq
vCreateTbBatchReq
;
void
*
ptr
=
vnodeMalloc
(
pVnode
,
pMsg
->
contLen
);
void
*
ptr
=
vnodeMalloc
(
pVnode
,
pMsg
->
contLen
);
if
(
ptr
==
NULL
)
{
// TODO: handle error
}
...
...
@@ -110,43 +110,11 @@ int vnodeApplyWMsg(SVnode *pVnode, SRpcMsg *pMsg, SRpcMsg **pRsp) {
}
break
;
case
TDMT_VND_MQ_SET_CONN
:
{
//TODO: wrap in a function
char
*
reqStr
=
ptr
;
SMqSetCVgReq
req
;
tDecodeSMqSetCVgReq
(
reqStr
,
&
req
);
STqConsumerHandle
*
pConsumer
=
calloc
(
sizeof
(
STqConsumerHandle
),
1
);
STqTopicHandle
*
pTopic
=
calloc
(
sizeof
(
STqTopicHandle
),
1
);
if
(
pTopic
==
NULL
)
{
// TODO: handle error
}
strcpy
(
pTopic
->
topicName
,
req
.
topicName
);
strcpy
(
pTopic
->
cgroup
,
req
.
cGroup
);
strcpy
(
pTopic
->
sql
,
req
.
sql
);
strcpy
(
pTopic
->
logicalPlan
,
req
.
logicalPlan
);
strcpy
(
pTopic
->
physicalPlan
,
req
.
physicalPlan
);
SArray
*
pArray
;
//TODO: deserialize to SQueryDag
SQueryDag
*
pDag
;
// convert to task
if
(
schedulerConvertDagToTaskList
(
pDag
,
&
pArray
)
<
0
)
{
// TODO: handle error
}
ASSERT
(
taosArrayGetSize
(
pArray
)
==
0
);
STaskInfo
*
pInfo
=
taosArrayGet
(
pArray
,
0
);
SArray
*
pTasks
;
schedulerCopyTask
(
pInfo
,
&
pTasks
,
TQ_BUFFER_SIZE
);
pTopic
->
buffer
.
firstOffset
=
-
1
;
pTopic
->
buffer
.
lastOffset
=
-
1
;
for
(
int
i
=
0
;
i
<
TQ_BUFFER_SIZE
;
i
++
)
{
SSubQueryMsg
*
pMsg
=
taosArrayGet
(
pTasks
,
i
);
pTopic
->
buffer
.
output
[
i
].
pMsg
=
pMsg
;
pTopic
->
buffer
.
output
[
i
].
status
=
0
;
tDecodeSMqSetCVgReq
(
ptr
,
&
req
);
if
(
tqProcessSetConnReq
(
pVnode
->
pTq
,
&
req
)
<
0
)
{
}
pTopic
->
pReadhandle
=
walOpenReadHandle
(
pVnode
->
pTq
->
pWal
);
// write mq meta
}
break
;
}
break
;
default:
ASSERT
(
0
);
break
;
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录