Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
5d471a0c
TDengine
项目概览
taosdata
/
TDengine
12 个月 前同步成功
通知
1180
Star
22014
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
5d471a0c
编写于
11月 21, 2022
作者:
L
Liu Jicong
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
enh(taosx): split block when none column not match
上级
9a603e56
变更
4
隐藏空白更改
内联
并排
Showing
4 changed file
with
68 addition
and
36 deletion
+68
-36
source/dnode/vnode/inc/vnode.h
source/dnode/vnode/inc/vnode.h
+1
-0
source/dnode/vnode/src/tq/tqExec.c
source/dnode/vnode/src/tq/tqExec.c
+56
-23
source/dnode/vnode/src/tq/tqRead.c
source/dnode/vnode/src/tq/tqRead.c
+2
-2
source/dnode/vnode/src/tq/tqSink.c
source/dnode/vnode/src/tq/tqSink.c
+9
-11
未找到文件。
source/dnode/vnode/inc/vnode.h
浏览文件 @
5d471a0c
...
...
@@ -239,6 +239,7 @@ int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t v
bool
tqNextDataBlock
(
STqReader
*
pReader
);
bool
tqNextDataBlockFilterOut
(
STqReader
*
pReader
,
SHashObj
*
filterOutUids
);
int32_t
tqRetrieveDataBlock
(
SSDataBlock
*
pBlock
,
STqReader
*
pReader
);
int32_t
tqRetrieveTaosxBlock
(
STqReader
*
pReader
,
SArray
*
blocks
,
SArray
*
schemas
);
int32_t
vnodeEnqueueStreamMsg
(
SVnode
*
pVnode
,
SRpcMsg
*
pMsg
);
...
...
source/dnode/vnode/src/tq/tqExec.c
浏览文件 @
5d471a0c
...
...
@@ -44,7 +44,7 @@ static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, SMqDataRsp* pRs
return
0
;
}
static
int32_t
tqAddTbNameToRsp
(
const
STQ
*
pTq
,
int64_t
uid
,
SMqDataRsp
*
pRsp
)
{
static
int32_t
tqAddTbNameToRsp
(
const
STQ
*
pTq
,
int64_t
uid
,
SMqDataRsp
*
pRsp
,
int32_t
n
)
{
SMetaReader
mr
=
{
0
};
metaReaderInit
(
&
mr
,
pTq
->
pVnode
->
pMeta
,
0
);
// TODO add reference to gurantee success
...
...
@@ -52,8 +52,10 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, int64_t uid, SMqDataRsp* pRsp) {
metaReaderClear
(
&
mr
);
return
-
1
;
}
char
*
tbName
=
strdup
(
mr
.
me
.
name
);
taosArrayPush
(
pRsp
->
blockTbName
,
&
tbName
);
for
(
int32_t
i
=
0
;
i
<
n
;
i
++
)
{
char
*
tbName
=
strdup
(
mr
.
me
.
name
);
taosArrayPush
(
pRsp
->
blockTbName
,
&
tbName
);
}
metaReaderClear
(
&
mr
);
return
0
;
}
...
...
@@ -111,7 +113,7 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs
if
(
pRsp
->
withTbName
)
{
if
(
pRsp
->
rspOffset
.
type
==
TMQ_OFFSET__LOG
)
{
int64_t
uid
=
pExec
->
pExecReader
->
msgIter
.
uid
;
tqAddTbNameToRsp
(
pTq
,
uid
,
pRsp
);
tqAddTbNameToRsp
(
pTq
,
uid
,
pRsp
,
1
);
}
else
{
pRsp
->
withTbName
=
false
;
}
...
...
@@ -155,7 +157,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta
int64_t
uid
=
0
;
if
(
pOffset
->
type
==
TMQ_OFFSET__LOG
)
{
uid
=
pExec
->
pExecReader
->
msgIter
.
uid
;
if
(
tqAddTbNameToRsp
(
pTq
,
uid
,
(
SMqDataRsp
*
)
pRsp
)
<
0
)
{
if
(
tqAddTbNameToRsp
(
pTq
,
uid
,
(
SMqDataRsp
*
)
pRsp
,
1
)
<
0
)
{
continue
;
}
}
else
{
...
...
@@ -225,18 +227,27 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp
STqExecHandle
*
pExec
=
&
pHandle
->
execHandle
;
ASSERT
(
pExec
->
subType
!=
TOPIC_SUB_TYPE__COLUMN
);
SArray
*
pBlocks
=
taosArrayInit
(
0
,
sizeof
(
SSDataBlock
));
SArray
*
pSchemas
=
taosArrayInit
(
0
,
sizeof
(
SSchemaWrapper
));
if
(
pExec
->
subType
==
TOPIC_SUB_TYPE__TABLE
)
{
STqReader
*
pReader
=
pExec
->
pExecReader
;
tqReaderSetDataMsg
(
pReader
,
pReq
,
0
);
while
(
tqNextDataBlock
(
pReader
))
{
SSDataBlock
block
=
{
0
};
if
(
tqRetrieveDataBlock
(
&
block
,
pReader
)
<
0
)
{
/*SSDataBlock block = {0};*/
/*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/
/*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/
/*}*/
taosArrayClear
(
pBlocks
);
taosArrayClear
(
pSchemas
);
if
(
tqRetrieveTaosxBlock
(
pReader
,
pBlocks
,
pSchemas
)
<
0
)
{
if
(
terrno
==
TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND
)
continue
;
}
if
(
pRsp
->
withTbName
)
{
int64_t
uid
=
pExec
->
pExecReader
->
msgIter
.
uid
;
if
(
tqAddTbNameToRsp
(
pTq
,
uid
,
(
SMqDataRsp
*
)
pRsp
)
<
0
)
{
blockDataFreeRes
(
&
block
);
if
(
tqAddTbNameToRsp
(
pTq
,
uid
,
(
SMqDataRsp
*
)
pRsp
,
taosArrayGetSize
(
pBlocks
)
)
<
0
)
{
taosArrayDestroyEx
(
pBlocks
,
(
FDelete
)
blockDataFreeRes
);
continue
;
}
}
...
...
@@ -255,24 +266,34 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp
pRsp
->
createTableNum
++
;
}
}
tqAddBlockDataToRsp
(
&
block
,
(
SMqDataRsp
*
)
pRsp
,
taosArrayGetSize
(
block
.
pDataBlock
),
pTq
->
pVnode
->
config
.
tsdbCfg
.
precision
);
blockDataFreeRes
(
&
block
);
tqAddBlockSchemaToRsp
(
pExec
,
(
SMqDataRsp
*
)
pRsp
);
pRsp
->
blockNum
++
;
for
(
int32_t
i
=
0
;
i
<
taosArrayGetSize
(
pBlocks
);
i
++
)
{
SSDataBlock
*
pBlock
=
taosArrayGet
(
pBlocks
,
i
);
tqAddBlockDataToRsp
(
pBlock
,
(
SMqDataRsp
*
)
pRsp
,
taosArrayGetSize
(
pBlock
->
pDataBlock
),
pTq
->
pVnode
->
config
.
tsdbCfg
.
precision
);
blockDataFreeRes
(
pBlock
);
SSchemaWrapper
*
pSW
=
taosArrayGet
(
pSchemas
,
i
);
taosArrayPush
(
pRsp
->
blockSchema
,
&
pSW
);
pRsp
->
blockNum
++
;
}
}
}
else
if
(
pExec
->
subType
==
TOPIC_SUB_TYPE__DB
)
{
STqReader
*
pReader
=
pExec
->
pExecReader
;
tqReaderSetDataMsg
(
pReader
,
pReq
,
0
);
while
(
tqNextDataBlockFilterOut
(
pReader
,
pExec
->
execDb
.
pFilterOutTbUid
))
{
SSDataBlock
block
=
{
0
};
if
(
tqRetrieveDataBlock
(
&
block
,
pReader
)
<
0
)
{
/*SSDataBlock block = {0};*/
/*if (tqRetrieveDataBlock(&block, pReader) < 0) {*/
/*if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue;*/
/*}*/
taosArrayClear
(
pBlocks
);
taosArrayClear
(
pSchemas
);
if
(
tqRetrieveTaosxBlock
(
pReader
,
pBlocks
,
pSchemas
)
<
0
)
{
if
(
terrno
==
TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND
)
continue
;
}
if
(
pRsp
->
withTbName
)
{
int64_t
uid
=
pExec
->
pExecReader
->
msgIter
.
uid
;
if
(
tqAddTbNameToRsp
(
pTq
,
uid
,
(
SMqDataRsp
*
)
pRsp
)
<
0
)
{
blockDataFreeRes
(
&
block
);
if
(
tqAddTbNameToRsp
(
pTq
,
uid
,
(
SMqDataRsp
*
)
pRsp
,
taosArrayGetSize
(
pBlocks
))
<
0
)
{
taosArrayDestroyEx
(
pBlocks
,
(
FDelete
)
blockDataFreeRes
);
/*blockDataFreeRes(&block);*/
continue
;
}
}
...
...
@@ -291,14 +312,26 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp
pRsp
->
createTableNum
++
;
}
}
tqAddBlockDataToRsp
(
&
block
,
(
SMqDataRsp
*
)
pRsp
,
taosArrayGetSize
(
block
.
pDataBlock
),
pTq
->
pVnode
->
config
.
tsdbCfg
.
precision
);
blockDataFreeRes
(
&
block
);
tqAddBlockSchemaToRsp
(
pExec
,
(
SMqDataRsp
*
)
pRsp
);
pRsp
->
blockNum
++
;
/*tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock),*/
/*pTq->pVnode->config.tsdbCfg.precision);*/
/*blockDataFreeRes(&block);*/
/*tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp);*/
/*pRsp->blockNum++;*/
for
(
int32_t
i
=
0
;
i
<
taosArrayGetSize
(
pBlocks
);
i
++
)
{
SSDataBlock
*
pBlock
=
taosArrayGet
(
pBlocks
,
i
);
tqAddBlockDataToRsp
(
pBlock
,
(
SMqDataRsp
*
)
pRsp
,
taosArrayGetSize
(
pBlock
->
pDataBlock
),
pTq
->
pVnode
->
config
.
tsdbCfg
.
precision
);
blockDataFreeRes
(
pBlock
);
SSchemaWrapper
*
pSW
=
taosArrayGet
(
pSchemas
,
i
);
taosArrayPush
(
pRsp
->
blockSchema
,
&
pSW
);
pRsp
->
blockNum
++
;
}
}
}
taosArrayDestroy
(
pBlocks
);
taosArrayDestroy
(
pSchemas
);
if
(
pRsp
->
blockNum
==
0
)
{
return
-
1
;
}
...
...
source/dnode/vnode/src/tq/tqRead.c
浏览文件 @
5d471a0c
...
...
@@ -556,7 +556,7 @@ FAIL:
return
-
1
;
}
int32_t
tq
SplitRetrieveData
Block
(
STqReader
*
pReader
,
SArray
*
blocks
,
SArray
*
schemas
)
{
int32_t
tq
RetrieveTaosx
Block
(
STqReader
*
pReader
,
SArray
*
blocks
,
SArray
*
schemas
)
{
int32_t
sversion
=
htonl
(
pReader
->
pBlock
->
sversion
);
if
(
pReader
->
cachedSchemaSuid
==
0
||
pReader
->
cachedSchemaVer
!=
sversion
||
...
...
@@ -594,7 +594,7 @@ int32_t tqSplitRetrieveDataBlock(STqReader* pReader, SArray* blocks, SArray* sch
int32_t
curRow
=
0
;
char
*
assigned
=
taosMemoryCalloc
(
1
,
pSchemaWrapper
->
nCols
);
if
(
assigned
)
return
-
1
;
if
(
assigned
==
NULL
)
return
-
1
;
tInitSubmitBlkIter
(
&
pReader
->
msgIter
,
pReader
->
pBlock
,
&
pReader
->
blkIter
);
STSRowIter
iter
=
{
0
};
...
...
source/dnode/vnode/src/tq/tqSink.c
浏览文件 @
5d471a0c
...
...
@@ -349,7 +349,6 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d
.
contLen
=
len
+
sizeof
(
SMsgHead
),
};
if
(
tmsgPutToQueue
(
&
pVnode
->
msgCb
,
WRITE_QUEUE
,
&
msg
)
!=
0
)
{
rpcFreeCont
(
serializedDeleteReq
);
tqDebug
(
"failed to put delete req into write-queue since %s"
,
terrstr
());
}
}
else
{
...
...
@@ -476,12 +475,12 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d
cap
+=
sizeof
(
SSubmitBlk
)
+
schemaLen
+
rows
*
maxLen
;
SSubmitReq
*
re
t
=
rpcMallocCont
(
cap
);
re
t
->
header
.
vgId
=
pVnode
->
config
.
vgId
;
re
t
->
length
=
sizeof
(
SSubmitReq
);
re
t
->
numOfBlocks
=
htonl
(
1
);
SSubmitReq
*
pSubmi
t
=
rpcMallocCont
(
cap
);
pSubmi
t
->
header
.
vgId
=
pVnode
->
config
.
vgId
;
pSubmi
t
->
length
=
sizeof
(
SSubmitReq
);
pSubmi
t
->
numOfBlocks
=
htonl
(
1
);
SSubmitBlk
*
blkHead
=
POINTER_SHIFT
(
re
t
,
sizeof
(
SSubmitReq
));
SSubmitBlk
*
blkHead
=
POINTER_SHIFT
(
pSubmi
t
,
sizeof
(
SSubmitReq
));
blkHead
->
numOfRows
=
htonl
(
pDataBlock
->
info
.
rows
);
blkHead
->
sversion
=
htonl
(
pTSchema
->
version
);
...
...
@@ -531,17 +530,16 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d
}
blkHead
->
dataLen
=
htonl
(
dataLen
);
re
t
->
length
+=
sizeof
(
SSubmitBlk
)
+
schemaLen
+
dataLen
;
ret
->
length
=
htonl
(
re
t
->
length
);
pSubmi
t
->
length
+=
sizeof
(
SSubmitBlk
)
+
schemaLen
+
dataLen
;
pSubmit
->
length
=
htonl
(
pSubmi
t
->
length
);
SRpcMsg
msg
=
{
.
msgType
=
TDMT_VND_SUBMIT
,
.
pCont
=
re
t
,
.
contLen
=
ntohl
(
re
t
->
length
),
.
pCont
=
pSubmi
t
,
.
contLen
=
ntohl
(
pSubmi
t
->
length
),
};
if
(
tmsgPutToQueue
(
&
pVnode
->
msgCb
,
WRITE_QUEUE
,
&
msg
)
!=
0
)
{
rpcFreeCont
(
ret
);
tqDebug
(
"failed to put into write-queue since %s"
,
terrstr
());
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录