Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
78222a17
T
TDengine
项目概览
taosdata
/
TDengine
大约 2 年 前同步成功
通知
1193
Star
22018
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
未验证
提交
78222a17
编写于
4月 21, 2022
作者:
L
Liu Jicong
提交者:
GitHub
4月 21, 2022
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #11706 from taosdata/feature/tq
refactor(tmq): rewrite tq read function
上级
b602a77c
3dd3ad1e
变更
6
显示空白变更内容
内联
并排
Showing
6 changed file
with
216 addition
and
157 deletion
+216
-157
include/libs/planner/planner.h
include/libs/planner/planner.h
+1
-0
source/dnode/mnode/impl/inc/mndScheduler.h
source/dnode/mnode/impl/inc/mndScheduler.h
+2
-0
source/dnode/mnode/impl/src/mndScheduler.c
source/dnode/mnode/impl/src/mndScheduler.c
+48
-0
source/dnode/vnode/inc/vnode.h
source/dnode/vnode/inc/vnode.h
+1
-2
source/dnode/vnode/src/tq/tqRead.c
source/dnode/vnode/src/tq/tqRead.c
+21
-26
source/libs/executor/src/scanoperator.c
source/libs/executor/src/scanoperator.c
+143
-129
未找到文件。
include/libs/planner/planner.h
浏览文件 @
78222a17
...
@@ -30,6 +30,7 @@ typedef struct SPlanContext {
...
@@ -30,6 +30,7 @@ typedef struct SPlanContext {
SNode
*
pAstRoot
;
SNode
*
pAstRoot
;
bool
topicQuery
;
bool
topicQuery
;
bool
streamQuery
;
bool
streamQuery
;
bool
rSmaQuery
;
bool
showRewrite
;
bool
showRewrite
;
int8_t
triggerType
;
int8_t
triggerType
;
int64_t
watermark
;
int64_t
watermark
;
...
...
source/dnode/mnode/impl/inc/mndScheduler.h
浏览文件 @
78222a17
...
@@ -29,6 +29,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
...
@@ -29,6 +29,8 @@ int32_t mndSchedInitSubEp(SMnode* pMnode, const SMqTopicObj* pTopic, SMqSubscrib
int32_t
mndScheduleStream
(
SMnode
*
pMnode
,
STrans
*
pTrans
,
SStreamObj
*
pStream
);
int32_t
mndScheduleStream
(
SMnode
*
pMnode
,
STrans
*
pTrans
,
SStreamObj
*
pStream
);
int32_t
mndConvertRSmaTask
(
const
char
*
ast
,
int8_t
triggerType
,
int64_t
watermark
,
char
**
pStr
,
int32_t
*
pLen
);
#ifdef __cplusplus
#ifdef __cplusplus
}
}
#endif
#endif
...
...
source/dnode/mnode/impl/src/mndScheduler.c
浏览文件 @
78222a17
...
@@ -34,6 +34,54 @@
...
@@ -34,6 +34,54 @@
extern
bool
tsStreamSchedV
;
extern
bool
tsStreamSchedV
;
int32_t
mndConvertRSmaTask
(
const
char
*
ast
,
int8_t
triggerType
,
int64_t
watermark
,
char
**
pStr
,
int32_t
*
pLen
)
{
SNode
*
pAst
=
NULL
;
SQueryPlan
*
pPlan
=
NULL
;
terrno
=
TSDB_CODE_SUCCESS
;
if
(
nodesStringToNode
(
ast
,
&
pAst
)
<
0
)
{
terrno
=
TSDB_CODE_QRY_INVALID_INPUT
;
goto
END
;
}
SPlanContext
cxt
=
{
.
pAstRoot
=
pAst
,
.
topicQuery
=
false
,
.
streamQuery
=
true
,
.
rSmaQuery
=
true
,
.
triggerType
=
triggerType
,
.
watermark
=
watermark
,
};
if
(
qCreateQueryPlan
(
&
cxt
,
&
pPlan
,
NULL
)
<
0
)
{
terrno
=
TSDB_CODE_QRY_INVALID_INPUT
;
goto
END
;
}
int32_t
levelNum
=
LIST_LENGTH
(
pPlan
->
pSubplans
);
if
(
levelNum
!=
1
)
{
terrno
=
TSDB_CODE_QRY_INVALID_INPUT
;
goto
END
;
}
SNodeListNode
*
inner
=
nodesListGetNode
(
pPlan
->
pSubplans
,
0
);
int32_t
opNum
=
LIST_LENGTH
(
inner
->
pNodeList
);
if
(
opNum
!=
1
)
{
terrno
=
TSDB_CODE_QRY_INVALID_INPUT
;
goto
END
;
}
SSubplan
*
plan
=
nodesListGetNode
(
inner
->
pNodeList
,
0
);
if
(
qSubPlanToString
(
plan
,
pStr
,
pLen
)
<
0
)
{
terrno
=
TSDB_CODE_QRY_INVALID_INPUT
;
goto
END
;
}
END:
if
(
pAst
)
nodesDestroyNode
(
pAst
);
if
(
pPlan
)
nodesDestroyNode
(
pPlan
);
return
terrno
;
}
int32_t
mndPersistTaskDeployReq
(
STrans
*
pTrans
,
SStreamTask
*
pTask
,
const
SEpSet
*
pEpSet
,
tmsg_t
type
,
int32_t
nodeId
)
{
int32_t
mndPersistTaskDeployReq
(
STrans
*
pTrans
,
SStreamTask
*
pTask
,
const
SEpSet
*
pEpSet
,
tmsg_t
type
,
int32_t
nodeId
)
{
SCoder
encoder
;
SCoder
encoder
;
tCoderInit
(
&
encoder
,
TD_LITTLE_ENDIAN
,
NULL
,
0
,
TD_ENCODER
);
tCoderInit
(
&
encoder
,
TD_LITTLE_ENDIAN
,
NULL
,
0
,
TD_ENCODER
);
...
...
source/dnode/vnode/inc/vnode.h
浏览文件 @
78222a17
...
@@ -109,8 +109,7 @@ int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList
...
@@ -109,8 +109,7 @@ int tqReadHandleSetTbUidList(STqReadHandle *pHandle, const SArray *tbUidList
int
tqReadHandleAddTbUidList
(
STqReadHandle
*
pHandle
,
const
SArray
*
tbUidList
);
int
tqReadHandleAddTbUidList
(
STqReadHandle
*
pHandle
,
const
SArray
*
tbUidList
);
int32_t
tqReadHandleSetMsg
(
STqReadHandle
*
pHandle
,
SSubmitReq
*
pMsg
,
int64_t
ver
);
int32_t
tqReadHandleSetMsg
(
STqReadHandle
*
pHandle
,
SSubmitReq
*
pMsg
,
int64_t
ver
);
bool
tqNextDataBlock
(
STqReadHandle
*
pHandle
);
bool
tqNextDataBlock
(
STqReadHandle
*
pHandle
);
int
tqRetrieveDataBlockInfo
(
STqReadHandle
*
pHandle
,
SDataBlockInfo
*
pBlockInfo
);
int32_t
tqRetrieveDataBlock
(
SArray
**
ppCols
,
STqReadHandle
*
pHandle
,
uint64_t
*
pGroupId
,
int32_t
*
pNumOfRows
);
SArray
*
tqRetrieveDataBlock
(
STqReadHandle
*
pHandle
);
// need to reposition
// need to reposition
...
...
source/dnode/vnode/src/tq/tqRead.c
浏览文件 @
78222a17
...
@@ -82,16 +82,7 @@ bool tqNextDataBlock(STqReadHandle* pHandle) {
...
@@ -82,16 +82,7 @@ bool tqNextDataBlock(STqReadHandle* pHandle) {
return
false
;
return
false
;
}
}
int
tqRetrieveDataBlockInfo
(
STqReadHandle
*
pHandle
,
SDataBlockInfo
*
pBlockInfo
)
{
int32_t
tqRetrieveDataBlock
(
SArray
**
ppCols
,
STqReadHandle
*
pHandle
,
uint64_t
*
pGroupId
,
int32_t
*
pNumOfRows
)
{
// currently only rows are used
pBlockInfo
->
numOfCols
=
taosArrayGetSize
(
pHandle
->
pColIdList
);
pBlockInfo
->
rows
=
pHandle
->
pBlock
->
numOfRows
;
// pBlockInfo->uid = pHandle->pBlock->uid; // the uid can not be assigned to pBlockData.
return
0
;
}
SArray
*
tqRetrieveDataBlock
(
STqReadHandle
*
pHandle
)
{
/*int32_t sversion = pHandle->pBlock->sversion;*/
/*int32_t sversion = pHandle->pBlock->sversion;*/
// TODO set to real sversion
// TODO set to real sversion
int32_t
sversion
=
0
;
int32_t
sversion
=
0
;
...
@@ -112,7 +103,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
...
@@ -112,7 +103,7 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
STSchema
*
pTschema
=
pHandle
->
pSchema
;
STSchema
*
pTschema
=
pHandle
->
pSchema
;
SSchemaWrapper
*
pSchemaWrapper
=
pHandle
->
pSchemaWrapper
;
SSchemaWrapper
*
pSchemaWrapper
=
pHandle
->
pSchemaWrapper
;
int32_t
n
umOfRows
=
pHandle
->
pBlock
->
numOfRows
;
*
pN
umOfRows
=
pHandle
->
pBlock
->
numOfRows
;
/*int32_t numOfCols = pHandle->pSchema->numOfCols;*/
/*int32_t numOfCols = pHandle->pSchema->numOfCols;*/
int32_t
colNumNeed
=
taosArrayGetSize
(
pHandle
->
pColIdList
);
int32_t
colNumNeed
=
taosArrayGetSize
(
pHandle
->
pColIdList
);
...
@@ -120,10 +111,11 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
...
@@ -120,10 +111,11 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
colNumNeed
=
pSchemaWrapper
->
nCols
;
colNumNeed
=
pSchemaWrapper
->
nCols
;
}
}
SArray
*
pArray
=
taosArrayInit
(
colNumNeed
,
sizeof
(
SColumnInfoData
));
*
ppCols
=
taosArrayInit
(
colNumNeed
,
sizeof
(
SColumnInfoData
));
if
(
pArray
==
NULL
)
{
if
(
*
ppCols
==
NULL
)
{
return
NULL
;
return
-
1
;
}
}
int32_t
colMeta
=
0
;
int32_t
colMeta
=
0
;
int32_t
colNeed
=
0
;
int32_t
colNeed
=
0
;
while
(
colMeta
<
pSchemaWrapper
->
nCols
&&
colNeed
<
colNumNeed
)
{
while
(
colMeta
<
pSchemaWrapper
->
nCols
&&
colNeed
<
colNumNeed
)
{
...
@@ -136,21 +128,24 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
...
@@ -136,21 +128,24 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
colNeed
++
;
colNeed
++
;
}
else
{
}
else
{
SColumnInfoData
colInfo
=
{
0
};
SColumnInfoData
colInfo
=
{
0
};
/*int sz = numOfRows * pColSchema->bytes;*/
colInfo
.
info
.
bytes
=
pColSchema
->
bytes
;
colInfo
.
info
.
bytes
=
pColSchema
->
bytes
;
colInfo
.
info
.
colId
=
pColSchema
->
colId
;
colInfo
.
info
.
colId
=
pColSchema
->
colId
;
colInfo
.
info
.
type
=
pColSchema
->
type
;
colInfo
.
info
.
type
=
pColSchema
->
type
;
if
(
colInfoDataEnsureCapacity
(
&
colInfo
,
0
,
numOfRows
)
<
0
)
{
if
(
colInfoDataEnsureCapacity
(
&
colInfo
,
0
,
*
pNumOfRows
)
<
0
)
{
taosArrayDestroyEx
(
pArray
,
(
void
(
*
)(
void
*
))
tDeleteSSDataBlock
);
goto
FAIL
;
return
NULL
;
}
}
taosArrayPush
(
pArray
,
&
colInfo
);
taosArrayPush
(
*
ppCols
,
&
colInfo
);
colMeta
++
;
colMeta
++
;
colNeed
++
;
colNeed
++
;
}
}
}
}
int32_t
colActual
=
taosArrayGetSize
(
*
ppCols
);
// TODO in stream shuffle case, fetch groupId
*
pGroupId
=
0
;
STSRowIter
iter
=
{
0
};
STSRowIter
iter
=
{
0
};
tdSTSRowIterInit
(
&
iter
,
pTschema
);
tdSTSRowIterInit
(
&
iter
,
pTschema
);
STSRow
*
row
;
STSRow
*
row
;
...
@@ -159,22 +154,22 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
...
@@ -159,22 +154,22 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
while
((
row
=
tGetSubmitBlkNext
(
&
pHandle
->
blkIter
))
!=
NULL
)
{
while
((
row
=
tGetSubmitBlkNext
(
&
pHandle
->
blkIter
))
!=
NULL
)
{
tdSTSRowIterReset
(
&
iter
,
row
);
tdSTSRowIterReset
(
&
iter
,
row
);
// get all wanted col of that block
// get all wanted col of that block
int32_t
colTot
=
taosArrayGetSize
(
pArray
);
for
(
int32_t
i
=
0
;
i
<
colActual
;
i
++
)
{
for
(
int32_t
i
=
0
;
i
<
colTot
;
i
++
)
{
SColumnInfoData
*
pColData
=
taosArrayGet
(
*
ppCols
,
i
);
SColumnInfoData
*
pColData
=
taosArrayGet
(
pArray
,
i
);
SCellVal
sVal
=
{
0
};
SCellVal
sVal
=
{
0
};
if
(
!
tdSTSRowIterNext
(
&
iter
,
pColData
->
info
.
colId
,
pColData
->
info
.
type
,
&
sVal
))
{
if
(
!
tdSTSRowIterNext
(
&
iter
,
pColData
->
info
.
colId
,
pColData
->
info
.
type
,
&
sVal
))
{
break
;
break
;
}
}
/*if (colDataAppend(pColData, curRow, sVal.val, false) < 0) {*/
if
(
colDataAppend
(
pColData
,
curRow
,
sVal
.
val
,
sVal
.
valType
==
TD_VTYPE_NULL
)
<
0
)
{
if
(
colDataAppend
(
pColData
,
curRow
,
sVal
.
val
,
sVal
.
valType
==
TD_VTYPE_NULL
)
<
0
)
{
taosArrayDestroyEx
(
pArray
,
(
void
(
*
)(
void
*
))
tDeleteSSDataBlock
);
goto
FAIL
;
return
NULL
;
}
}
}
}
curRow
++
;
curRow
++
;
}
}
return
pArray
;
return
0
;
FAIL:
taosArrayDestroy
(
*
ppCols
);
return
-
1
;
}
}
void
tqReadHandleSetColIdList
(
STqReadHandle
*
pReadHandle
,
SArray
*
pColIdList
)
{
pReadHandle
->
pColIdList
=
pColIdList
;
}
void
tqReadHandleSetColIdList
(
STqReadHandle
*
pReadHandle
,
SArray
*
pColIdList
)
{
pReadHandle
->
pColIdList
=
pColIdList
;
}
...
...
source/libs/executor/src/scanoperator.c
浏览文件 @
78222a17
...
@@ -30,13 +30,12 @@
...
@@ -30,13 +30,12 @@
#include "query.h"
#include "query.h"
#include "tcompare.h"
#include "tcompare.h"
#include "thash.h"
#include "thash.h"
#include "vnode.h"
#include "ttypes.h"
#include "ttypes.h"
#include "vnode.h"
#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
#define SET_REVERSE_SCAN_FLAG(_info) ((_info)->scanFlag = REVERSE_SCAN)
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
#define SWITCH_ORDER(n) (((n) = ((n) == TSDB_ORDER_ASC) ? TSDB_ORDER_DESC : TSDB_ORDER_ASC))
void
switchCtxOrder
(
SqlFunctionCtx
*
pCtx
,
int32_t
numOfOutput
)
{
void
switchCtxOrder
(
SqlFunctionCtx
*
pCtx
,
int32_t
numOfOutput
)
{
for
(
int32_t
i
=
0
;
i
<
numOfOutput
;
++
i
)
{
for
(
int32_t
i
=
0
;
i
<
numOfOutput
;
++
i
)
{
SWITCH_ORDER
(
pCtx
[
i
].
order
);
SWITCH_ORDER
(
pCtx
[
i
].
order
);
...
@@ -90,7 +89,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord
...
@@ -90,7 +89,7 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord
}
}
int64_t
key
=
tw
->
skey
,
interval
=
pInterval
->
interval
;
int64_t
key
=
tw
->
skey
,
interval
=
pInterval
->
interval
;
//convert key to second
//
convert key to second
key
=
convertTimePrecision
(
key
,
pInterval
->
precision
,
TSDB_TIME_PRECISION_MILLI
)
/
1000
;
key
=
convertTimePrecision
(
key
,
pInterval
->
precision
,
TSDB_TIME_PRECISION_MILLI
)
/
1000
;
if
(
pInterval
->
intervalUnit
==
'y'
)
{
if
(
pInterval
->
intervalUnit
==
'y'
)
{
...
@@ -125,8 +124,8 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
...
@@ -125,8 +124,8 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
// todo handle the time range case
// todo handle the time range case
TSKEY
sk
=
INT64_MIN
;
TSKEY
sk
=
INT64_MIN
;
TSKEY
ek
=
INT64_MAX
;
TSKEY
ek
=
INT64_MAX
;
// TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
// TSKEY sk = MIN(pQueryAttr->window.skey, pQueryAttr->window.ekey);
// TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
// TSKEY ek = MAX(pQueryAttr->window.skey, pQueryAttr->window.ekey);
if
(
true
)
{
if
(
true
)
{
getAlignQueryTimeWindow
(
pInterval
,
pInterval
->
precision
,
pBlockInfo
->
window
.
skey
,
&
w
);
getAlignQueryTimeWindow
(
pInterval
,
pInterval
->
precision
,
pBlockInfo
->
window
.
skey
,
&
w
);
...
@@ -136,7 +135,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
...
@@ -136,7 +135,7 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
return
true
;
return
true
;
}
}
while
(
1
)
{
// todo handle the desc order scan case
while
(
1
)
{
// todo handle the desc order scan case
getNextTimeWindow
(
pInterval
,
&
w
,
TSDB_ORDER_ASC
);
getNextTimeWindow
(
pInterval
,
&
w
,
TSDB_ORDER_ASC
);
if
(
w
.
skey
>
pBlockInfo
->
window
.
ekey
)
{
if
(
w
.
skey
>
pBlockInfo
->
window
.
ekey
)
{
break
;
break
;
...
@@ -148,24 +147,24 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
...
@@ -148,24 +147,24 @@ static bool overlapWithTimeWindow(SInterval* pInterval, SDataBlockInfo* pBlockIn
}
}
}
}
}
else
{
}
else
{
// getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w);
// getAlignQueryTimeWindow(pQueryAttr, pBlockInfo->window.ekey, sk, ek, &w);
// assert(w.skey <= pBlockInfo->window.ekey);
// assert(w.skey <= pBlockInfo->window.ekey);
//
//
// if (w.skey > pBlockInfo->window.skey) {
// if (w.skey > pBlockInfo->window.skey) {
// return true;
// return true;
// }
// }
//
//
// while(1) {
// while(1) {
// getNextTimeWindow(pQueryAttr, &w);
// getNextTimeWindow(pQueryAttr, &w);
// if (w.ekey < pBlockInfo->window.skey) {
// if (w.ekey < pBlockInfo->window.skey) {
// break;
// break;
// }
// }
//
//
// assert(w.skey < pBlockInfo->window.skey);
// assert(w.skey < pBlockInfo->window.skey);
// if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) {
// if (w.ekey < pBlockInfo->window.ekey && w.ekey >= pBlockInfo->window.skey) {
// return true;
// return true;
// }
// }
// }
// }
}
}
return
false
;
return
false
;
...
@@ -189,13 +188,13 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
...
@@ -189,13 +188,13 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
taosMemoryFreeClear
(
pBlock
->
pBlockAgg
);
taosMemoryFreeClear
(
pBlock
->
pBlockAgg
);
if
(
*
status
==
FUNC_DATA_REQUIRED_FILTEROUT
)
{
if
(
*
status
==
FUNC_DATA_REQUIRED_FILTEROUT
)
{
qDebug
(
"%s data block filter out, brange:%"
PRId64
"-%"
PRId64
", rows:%d"
,
GET_TASKID
(
pTaskInfo
),
pBlockInfo
->
window
.
skey
,
qDebug
(
"%s data block filter out, brange:%"
PRId64
"-%"
PRId64
", rows:%d"
,
GET_TASKID
(
pTaskInfo
),
pBlockInfo
->
window
.
ekey
,
pBlockInfo
->
rows
);
pBlockInfo
->
window
.
skey
,
pBlockInfo
->
window
.
ekey
,
pBlockInfo
->
rows
);
pCost
->
filterOutBlocks
+=
1
;
pCost
->
filterOutBlocks
+=
1
;
return
TSDB_CODE_SUCCESS
;
return
TSDB_CODE_SUCCESS
;
}
else
if
(
*
status
==
FUNC_DATA_REQUIRED_NOT_LOAD
)
{
}
else
if
(
*
status
==
FUNC_DATA_REQUIRED_NOT_LOAD
)
{
qDebug
(
"%s data block skipped, brange:%"
PRId64
"-%"
PRId64
", rows:%d"
,
GET_TASKID
(
pTaskInfo
),
pBlockInfo
->
window
.
skey
,
qDebug
(
"%s data block skipped, brange:%"
PRId64
"-%"
PRId64
", rows:%d"
,
GET_TASKID
(
pTaskInfo
),
pBlockInfo
->
window
.
ekey
,
pBlockInfo
->
rows
);
pBlockInfo
->
window
.
skey
,
pBlockInfo
->
window
.
ekey
,
pBlockInfo
->
rows
);
pCost
->
skipBlocks
+=
1
;
pCost
->
skipBlocks
+=
1
;
return
TSDB_CODE_SUCCESS
;
return
TSDB_CODE_SUCCESS
;
}
else
if
(
*
status
==
FUNC_DATA_REQUIRED_STATIS_LOAD
)
{
}
else
if
(
*
status
==
FUNC_DATA_REQUIRED_STATIS_LOAD
)
{
...
@@ -223,7 +222,7 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
...
@@ -223,7 +222,7 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
}
}
}
}
ASSERT
(
*
status
==
FUNC_DATA_REQUIRED_DATA_LOAD
);
ASSERT
(
*
status
==
FUNC_DATA_REQUIRED_DATA_LOAD
);
// todo filter data block according to the block sma data firstly
// todo filter data block according to the block sma data firstly
#if 0
#if 0
...
@@ -249,8 +248,8 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
...
@@ -249,8 +248,8 @@ int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableScanInfo,
doFilter
(
pTableScanInfo
->
pFilterNode
,
pBlock
);
doFilter
(
pTableScanInfo
->
pFilterNode
,
pBlock
);
if
(
pBlock
->
info
.
rows
==
0
)
{
if
(
pBlock
->
info
.
rows
==
0
)
{
pCost
->
filterOutBlocks
+=
1
;
pCost
->
filterOutBlocks
+=
1
;
qDebug
(
"%s data block filter out, brange:%"
PRId64
"-%"
PRId64
", rows:%d"
,
GET_TASKID
(
pTaskInfo
),
pBlockInfo
->
window
.
skey
,
qDebug
(
"%s data block filter out, brange:%"
PRId64
"-%"
PRId64
", rows:%d"
,
GET_TASKID
(
pTaskInfo
),
pBlockInfo
->
window
.
ekey
,
pBlockInfo
->
rows
);
pBlockInfo
->
window
.
skey
,
pBlockInfo
->
window
.
ekey
,
pBlockInfo
->
rows
);
}
}
return
TSDB_CODE_SUCCESS
;
return
TSDB_CODE_SUCCESS
;
...
@@ -348,9 +347,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
...
@@ -348,9 +347,9 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
setTaskStatus
(
pTaskInfo
,
TASK_NOT_COMPLETED
);
setTaskStatus
(
pTaskInfo
,
TASK_NOT_COMPLETED
);
pTableScanInfo
->
scanFlag
=
REPEAT_SCAN
;
pTableScanInfo
->
scanFlag
=
REPEAT_SCAN
;
// if (pResultRowInfo->size > 0) {
// if (pResultRowInfo->size > 0) {
// pResultRowInfo->curPos = 0;
// pResultRowInfo->curPos = 0;
// }
// }
qDebug
(
"%s start to repeat scan data blocks due to query func required, qrange:%"
PRId64
"-%"
PRId64
,
qDebug
(
"%s start to repeat scan data blocks due to query func required, qrange:%"
PRId64
"-%"
PRId64
,
GET_TASKID
(
pTaskInfo
),
pTaskInfo
->
window
.
skey
,
pTaskInfo
->
window
.
ekey
);
GET_TASKID
(
pTaskInfo
),
pTaskInfo
->
window
.
skey
,
pTaskInfo
->
window
.
ekey
);
...
@@ -367,7 +366,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
...
@@ -367,7 +366,7 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
GET_TASKID
(
pTaskInfo
),
pTaskInfo
->
window
.
skey
,
pTaskInfo
->
window
.
ekey
);
GET_TASKID
(
pTaskInfo
),
pTaskInfo
->
window
.
skey
,
pTaskInfo
->
window
.
ekey
);
if
(
pResultRowInfo
->
size
>
0
)
{
if
(
pResultRowInfo
->
size
>
0
)
{
// pResultRowInfo->curPos = pResultRowInfo->size - 1;
// pResultRowInfo->curPos = pResultRowInfo->size - 1;
}
}
p
=
doTableScanImpl
(
pOperator
,
newgroup
);
p
=
doTableScanImpl
(
pOperator
,
newgroup
);
...
@@ -376,9 +375,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
...
@@ -376,9 +375,10 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator, bool* newgroup) {
return
p
;
return
p
;
}
}
SOperatorInfo
*
createTableScanOperatorInfo
(
void
*
pTsdbReadHandle
,
int32_t
order
,
int32_t
numOfOutput
,
int32_t
dataLoadFlag
,
SOperatorInfo
*
createTableScanOperatorInfo
(
void
*
pTsdbReadHandle
,
int32_t
order
,
int32_t
numOfOutput
,
int32_t
repeatTime
,
int32_t
reverseTime
,
SArray
*
pColMatchInfo
,
SSDataBlock
*
pResBlock
,
int32_t
dataLoadFlag
,
int32_t
repeatTime
,
int32_t
reverseTime
,
SNode
*
pCondition
,
SInterval
*
pInterval
,
double
sampleRatio
,
SExecTaskInfo
*
pTaskInfo
)
{
SArray
*
pColMatchInfo
,
SSDataBlock
*
pResBlock
,
SNode
*
pCondition
,
SInterval
*
pInterval
,
double
sampleRatio
,
SExecTaskInfo
*
pTaskInfo
)
{
assert
(
repeatTime
>
0
);
assert
(
repeatTime
>
0
);
STableScanInfo
*
pInfo
=
taosMemoryCalloc
(
1
,
sizeof
(
STableScanInfo
));
STableScanInfo
*
pInfo
=
taosMemoryCalloc
(
1
,
sizeof
(
STableScanInfo
));
...
@@ -393,7 +393,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
...
@@ -393,7 +393,7 @@ SOperatorInfo* createTableScanOperatorInfo(void* pTsdbReadHandle, int32_t order,
pInfo
->
interval
=
*
pInterval
;
pInfo
->
interval
=
*
pInterval
;
pInfo
->
sampleRatio
=
sampleRatio
;
pInfo
->
sampleRatio
=
sampleRatio
;
pInfo
->
dataBlockLoadFlag
=
dataLoadFlag
;
pInfo
->
dataBlockLoadFlag
=
dataLoadFlag
;
pInfo
->
pResBlock
=
pResBlock
;
pInfo
->
pResBlock
=
pResBlock
;
pInfo
->
pFilterNode
=
pCondition
;
pInfo
->
pFilterNode
=
pCondition
;
pInfo
->
dataReader
=
pTsdbReadHandle
;
pInfo
->
dataReader
=
pTsdbReadHandle
;
...
@@ -463,25 +463,25 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) {
...
@@ -463,25 +463,25 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator, bool* newgroup) {
tableBlockDist
.
minRows
=
INT_MAX
;
tableBlockDist
.
minRows
=
INT_MAX
;
tsdbGetFileBlocksDistInfo
(
pTableScanInfo
->
dataReader
,
&
tableBlockDist
);
tsdbGetFileBlocksDistInfo
(
pTableScanInfo
->
dataReader
,
&
tableBlockDist
);
tableBlockDist
.
numOfRowsInMemTable
=
(
int32_t
)
tsdbGetNumOfRowsInMemTable
(
pTableScanInfo
->
dataReader
);
tableBlockDist
.
numOfRowsInMemTable
=
(
int32_t
)
tsdbGetNumOfRowsInMemTable
(
pTableScanInfo
->
dataReader
);
SSDataBlock
*
pBlock
=
pTableScanInfo
->
pResBlock
;
SSDataBlock
*
pBlock
=
pTableScanInfo
->
pResBlock
;
pBlock
->
info
.
rows
=
1
;
pBlock
->
info
.
rows
=
1
;
pBlock
->
info
.
numOfCols
=
1
;
pBlock
->
info
.
numOfCols
=
1
;
// SBufferWriter bw = tbufInitWriter(NULL, false);
// SBufferWriter bw = tbufInitWriter(NULL, false);
// blockDistInfoToBinary(&tableBlockDist, &bw);
// blockDistInfoToBinary(&tableBlockDist, &bw);
SColumnInfoData
*
pColInfo
=
taosArrayGet
(
pBlock
->
pDataBlock
,
0
);
SColumnInfoData
*
pColInfo
=
taosArrayGet
(
pBlock
->
pDataBlock
,
0
);
// int32_t len = (int32_t) tbufTell(&bw);
// int32_t len = (int32_t) tbufTell(&bw);
// pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t));
// pColInfo->pData = taosMemoryMalloc(len + sizeof(int32_t));
// *(int32_t*) pColInfo->pData = len;
// *(int32_t*) pColInfo->pData = len;
// memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len);
// memcpy(pColInfo->pData + sizeof(int32_t), tbufGetData(&bw, false), len);
//
//
// tbufCloseWriter(&bw);
// tbufCloseWriter(&bw);
// SArray* g = GET_TABLEGROUP(pOperator->, 0);
// SArray* g = GET_TABLEGROUP(pOperator->, 0);
// pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0);
// pOperator->pRuntimeEnv->current = taosArrayGetP(g, 0);
pOperator
->
status
=
OP_EXEC_DONE
;
pOperator
->
status
=
OP_EXEC_DONE
;
return
pBlock
;
return
pBlock
;
...
@@ -496,13 +496,13 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo*
...
@@ -496,13 +496,13 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo*
}
}
pInfo
->
dataReader
=
dataReader
;
pInfo
->
dataReader
=
dataReader
;
// pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
// pInfo->block.pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData));
SColumnInfoData
infoData
=
{
0
};
SColumnInfoData
infoData
=
{
0
};
infoData
.
info
.
type
=
TSDB_DATA_TYPE_BINARY
;
infoData
.
info
.
type
=
TSDB_DATA_TYPE_BINARY
;
infoData
.
info
.
bytes
=
1024
;
infoData
.
info
.
bytes
=
1024
;
infoData
.
info
.
colId
=
0
;
infoData
.
info
.
colId
=
0
;
// taosArrayPush(pInfo->block.pDataBlock, &infoData);
// taosArrayPush(pInfo->block.pDataBlock, &infoData);
pOperator
->
name
=
"DataBlockInfoScanOperator"
;
pOperator
->
name
=
"DataBlockInfoScanOperator"
;
// pOperator->operatorType = OP_TableBlockInfoScan;
// pOperator->operatorType = OP_TableBlockInfoScan;
...
@@ -516,7 +516,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo*
...
@@ -516,7 +516,7 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SExecTaskInfo*
return
pOperator
;
return
pOperator
;
_error:
_error:
taosMemoryFreeClear
(
pInfo
);
taosMemoryFreeClear
(
pInfo
);
taosMemoryFreeClear
(
pOperator
);
taosMemoryFreeClear
(
pOperator
);
return
NULL
;
return
NULL
;
...
@@ -558,29 +558,42 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup)
...
@@ -558,29 +558,42 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup)
blockDataCleanup
(
pInfo
->
pRes
);
blockDataCleanup
(
pInfo
->
pRes
);
while
(
tqNextDataBlock
(
pInfo
->
readerHandle
))
{
while
(
tqNextDataBlock
(
pInfo
->
readerHandle
))
{
pTaskInfo
->
code
=
tqRetrieveDataBlockInfo
(
pInfo
->
readerHandle
,
pBlockInfo
);
SArray
*
pCols
=
NULL
;
if
(
pTaskInfo
->
code
!=
TSDB_CODE_SUCCESS
)
{
uint64_t
groupId
;
terrno
=
pTaskInfo
->
code
;
int32_t
numOfRows
;
pOperator
->
status
=
OP_EXEC_DONE
;
int32_t
code
=
tqRetrieveDataBlock
(
&
pCols
,
pInfo
->
readerHandle
,
&
groupId
,
&
numOfRows
);
return
NULL
;
}
if
(
pBlockInfo
->
rows
==
0
)
{
if
(
code
!=
TSDB_CODE_SUCCESS
||
numOfRows
==
0
)
{
break
;
pTaskInfo
->
code
=
code
;
return
NULL
;
}
}
SArray
*
pCols
=
tqRetrieveDataBlock
(
pInfo
->
readerHandle
);
pInfo
->
pRes
->
info
.
groupId
=
groupId
;
pInfo
->
pRes
->
info
.
rows
=
numOfRows
;
int32_t
numOfCols
=
pInfo
->
pRes
->
info
.
numOfCols
;
int32_t
numOfCols
=
pInfo
->
pRes
->
info
.
numOfCols
;
for
(
int32_t
i
=
0
;
i
<
numOfCols
;
++
i
)
{
for
(
int32_t
i
=
0
;
i
<
numOfCols
;
++
i
)
{
SColumnInfoData
*
p
=
taosArrayGet
(
pCols
,
i
);
SColMatchInfo
*
pColMatchInfo
=
taosArrayGet
(
pInfo
->
pColMatchInfo
,
i
);
SColMatchInfo
*
pColMatchInfo
=
taosArrayGet
(
pInfo
->
pColMatchInfo
,
i
);
if
(
!
pColMatchInfo
->
output
)
{
if
(
!
pColMatchInfo
->
output
)
{
continue
;
continue
;
}
}
ASSERT
(
pColMatchInfo
->
colId
==
p
->
info
.
colId
);
bool
colExists
=
false
;
taosArraySet
(
pInfo
->
pRes
->
pDataBlock
,
pColMatchInfo
->
targetSlotId
,
p
);
for
(
int32_t
j
=
0
;
j
<
taosArrayGetSize
(
pCols
);
++
j
)
{
SColumnInfoData
*
pResCol
=
taosArrayGet
(
pCols
,
j
);
if
(
pResCol
->
info
.
colId
==
pColMatchInfo
->
colId
)
{
taosArraySet
(
pInfo
->
pRes
->
pDataBlock
,
pColMatchInfo
->
targetSlotId
,
pResCol
);
colExists
=
true
;
break
;
}
}
// the required column does not exists in submit block, let's set it to be all null value
if
(
!
colExists
)
{
SColumnInfoData
*
pDst
=
taosArrayGet
(
pInfo
->
pRes
->
pDataBlock
,
pColMatchInfo
->
targetSlotId
);
colInfoDataEnsureCapacity
(
pDst
,
0
,
pBlockInfo
->
rows
);
colDataAppendNNULL
(
pDst
,
0
,
pBlockInfo
->
rows
);
}
}
}
if
(
pInfo
->
pRes
->
pDataBlock
==
NULL
)
{
if
(
pInfo
->
pRes
->
pDataBlock
==
NULL
)
{
...
@@ -605,7 +618,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup)
...
@@ -605,7 +618,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator, bool* newgroup)
}
}
}
}
SOperatorInfo
*
createStreamScanOperatorInfo
(
void
*
streamReadHandle
,
SSDataBlock
*
pResBlock
,
SArray
*
pColList
,
SArray
*
pTableIdList
,
SExecTaskInfo
*
pTaskInfo
)
{
SOperatorInfo
*
createStreamScanOperatorInfo
(
void
*
streamReadHandle
,
SSDataBlock
*
pResBlock
,
SArray
*
pColList
,
SArray
*
pTableIdList
,
SExecTaskInfo
*
pTaskInfo
)
{
SStreamBlockScanInfo
*
pInfo
=
taosMemoryCalloc
(
1
,
sizeof
(
SStreamBlockScanInfo
));
SStreamBlockScanInfo
*
pInfo
=
taosMemoryCalloc
(
1
,
sizeof
(
SStreamBlockScanInfo
));
SOperatorInfo
*
pOperator
=
taosMemoryCalloc
(
1
,
sizeof
(
SOperatorInfo
));
SOperatorInfo
*
pOperator
=
taosMemoryCalloc
(
1
,
sizeof
(
SOperatorInfo
));
if
(
pInfo
==
NULL
||
pOperator
==
NULL
)
{
if
(
pInfo
==
NULL
||
pOperator
==
NULL
)
{
...
@@ -618,7 +632,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
...
@@ -618,7 +632,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* streamReadHandle, SSDataBlock*
int32_t
numOfOutput
=
taosArrayGetSize
(
pColList
);
int32_t
numOfOutput
=
taosArrayGetSize
(
pColList
);
SArray
*
pColIds
=
taosArrayInit
(
4
,
sizeof
(
int16_t
));
SArray
*
pColIds
=
taosArrayInit
(
4
,
sizeof
(
int16_t
));
for
(
int32_t
i
=
0
;
i
<
numOfOutput
;
++
i
)
{
for
(
int32_t
i
=
0
;
i
<
numOfOutput
;
++
i
)
{
int16_t
*
id
=
taosArrayGet
(
pColList
,
i
);
int16_t
*
id
=
taosArrayGet
(
pColList
,
i
);
taosArrayPush
(
pColIds
,
id
);
taosArrayPush
(
pColIds
,
id
);
}
}
...
@@ -828,7 +842,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
...
@@ -828,7 +842,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator, bool* newgroup) {
SColumnInfoData
*
pColInfoData
=
taosArrayGet
(
pInfo
->
pRes
->
pDataBlock
,
i
);
SColumnInfoData
*
pColInfoData
=
taosArrayGet
(
pInfo
->
pRes
->
pDataBlock
,
i
);
int64_t
tmp
=
0
;
int64_t
tmp
=
0
;
char
t
[
10
]
=
{
0
};
char
t
[
10
]
=
{
0
};
STR_TO_VARSTR
(
t
,
"_"
);
//TODO
STR_TO_VARSTR
(
t
,
"_"
);
//
TODO
if
(
IS_VAR_DATA_TYPE
(
pColInfoData
->
info
.
type
))
{
if
(
IS_VAR_DATA_TYPE
(
pColInfoData
->
info
.
type
))
{
colDataAppend
(
pColInfoData
,
numOfRows
,
t
,
false
);
colDataAppend
(
pColInfoData
,
numOfRows
,
t
,
false
);
}
else
{
}
else
{
...
@@ -988,7 +1002,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
...
@@ -988,7 +1002,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* pSysTableReadHandle, SSDataB
tableType
=
TSDB_MGMT_TABLE_QUERIES
;
tableType
=
TSDB_MGMT_TABLE_QUERIES
;
}
else
if
(
strncasecmp
(
name
,
TSDB_INS_TABLE_VNODES
,
tListLen
(
pName
->
tname
))
==
0
)
{
}
else
if
(
strncasecmp
(
name
,
TSDB_INS_TABLE_VNODES
,
tListLen
(
pName
->
tname
))
==
0
)
{
tableType
=
TSDB_MGMT_TABLE_VNODES
;
tableType
=
TSDB_MGMT_TABLE_VNODES
;
}
else
{
}
else
{
ASSERT
(
0
);
ASSERT
(
0
);
}
}
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录