Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
42a512c3
T
TDengine
项目概览
taosdata
/
TDengine
1 年多 前同步成功
通知
1185
Star
22016
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
提交
42a512c3
编写于
12月 02, 2022
作者:
H
Haojun Liao
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
refactor: do some internal refactor.
上级
83782fb2
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
116 addition
and
143 deletion
+116
-143
source/dnode/vnode/inc/vnode.h
source/dnode/vnode/inc/vnode.h
+11
-11
source/dnode/vnode/src/tsdb/tsdbRead.c
source/dnode/vnode/src/tsdb/tsdbRead.c
+80
-96
source/libs/executor/src/executor.c
source/libs/executor/src/executor.c
+12
-6
source/libs/executor/src/scanoperator.c
source/libs/executor/src/scanoperator.c
+10
-27
tests/system-test/2-query/unique.py
tests/system-test/2-query/unique.py
+3
-3
未找到文件。
source/dnode/vnode/inc/vnode.h
浏览文件 @
42a512c3
...
...
@@ -161,17 +161,17 @@ int32_t tsdbSetTableList(STsdbReader *pReader, const void *pTableList, int32_t n
int32_t
tsdbReaderOpen
(
SVnode
*
pVnode
,
SQueryTableDataCond
*
pCond
,
void
*
pTableList
,
int32_t
numOfTables
,
SSDataBlock
*
pResBlock
,
STsdbReader
**
ppReader
,
const
char
*
idstr
);
void
tsdbReaderClose
(
STsdbReader
*
pReader
);
bool
tsdbNextDataBlock
(
STsdbReader
*
pReader
);
void
tsdbRetrieveDataBlockInfo
(
const
STsdbReader
*
pReader
,
int32_t
*
rows
,
uint64_t
*
uid
,
STimeWindow
*
pWindow
);
int32_t
tsdbRetrieveDatablockSMA
(
STsdbReader
*
pReader
,
SColumnDataAgg
***
pBlockStatis
,
bool
*
allHave
);
S
Array
*
tsdbRetrieveDataBlock
(
STsdbReader
*
pTsdbReadHandle
,
SArray
*
pColumnIdList
);
int32_t
tsdbReaderReset
(
STsdbReader
*
pReader
,
SQueryTableDataCond
*
pCond
);
int32_t
tsdbGetFileBlocksDistInfo
(
STsdbReader
*
pReader
,
STableBlockDistInfo
*
pTableBlockInfo
);
int64_t
tsdbGetNumOfRowsInMemTable
(
STsdbReader
*
pHandle
);
void
*
tsdbGetIdx
(
SMeta
*
pMeta
);
void
*
tsdbGetIvtIdx
(
SMeta
*
pMeta
);
uint64_t
getReaderMaxVersion
(
STsdbReader
*
pReader
);
void
tsdbReaderClose
(
STsdbReader
*
pReader
);
bool
tsdbNextDataBlock
(
STsdbReader
*
pReader
);
void
tsdbRetrieveDataBlockInfo
(
const
STsdbReader
*
pReader
,
int32_t
*
rows
,
uint64_t
*
uid
,
STimeWindow
*
pWindow
);
int32_t
tsdbRetrieveDatablockSMA
(
STsdbReader
*
pReader
,
SColumnDataAgg
***
pBlockSMA
,
bool
*
allHave
);
S
SDataBlock
*
tsdbRetrieveDataBlock
(
STsdbReader
*
pTsdbReadHandle
,
SArray
*
pColumnIdList
);
int32_t
tsdbReaderReset
(
STsdbReader
*
pReader
,
SQueryTableDataCond
*
pCond
);
int32_t
tsdbGetFileBlocksDistInfo
(
STsdbReader
*
pReader
,
STableBlockDistInfo
*
pTableBlockInfo
);
int64_t
tsdbGetNumOfRowsInMemTable
(
STsdbReader
*
pHandle
);
void
*
tsdbGetIdx
(
SMeta
*
pMeta
);
void
*
tsdbGetIvtIdx
(
SMeta
*
pMeta
);
uint64_t
getReaderMaxVersion
(
STsdbReader
*
pReader
);
int32_t
tsdbCacherowsReaderOpen
(
void
*
pVnode
,
int32_t
type
,
void
*
pTableIdList
,
int32_t
numOfTables
,
int32_t
numOfCols
,
uint64_t
suid
,
void
**
pReader
);
...
...
source/dnode/vnode/src/tsdb/tsdbRead.c
浏览文件 @
42a512c3
...
...
@@ -81,12 +81,15 @@ typedef struct SIOCostSummary {
double
createScanInfoList
;
}
SIOCostSummary
;
typedef
struct
SColInfo
{
int16_t
colId
;
int16_t
slotId
;
}
SColInfo
;
typedef
struct
SBlockLoadSuppInfo
{
SArray
*
pColAgg
;
SColumnDataAgg
tsColAgg
;
SColumnDataAgg
**
plist
;
// todo remove this
int16_t
*
colIds
;
// column ids for loading file block data
int16_t
*
slotIds
;
// the ordinal index in the destination SSDataBlock
SColInfo
*
colInfo
;
// column ids for loading file block data
int32_t
numOfCols
;
char
**
buildBuf
;
// build string tmp buffer, todo remove it later after all string format being updated.
bool
smaValid
;
// the sma on all queried columns are activated
...
...
@@ -159,6 +162,7 @@ struct STsdbReader {
STsdb
*
pTsdb
;
uint64_t
suid
;
int16_t
order
;
bool
freeBlock
;
STimeWindow
window
;
// the primary query time window that applies to all queries
SSDataBlock
*
pResBlock
;
int32_t
capacity
;
...
...
@@ -218,22 +222,21 @@ static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWi
static
int32_t
setColumnIdSlotList
(
SBlockLoadSuppInfo
*
pSupInfo
,
SColumnInfo
*
pCols
,
const
int32_t
*
pSlotIdList
,
int32_t
numOfCols
)
{
pSupInfo
->
smaValid
=
true
;
pSupInfo
->
numOfCols
=
numOfCols
;
pSupInfo
->
colIds
=
taosMemoryMalloc
(
numOfCols
*
sizeof
(
int16_t
));
pSupInfo
->
buildBuf
=
taosMemoryCalloc
(
numOfCols
,
POINTER_BYTES
);
pSupInfo
->
slotIds
=
taosMemoryMalloc
(
numOfCols
*
sizeof
(
int16_t
));
if
(
pSupInfo
->
buildBuf
==
NULL
||
pSupInfo
->
colIds
==
NULL
||
pSupInfo
->
slotIds
==
NULL
)
{
taosMemoryFree
(
pSupInfo
->
colIds
);
taosMemoryFree
(
pSupInfo
->
buildBuf
);
taosMemoryFree
(
pSupInfo
->
slotIds
);
pSupInfo
->
colInfo
=
taosMemoryMalloc
(
numOfCols
*
(
sizeof
(
SColInfo
)
+
POINTER_BYTES
));
if
(
pSupInfo
->
colInfo
==
NULL
)
{
taosMemoryFree
(
pSupInfo
->
colInfo
);
return
TSDB_CODE_OUT_OF_MEMORY
;
}
pSupInfo
->
buildBuf
=
(
char
**
)((
char
*
)
pSupInfo
->
colInfo
+
(
sizeof
(
SColInfo
)
*
numOfCols
));
for
(
int32_t
i
=
0
;
i
<
numOfCols
;
++
i
)
{
pSupInfo
->
colI
ds
[
i
]
=
pCols
[
i
].
colId
;
pSupInfo
->
slotIds
[
i
]
=
pSlotIdList
[
i
];
pSupInfo
->
colI
nfo
[
i
].
colId
=
pCols
[
i
].
colId
;
pSupInfo
->
colInfo
[
i
].
slotId
=
pSlotIdList
[
i
];
if
(
IS_VAR_DATA_TYPE
(
pCols
[
i
].
type
))
{
pSupInfo
->
buildBuf
[
i
]
=
taosMemoryMalloc
(
pCols
[
i
].
bytes
);
}
else
{
pSupInfo
->
buildBuf
[
i
]
=
NULL
;
}
}
...
...
@@ -245,7 +248,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
while
(
i
<
pSchema
->
numOfCols
&&
j
<
pSupInfo
->
numOfCols
)
{
STColumn
*
pTCol
=
&
pSchema
->
columns
[
i
];
if
(
pTCol
->
colId
==
pSupInfo
->
colI
ds
[
j
]
)
{
if
(
pTCol
->
colId
==
pSupInfo
->
colI
nfo
[
j
].
colId
)
{
if
(
!
IS_BSMA_ON
(
pTCol
))
{
pSupInfo
->
smaValid
=
false
;
return
;
...
...
@@ -253,7 +256,7 @@ static void updateBlockSMAInfo(STSchema* pSchema, SBlockLoadSuppInfo* pSupInfo)
i
+=
1
;
j
+=
1
;
}
else
if
(
pTCol
->
colId
<
pSupInfo
->
colI
ds
[
j
]
)
{
}
else
if
(
pTCol
->
colId
<
pSupInfo
->
colI
nfo
[
j
].
colId
)
{
// do nothing
i
+=
1
;
}
else
{
...
...
@@ -455,7 +458,7 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb
if
(
pLReader
->
pInfo
==
NULL
)
{
// here we ignore the first column, which is always be the primary timestamp column
pLReader
->
pInfo
=
tCreateLastBlockLoadInfo
(
pReader
->
pSchema
,
&
pReader
->
suppInfo
.
colI
ds
[
1
]
,
pReader
->
suppInfo
.
numOfCols
-
1
);
tCreateLastBlockLoadInfo
(
pReader
->
pSchema
,
&
pReader
->
suppInfo
.
colI
nfo
[
1
].
colId
,
pReader
->
suppInfo
.
numOfCols
-
1
);
if
(
pLReader
->
pInfo
==
NULL
)
{
tsdbDebug
(
"init fileset iterator failed, code:%s %s"
,
tstrerror
(
terrno
),
pReader
->
idStr
);
return
terrno
;
...
...
@@ -594,14 +597,22 @@ static int32_t tsdbReaderCreate(SVnode* pVnode, SQueryTableDataCond* pCond, STsd
pReader
->
blockInfoBuf
.
numPerBucket
=
1000
;
// 1000 tables per bucket
ASSERT
(
pCond
->
numOfCols
>
0
);
if
(
pReader
->
pResBlock
==
NULL
)
{
pReader
->
freeBlock
=
true
;
pReader
->
pResBlock
=
createResBlock
(
pCond
,
pReader
->
capacity
);
if
(
pReader
->
pResBlock
==
NULL
)
{
code
=
terrno
;
goto
_end
;
}
}
// todo refactor.
limitOutputBufferSize
(
pCond
,
&
pReader
->
capacity
);
// allocate buffer in order to load data blocks from file
SBlockLoadSuppInfo
*
pSup
=
&
pReader
->
suppInfo
;
pSup
->
pColAgg
=
taosArrayInit
(
pCond
->
numOfCols
,
sizeof
(
SColumnDataAgg
));
pSup
->
plist
=
taosMemoryCalloc
(
pCond
->
numOfCols
,
POINTER_BYTES
);
if
(
pSup
->
pColAgg
==
NULL
||
pSup
->
plist
==
NULL
)
{
if
(
pSup
->
pColAgg
==
NULL
)
{
code
=
TSDB_CODE_OUT_OF_MEMORY
;
goto
_end
;
}
...
...
@@ -1083,8 +1094,8 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
int32_t
i
=
0
;
int32_t
rowIndex
=
0
;
SColumnInfoData
*
pColData
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
if
(
pSupInfo
->
colI
ds
[
i
]
==
PRIMARYKEY_TIMESTAMP_COL_ID
)
{
SColumnInfoData
*
pColData
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
if
(
pSupInfo
->
colI
nfo
[
i
].
colId
==
PRIMARYKEY_TIMESTAMP_COL_ID
)
{
copyPrimaryTsCol
(
pBlockData
,
pDumpInfo
,
pColData
,
dumpedRows
,
asc
);
i
+=
1
;
}
...
...
@@ -1095,10 +1106,10 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
rowIndex
=
0
;
SColData
*
pData
=
tBlockDataGetColDataByIdx
(
pBlockData
,
colIndex
);
if
(
pData
->
cid
<
pSupInfo
->
colI
ds
[
i
]
)
{
if
(
pData
->
cid
<
pSupInfo
->
colI
nfo
[
i
].
colId
)
{
colIndex
+=
1
;
}
else
if
(
pData
->
cid
==
pSupInfo
->
colI
ds
[
i
]
)
{
pColData
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
}
else
if
(
pData
->
cid
==
pSupInfo
->
colI
nfo
[
i
].
colId
)
{
pColData
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
if
(
pData
->
flag
==
HAS_NONE
||
pData
->
flag
==
HAS_NULL
||
pData
->
flag
==
(
HAS_NULL
|
HAS_NONE
))
{
colDataAppendNNULL
(
pColData
,
0
,
dumpedRows
);
...
...
@@ -1116,7 +1127,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
colIndex
+=
1
;
i
+=
1
;
}
else
{
// the specified column does not exist in file block, fill with null data
pColData
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
pColData
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
colDataAppendNNULL
(
pColData
,
0
,
dumpedRows
);
i
+=
1
;
}
...
...
@@ -1124,7 +1135,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
// fill the mis-matched columns with null value
while
(
i
<
numOfOutputCols
)
{
pColData
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
pColData
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
colDataAppendNNULL
(
pColData
,
0
,
dumpedRows
);
i
+=
1
;
}
...
...
@@ -1162,7 +1173,7 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI
tBlockDataReset
(
pBlockData
);
TABLEID
tid
=
{.
suid
=
pReader
->
suid
,
.
uid
=
uid
};
int32_t
code
=
tBlockDataInit
(
pBlockData
,
&
tid
,
pReader
->
pSchema
,
&
pReader
->
suppInfo
.
colI
ds
[
1
]
,
pReader
->
suppInfo
.
numOfCols
-
1
);
tBlockDataInit
(
pBlockData
,
&
tid
,
pReader
->
pSchema
,
&
pReader
->
suppInfo
.
colI
nfo
[
1
].
colId
,
pReader
->
suppInfo
.
numOfCols
-
1
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
return
code
;
}
...
...
@@ -1621,7 +1632,7 @@ static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo*
int64_t
st
=
taosGetTimestampUs
();
int32_t
code
=
buildDataBlockFromBufImpl
(
pBlockScanInfo
,
endKey
,
pReader
->
capacity
,
pReader
);
blockDataUpdateTsWindow
(
pBlock
,
pReader
->
suppInfo
.
slotIds
[
0
]
);
blockDataUpdateTsWindow
(
pBlock
,
pReader
->
suppInfo
.
colInfo
[
0
].
slotId
);
pBlock
->
info
.
id
.
uid
=
pBlockScanInfo
->
uid
;
setComposedBlockFlag
(
pReader
,
true
);
...
...
@@ -2493,7 +2504,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) {
_end:
pResBlock
->
info
.
id
.
uid
=
(
pBlockScanInfo
!=
NULL
)
?
pBlockScanInfo
->
uid
:
0
;
blockDataUpdateTsWindow
(
pResBlock
,
pReader
->
suppInfo
.
slotIds
[
0
]
);
blockDataUpdateTsWindow
(
pResBlock
,
pReader
->
suppInfo
.
colInfo
[
0
].
slotId
);
setComposedBlockFlag
(
pReader
,
true
);
double
el
=
(
taosGetTimestampUs
()
-
st
)
/
1000
.
0
;
...
...
@@ -3542,24 +3553,24 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
SColVal
colVal
=
{
0
};
int32_t
i
=
0
,
j
=
0
;
if
(
pSupInfo
->
colI
ds
[
i
]
==
PRIMARYKEY_TIMESTAMP_COL_ID
)
{
SColumnInfoData
*
pColData
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
if
(
pSupInfo
->
colI
nfo
[
i
].
colId
==
PRIMARYKEY_TIMESTAMP_COL_ID
)
{
SColumnInfoData
*
pColData
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
((
int64_t
*
)
pColData
->
pData
)[
outputRowIndex
]
=
pTSRow
->
ts
;
i
+=
1
;
}
while
(
i
<
pSupInfo
->
numOfCols
&&
j
<
pSchema
->
numOfCols
)
{
col_id_t
colId
=
pSupInfo
->
colI
ds
[
i
]
;
col_id_t
colId
=
pSupInfo
->
colI
nfo
[
i
].
colId
;
if
(
colId
==
pSchema
->
columns
[
j
].
colId
)
{
SColumnInfoData
*
pColInfoData
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
SColumnInfoData
*
pColInfoData
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
tTSRowGetVal
(
pTSRow
,
pSchema
,
j
,
&
colVal
);
doCopyColVal
(
pColInfoData
,
outputRowIndex
,
i
,
&
colVal
,
pSupInfo
);
i
+=
1
;
j
+=
1
;
}
else
if
(
colId
<
pSchema
->
columns
[
j
].
colId
)
{
SColumnInfoData
*
pColInfoData
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
SColumnInfoData
*
pColInfoData
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
colDataAppendNULL
(
pColInfoData
,
outputRowIndex
);
i
+=
1
;
...
...
@@ -3570,7 +3581,7 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow*
// set null value since current column does not exist in the "pSchema"
while
(
i
<
pSupInfo
->
numOfCols
)
{
SColumnInfoData
*
pColInfoData
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
SColumnInfoData
*
pColInfoData
=
taosArrayGet
(
pBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
colDataAppendNULL
(
pColInfoData
,
outputRowIndex
);
i
+=
1
;
}
...
...
@@ -3586,8 +3597,8 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
int32_t
outputRowIndex
=
pResBlock
->
info
.
rows
;
SBlockLoadSuppInfo
*
pSupInfo
=
&
pReader
->
suppInfo
;
if
(
pReader
->
suppInfo
.
colI
ds
[
i
]
==
PRIMARYKEY_TIMESTAMP_COL_ID
)
{
SColumnInfoData
*
pColData
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
if
(
pReader
->
suppInfo
.
colI
nfo
[
i
].
colId
==
PRIMARYKEY_TIMESTAMP_COL_ID
)
{
SColumnInfoData
*
pColData
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
((
int64_t
*
)
pColData
->
pData
)[
outputRowIndex
]
=
pBlockData
->
aTSKEY
[
rowIndex
];
i
+=
1
;
}
...
...
@@ -3598,13 +3609,13 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
while
(
i
<
numOfOutputCols
&&
j
<
numOfInputCols
)
{
SColData
*
pData
=
tBlockDataGetColDataByIdx
(
pBlockData
,
j
);
if
(
pData
->
cid
<
pSupInfo
->
colI
ds
[
i
]
)
{
if
(
pData
->
cid
<
pSupInfo
->
colI
nfo
[
i
].
colId
)
{
j
+=
1
;
continue
;
}
SColumnInfoData
*
pCol
=
TARRAY_GET_ELEM
(
pResBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
if
(
pData
->
cid
==
pSupInfo
->
colI
ds
[
i
]
)
{
SColumnInfoData
*
pCol
=
TARRAY_GET_ELEM
(
pResBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
if
(
pData
->
cid
==
pSupInfo
->
colI
nfo
[
i
].
colId
)
{
tColDataGetValue
(
pData
,
rowIndex
,
&
cv
);
doCopyColVal
(
pCol
,
outputRowIndex
,
i
,
&
cv
,
pSupInfo
);
j
+=
1
;
...
...
@@ -3617,7 +3628,7 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S
}
while
(
i
<
numOfOutputCols
)
{
SColumnInfoData
*
pCol
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
slotIds
[
i
]
);
SColumnInfoData
*
pCol
=
taosArrayGet
(
pResBlock
->
pDataBlock
,
pSupInfo
->
colInfo
[
i
].
slotId
);
colDataAppendNULL
(
pCol
,
outputRowIndex
);
i
+=
1
;
}
...
...
@@ -3723,7 +3734,14 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, void* pTableL
pCond
->
twindows
.
ekey
-=
1
;
}
int32_t
code
=
tsdbReaderCreate
(
pVnode
,
pCond
,
ppReader
,
pResBlock
->
info
.
capacity
,
pResBlock
,
idstr
);
int32_t
capacity
=
0
;
if
(
pResBlock
==
NULL
)
{
capacity
=
4096
;
}
else
{
capacity
=
pResBlock
->
info
.
capacity
;
}
int32_t
code
=
tsdbReaderCreate
(
pVnode
,
pCond
,
ppReader
,
capacity
,
pResBlock
,
idstr
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
goto
_err
;
}
...
...
@@ -3868,10 +3886,6 @@ void tsdbReaderClose(STsdbReader* pReader) {
SBlockLoadSuppInfo
*
pSupInfo
=
&
pReader
->
suppInfo
;
taosMemoryFreeClear
(
pSupInfo
->
plist
);
taosMemoryFree
(
pSupInfo
->
colIds
);
taosMemoryFree
(
pSupInfo
->
slotIds
);
taosArrayDestroy
(
pSupInfo
->
pColAgg
);
for
(
int32_t
i
=
0
;
i
<
pSupInfo
->
numOfCols
;
++
i
)
{
if
(
pSupInfo
->
buildBuf
[
i
]
!=
NULL
)
{
...
...
@@ -3879,9 +3893,12 @@ void tsdbReaderClose(STsdbReader* pReader) {
}
}
taosMemoryFree
(
pSupInfo
->
buildBuf
);
tBlockDataDestroy
(
&
pReader
->
status
.
fileBlockData
,
true
);
if
(
pReader
->
freeBlock
)
{
pReader
->
pResBlock
=
blockDataDestroy
(
pReader
->
pResBlock
);
}
taosMemoryFree
(
pSupInfo
->
colInfo
);
tBlockDataDestroy
(
&
pReader
->
status
.
fileBlockData
,
true
);
cleanupDataBlockIterator
(
&
pReader
->
status
.
blockIter
);
size_t
numOfTables
=
taosHashGetSize
(
pReader
->
status
.
pTableMap
);
...
...
@@ -4025,18 +4042,18 @@ void tsdbRetrieveDataBlockInfo(const STsdbReader* pReader, int32_t* rows, uint64
}
}
int32_t
tsdbRetrieveDatablockSMA
(
STsdbReader
*
pReader
,
SColumnDataAgg
***
pBlockStatis
,
bool
*
allHave
)
{
int32_t
tsdbRetrieveDatablockSMA
(
STsdbReader
*
pReader
,
SColumnDataAgg
***
pBlockSMA
,
bool
*
allHave
)
{
int32_t
code
=
0
;
*
allHave
=
false
;
if
(
pReader
->
type
==
TIMEWINDOW_RANGE_EXTERNAL
)
{
*
pBlockS
tatis
=
NULL
;
*
pBlockS
MA
=
NULL
;
return
TSDB_CODE_SUCCESS
;
}
// there is no statistics data for composed block
if
(
pReader
->
status
.
composedDataBlock
||
(
!
pReader
->
suppInfo
.
smaValid
))
{
*
pBlockS
tatis
=
NULL
;
*
pBlockS
MA
=
NULL
;
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -4054,7 +4071,7 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
return
code
;
}
}
else
{
*
pBlockS
tatis
=
NULL
;
*
pBlockS
MA
=
NULL
;
return
TSDB_CODE_SUCCESS
;
}
...
...
@@ -4067,32 +4084,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
pTsAgg
->
colId
=
PRIMARYKEY_TIMESTAMP_COL_ID
;
pTsAgg
->
min
=
pReader
->
pResBlock
->
info
.
window
.
skey
;
pTsAgg
->
max
=
pReader
->
pResBlock
->
info
.
window
.
ekey
;
pSup
->
plist
[
0
]
=
pTsAgg
;
// update the number of NULL data rows
size_t
numOfCols
=
pSup
->
numOfCols
;
int32_t
i
=
0
,
j
=
0
;
size_t
size
=
taosArrayGetSize
(
pSup
->
pColAgg
);
#if 0
while (j < numOfCols && i < size) {
SColumnDataAgg* pAgg = taosArrayGet(pSup->pColAgg, i);
if (pAgg->colId == pSup->colIds[j]) {
if (IS_BSMA_ON(&(pReader->pSchema->columns[i]))) {
pSup->plist[j] = pAgg;
} else {
*allHave = false;
break;
}
i += 1;
j += 1;
} else if (pAgg->colId < pSup->colIds[j]) {
i += 1;
} else if (pSup->colIds[j] < pAgg->colId) {
j += 1;
}
}
#else
SSDataBlock
*
pResBlock
=
pReader
->
pResBlock
;
if
(
pResBlock
->
pBlockAgg
==
NULL
)
{
...
...
@@ -4100,52 +4097,40 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS
pResBlock
->
pBlockAgg
=
taosMemoryCalloc
(
num
,
sizeof
(
SColumnDataAgg
));
}
// fill the all null data column
// SArray* pNewAggList = taosArrayInit(numOfCols, sizeof(SColumnDataAgg));
while
(
j
<
numOfCols
&&
i
<
size
)
{
SColumnDataAgg
*
pAgg
=
taosArrayGet
(
pSup
->
pColAgg
,
i
);
if
(
pAgg
->
colId
==
pSup
->
colI
ds
[
j
]
)
{
pResBlock
->
pBlockAgg
[
pSup
->
slotIds
[
j
]
]
=
pAgg
;
if
(
pAgg
->
colId
==
pSup
->
colI
nfo
[
j
].
colId
)
{
pResBlock
->
pBlockAgg
[
pSup
->
colInfo
[
j
].
slotId
]
=
pAgg
;
i
+=
1
;
j
+=
1
;
}
else
if
(
pAgg
->
colId
<
pSup
->
colI
ds
[
j
]
)
{
}
else
if
(
pAgg
->
colId
<
pSup
->
colI
nfo
[
j
].
colId
)
{
i
+=
1
;
}
else
if
(
pSup
->
colI
ds
[
j
]
<
pAgg
->
colId
)
{
if
(
pSup
->
colI
ds
[
j
]
==
PRIMARYKEY_TIMESTAMP_COL_ID
)
{
pResBlock
->
pBlockAgg
[
pSup
->
slotIds
[
j
]
]
=
&
pSup
->
tsColAgg
;
}
else
if
(
pSup
->
colI
nfo
[
j
].
colId
<
pAgg
->
colId
)
{
if
(
pSup
->
colI
nfo
[
j
].
colId
==
PRIMARYKEY_TIMESTAMP_COL_ID
)
{
pResBlock
->
pBlockAgg
[
pSup
->
colInfo
[
j
].
slotId
]
=
&
pSup
->
tsColAgg
;
}
else
{
// all date in this block are null
SColumnDataAgg
nullColAgg
=
{.
colId
=
pSup
->
colI
ds
[
j
]
,
.
numOfNull
=
pBlock
->
nRow
};
SColumnDataAgg
nullColAgg
=
{.
colId
=
pSup
->
colI
nfo
[
j
].
colId
,
.
numOfNull
=
pBlock
->
nRow
};
taosArrayPush
(
pSup
->
pColAgg
,
&
nullColAgg
);
pResBlock
->
pBlockAgg
[
pSup
->
slotIds
[
j
]
]
=
taosArrayGetLast
(
pSup
->
pColAgg
);
pResBlock
->
pBlockAgg
[
pSup
->
colInfo
[
j
].
slotId
]
=
taosArrayGetLast
(
pSup
->
pColAgg
);
}
j
+=
1
;
}
}
// taosArrayClear(pSup->pColAgg);
// size_t num = taosArrayGetSize(pSup->pColAgg);
// for(int32_t k = 0; k < num; ++k) {
// pSup->plist[k] = taosArrayGet(pSup->pColAgg, k);
// }
#endif
*
pBlockSMA
=
pResBlock
->
pBlockAgg
;
pReader
->
cost
.
smaDataLoad
+=
1
;
// *pBlockStatis = pSup->plist;
tsdbDebug
(
"vgId:%d, succeed to load block SMA for uid %"
PRIu64
", %s"
,
0
,
pFBlock
->
uid
,
pReader
->
idStr
);
return
code
;
}
static
S
Array
*
doRetrieveDataBlock
(
STsdbReader
*
pReader
)
{
static
S
SDataBlock
*
doRetrieveDataBlock
(
STsdbReader
*
pReader
)
{
SReaderStatus
*
pStatus
=
&
pReader
->
status
;
if
(
pStatus
->
composedDataBlock
)
{
return
pReader
->
pResBlock
->
pDataBlock
;
return
pReader
->
pResBlock
;
}
SFileDataBlockInfo
*
pBlockInfo
=
getCurrentBlockInfo
(
&
pStatus
->
blockIter
);
...
...
@@ -4166,10 +4151,10 @@ static SArray* doRetrieveDataBlock(STsdbReader* pReader) {
}
copyBlockDataToSDataBlock
(
pReader
,
pBlockScanInfo
);
return
pReader
->
pResBlock
->
pDataBlock
;
return
pReader
->
pResBlock
;
}
S
Array
*
tsdbRetrieveDataBlock
(
STsdbReader
*
pReader
,
SArray
*
pIdList
)
{
S
SDataBlock
*
tsdbRetrieveDataBlock
(
STsdbReader
*
pReader
,
SArray
*
pIdList
)
{
if
(
pReader
->
type
==
TIMEWINDOW_RANGE_EXTERNAL
)
{
if
(
pReader
->
step
==
EXTERNAL_ROWS_PREV
)
{
return
doRetrieveDataBlock
(
pReader
->
innerReader
[
0
]);
...
...
@@ -4196,7 +4181,6 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) {
// allocate buffer in order to load data blocks from file
memset
(
&
pReader
->
suppInfo
.
tsColAgg
,
0
,
sizeof
(
SColumnDataAgg
));
memset
(
pReader
->
suppInfo
.
plist
,
0
,
POINTER_BYTES
);
pReader
->
suppInfo
.
tsColAgg
.
colId
=
PRIMARYKEY_TIMESTAMP_COL_ID
;
tsdbDataFReaderClose
(
&
pReader
->
pFileReader
);
...
...
source/libs/executor/src/executor.c
浏览文件 @
42a512c3
...
...
@@ -957,21 +957,27 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s
pCond
->
order
=
TSDB_ORDER_ASC
;
pCond
->
numOfCols
=
pMtInfo
->
schema
->
nCols
;
pCond
->
colList
=
taosMemoryCalloc
(
pCond
->
numOfCols
,
sizeof
(
SColumnInfo
));
if
(
pCond
->
colList
==
NULL
)
{
pCond
->
pSlotList
=
taosMemoryMalloc
(
sizeof
(
int32_t
)
*
pCond
->
numOfCols
);
if
(
pCond
->
colList
==
NULL
||
pCond
->
pSlotList
==
NULL
)
{
taosMemoryFreeClear
(
pCond
->
colList
);
taosMemoryFreeClear
(
pCond
->
pSlotList
);
terrno
=
TSDB_CODE_QRY_OUT_OF_MEMORY
;
return
terrno
;
}
pCond
->
twindows
=
(
STimeWindow
){.
skey
=
INT64_MIN
,
.
ekey
=
INT64_MAX
}
;
pCond
->
twindows
=
TSWINDOW_INITIALIZER
;
pCond
->
suid
=
pMtInfo
->
suid
;
pCond
->
type
=
TIMEWINDOW_RANGE_CONTAINED
;
pCond
->
startVersion
=
-
1
;
pCond
->
endVersion
=
sContext
->
snapVersion
;
for
(
int32_t
i
=
0
;
i
<
pCond
->
numOfCols
;
++
i
)
{
pCond
->
colList
[
i
].
type
=
pMtInfo
->
schema
->
pSchema
[
i
].
type
;
pCond
->
colList
[
i
].
bytes
=
pMtInfo
->
schema
->
pSchema
[
i
].
bytes
;
pCond
->
colList
[
i
].
colId
=
pMtInfo
->
schema
->
pSchema
[
i
].
colId
;
SColumnInfo
*
pColInfo
=
&
pCond
->
colList
[
i
];
pColInfo
->
type
=
pMtInfo
->
schema
->
pSchema
[
i
].
type
;
pColInfo
->
bytes
=
pMtInfo
->
schema
->
pSchema
[
i
].
bytes
;
pColInfo
->
colId
=
pMtInfo
->
schema
->
pSchema
[
i
].
colId
;
pCond
->
pSlotList
[
i
]
=
i
;
}
return
TSDB_CODE_SUCCESS
;
...
...
@@ -1116,7 +1122,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT
int32_t
size
=
tableListGetSize
(
pTaskInfo
->
pTableInfoList
);
ASSERT
(
size
==
1
);
tsdbReaderOpen
(
pInfo
->
vnode
,
&
pTaskInfo
->
streamInfo
.
tableCond
,
pList
,
size
,
&
pInfo
->
pRes
,
&
pInfo
->
dataReader
,
NULL
);
tsdbReaderOpen
(
pInfo
->
vnode
,
&
pTaskInfo
->
streamInfo
.
tableCond
,
pList
,
size
,
NULL
,
&
pInfo
->
dataReader
,
NULL
);
cleanupQueryTableDataCond
(
&
pTaskInfo
->
streamInfo
.
tableCond
);
strcpy
(
pTaskInfo
->
streamInfo
.
tbName
,
mtInfo
.
tbName
);
...
...
source/libs/executor/src/scanoperator.c
浏览文件 @
42a512c3
...
...
@@ -224,10 +224,8 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA
}
static
bool
doLoadBlockSMA
(
STableScanBase
*
pTableScanInfo
,
SSDataBlock
*
pBlock
,
SExecTaskInfo
*
pTaskInfo
)
{
bool
allColumnsHaveAgg
=
true
;
SColumnDataAgg
**
pColAgg
=
NULL
;
int32_t
code
=
tsdbRetrieveDatablockSMA
(
pTableScanInfo
->
dataReader
,
&
pColAgg
,
&
allColumnsHaveAgg
);
bool
allColumnsHaveAgg
=
true
;
int32_t
code
=
tsdbRetrieveDatablockSMA
(
pTableScanInfo
->
dataReader
,
&
pBlock
->
pBlockAgg
,
&
allColumnsHaveAgg
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
T_LONG_JMP
(
pTaskInfo
->
env
,
code
);
}
...
...
@@ -387,11 +385,12 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca
pCost
->
totalCheckedRows
+=
pBlock
->
info
.
rows
;
pCost
->
loadBlocks
+=
1
;
S
Array
*
pCols
=
tsdbRetrieveDataBlock
(
pTableScanInfo
->
dataReader
,
NULL
);
if
(
p
Cols
==
NULL
)
{
S
SDataBlock
*
p
=
tsdbRetrieveDataBlock
(
pTableScanInfo
->
dataReader
,
NULL
);
if
(
p
==
NULL
)
{
return
terrno
;
}
ASSERT
(
p
==
pBlock
);
doSetTagColumnData
(
pTableScanInfo
,
pBlock
,
pTaskInfo
,
pBlock
->
info
.
rows
);
// restore the previous value
...
...
@@ -994,19 +993,10 @@ static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbU
return
NULL
;
}
bool
hasBlock
=
tsdbNextDataBlock
(
pReader
);
if
(
hasBlock
)
{
SDataBlockInfo
*
pBInfo
=
&
pBlock
->
info
;
// int32_t rows = 0;
// tsdbRetrieveDataBlockInfo(pReader, &rows, &pBInfo->id.uid, &pBInfo->window);
SArray
*
pCols
=
tsdbRetrieveDataBlock
(
pReader
,
NULL
);
// relocateColumnData(pBlock, pTableScanInfo->base.matchInfo.pList, pCols, true);
if
(
tsdbNextDataBlock
(
pReader
))
{
/*SSDataBlock* p = */
tsdbRetrieveDataBlock
(
pReader
,
NULL
);
doSetTagColumnData
(
&
pTableScanInfo
->
base
,
pBlock
,
pTaskInfo
,
pBlock
->
info
.
rows
);
pBlock
->
info
.
id
.
groupId
=
getTableGroupId
(
pTaskInfo
->
pTableInfoList
,
pBInfo
->
id
.
uid
);
pBlock
->
info
.
id
.
groupId
=
getTableGroupId
(
pTaskInfo
->
pTableInfoList
,
pBlock
->
info
.
id
.
uid
);
}
tsdbReaderClose
(
pReader
);
...
...
@@ -2030,20 +2020,13 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) {
qDebug
(
"tmqsnap doRawScan called"
);
if
(
pTaskInfo
->
streamInfo
.
prepareStatus
.
type
==
TMQ_OFFSET__SNAPSHOT_DATA
)
{
SSDataBlock
*
pBlock
=
&
pInfo
->
pRes
;
if
(
pInfo
->
dataReader
&&
tsdbNextDataBlock
(
pInfo
->
dataReader
))
{
if
(
isTaskKilled
(
pTaskInfo
))
{
longjmp
(
pTaskInfo
->
env
,
TSDB_CODE_TSC_QUERY_CANCELLED
);
}
int32_t
rows
=
0
;
tsdbRetrieveDataBlockInfo
(
pInfo
->
dataReader
,
&
rows
,
&
pBlock
->
info
.
id
.
uid
,
&
pBlock
->
info
.
window
);
pBlock
->
info
.
rows
=
rows
;
SArray
*
pCols
=
tsdbRetrieveDataBlock
(
pInfo
->
dataReader
,
NULL
);
pBlock
->
pDataBlock
=
pCols
;
if
(
pCols
==
NULL
)
{
SSDataBlock
*
pBlock
=
tsdbRetrieveDataBlock
(
pInfo
->
dataReader
,
NULL
);
if
(
pBlock
==
NULL
)
{
longjmp
(
pTaskInfo
->
env
,
terrno
);
}
...
...
tests/system-test/2-query/unique.py
浏览文件 @
42a512c3
...
...
@@ -429,10 +429,10 @@ class TDTestCase:
tdSql
.
checkRows
(
2
)
# nest query
tdSql
.
query
(
f
"select unique(c1)
from (select _rowts , t1 ,c1 , tbname from
{
dbname
}
.stb1 )
"
)
tdSql
.
query
(
f
"select unique(c1)
v from (select _rowts , t1 ,c1 , tbname from
{
dbname
}
.stb1 ) order by v
"
)
tdSql
.
checkRows
(
11
)
tdSql
.
checkData
(
0
,
0
,
6
)
tdSql
.
checkData
(
10
,
0
,
3
)
tdSql
.
checkData
(
1
,
0
,
0
)
tdSql
.
checkData
(
10
,
0
,
9
)
tdSql
.
query
(
f
"select unique(t1) from (select _rowts , t1 , tbname from
{
dbname
}
.stb1 )"
)
tdSql
.
checkRows
(
2
)
tdSql
.
checkData
(
0
,
0
,
4
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录