Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
taosdata
TDengine
提交
d1a1976a
T
TDengine
项目概览
taosdata
/
TDengine
大约 1 年 前同步成功
通知
1184
Star
22015
Fork
4786
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
1
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
T
TDengine
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
1
Issue
1
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
d1a1976a
编写于
7月 19, 2023
作者:
S
slzhou
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
enhance: add limit to merge sort
上级
b9aeda26
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
51 addition
and
27 deletion
+51
-27
source/libs/executor/inc/tsort.h
source/libs/executor/inc/tsort.h
+8
-4
source/libs/executor/src/scanoperator.c
source/libs/executor/src/scanoperator.c
+6
-2
source/libs/executor/src/tsort.c
source/libs/executor/src/tsort.c
+37
-21
未找到文件。
source/libs/executor/inc/tsort.h
浏览文件 @
d1a1976a
...
...
@@ -26,7 +26,7 @@ extern "C" {
enum
{
SORT_MULTISOURCE_MERGE
=
0x1
,
SORT_SINGLESOURCE_SORT
=
0x2
,
SORT_
TABLE_MERGE_SCAN
=
0x3
SORT_
BLOCK_TS_MERGE
=
0x3
};
typedef
struct
SMultiMergeSource
{
...
...
@@ -56,7 +56,7 @@ typedef struct SMsortComparParam {
bool
cmpGroupId
;
int32_t
sortType
;
// the following field to speed up when sortType == SORT_
TABLE_MERGE_SCAN
// the following field to speed up when sortType == SORT_
BLOCK_TS_MERGE
int32_t
tsSlotId
;
int32_t
order
;
__compar_fn_t
cmpFn
;
...
...
@@ -77,8 +77,8 @@ typedef int32_t (*_sort_merge_compar_fn_t)(const void* p1, const void* p2, void*
* @return
*/
SSortHandle
*
tsortCreateSortHandle
(
SArray
*
pOrderInfo
,
int32_t
type
,
int32_t
pageSize
,
int32_t
numOfPages
,
SSDataBlock
*
pBlock
,
const
char
*
idstr
,
uint64_t
maxRows
,
uint32_t
m
axTupleLength
,
uint32_t
s
ortBufSize
);
SSDataBlock
*
pBlock
,
const
char
*
idstr
,
uint64_t
pqMaxRows
,
uint32_t
pqM
axTupleLength
,
uint32_t
pqS
ortBufSize
);
void
tsortSetForceUsePQSort
(
SSortHandle
*
pHandle
);
...
...
@@ -117,6 +117,10 @@ int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetc
*/
int32_t
tsortSetComparFp
(
SSortHandle
*
pHandle
,
_sort_merge_compar_fn_t
fp
);
/**
*
*/
void
tsortSetMergeLimit
(
SSortHandle
*
pHandle
,
int64_t
mergeLimit
);
/**
*
*/
...
...
source/libs/executor/src/scanoperator.c
浏览文件 @
d1a1976a
...
...
@@ -2842,9 +2842,13 @@ int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) {
pInfo
->
sortBufSize
=
1024
*
pInfo
->
bufPageSize
;
int32_t
numOfBufPage
=
pInfo
->
sortBufSize
/
pInfo
->
bufPageSize
;
pInfo
->
pSortHandle
=
tsortCreateSortHandle
(
pInfo
->
pSortInfo
,
SORT_
TABLE_MERGE_SCAN
,
pInfo
->
bufPageSize
,
numOfBufPage
,
pInfo
->
pSortHandle
=
tsortCreateSortHandle
(
pInfo
->
pSortInfo
,
SORT_
BLOCK_TS_MERGE
,
pInfo
->
bufPageSize
,
numOfBufPage
,
pInfo
->
pSortInputBlock
,
pTaskInfo
->
id
.
str
,
0
,
0
,
0
);
int64_t
mergeLimit
=
-
1
;
if
(
pInfo
->
limitInfo
.
limit
.
limit
!=
-
1
||
pInfo
->
limitInfo
.
limit
.
offset
!=
-
1
)
{
mergeLimit
=
pInfo
->
limitInfo
.
limit
.
limit
+
pInfo
->
limitInfo
.
limit
.
offset
;
}
tsortSetMergeLimit
(
pInfo
->
pSortHandle
,
mergeLimit
);
tsortSetFetchRawDataFp
(
pInfo
->
pSortHandle
,
getTableDataBlockImpl
,
NULL
,
NULL
);
// one table has one data block
...
...
source/libs/executor/src/tsort.c
浏览文件 @
d1a1976a
...
...
@@ -42,13 +42,15 @@ struct SSortHandle {
int64_t
startTs
;
uint64_t
totalElapsed
;
uint64_t
m
axRows
;
uint32_t
m
axTupleLength
;
uint32_t
s
ortBufSize
;
uint64_t
pqM
axRows
;
uint32_t
pqM
axTupleLength
;
uint32_t
pqS
ortBufSize
;
bool
forceUsePQSort
;
BoundedQueue
*
pBoundedQueue
;
uint32_t
tmpRowIdx
;
int64_t
mergeLimit
;
int32_t
sourceId
;
SSDataBlock
*
pDataBlock
;
SMsortComparParam
cmpParam
;
...
...
@@ -173,8 +175,8 @@ void destroyTuple(void* t) {
* @return
*/
SSortHandle
*
tsortCreateSortHandle
(
SArray
*
pSortInfo
,
int32_t
type
,
int32_t
pageSize
,
int32_t
numOfPages
,
SSDataBlock
*
pBlock
,
const
char
*
idstr
,
uint64_t
maxRows
,
uint32_t
m
axTupleLength
,
uint32_t
s
ortBufSize
)
{
SSDataBlock
*
pBlock
,
const
char
*
idstr
,
uint64_t
pqMaxRows
,
uint32_t
pqM
axTupleLength
,
uint32_t
pqS
ortBufSize
)
{
SSortHandle
*
pSortHandle
=
taosMemoryCalloc
(
1
,
sizeof
(
SSortHandle
));
pSortHandle
->
type
=
type
;
...
...
@@ -183,10 +185,10 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page
pSortHandle
->
pSortInfo
=
pSortInfo
;
pSortHandle
->
loops
=
0
;
pSortHandle
->
maxTupleLength
=
m
axTupleLength
;
if
(
m
axRows
!=
0
)
{
pSortHandle
->
sortBufSize
=
s
ortBufSize
;
pSortHandle
->
maxRows
=
m
axRows
;
pSortHandle
->
pqMaxTupleLength
=
pqM
axTupleLength
;
if
(
pqM
axRows
!=
0
)
{
pSortHandle
->
pqSortBufSize
=
pqS
ortBufSize
;
pSortHandle
->
pqMaxRows
=
pqM
axRows
;
}
pSortHandle
->
forceUsePQSort
=
false
;
...
...
@@ -194,11 +196,13 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page
pSortHandle
->
pDataBlock
=
createOneDataBlock
(
pBlock
,
false
);
}
pSortHandle
->
mergeLimit
=
-
1
;
pSortHandle
->
pOrderedSource
=
taosArrayInit
(
4
,
POINTER_BYTES
);
pSortHandle
->
cmpParam
.
orderInfo
=
pSortInfo
;
pSortHandle
->
cmpParam
.
cmpGroupId
=
false
;
pSortHandle
->
cmpParam
.
sortType
=
type
;
if
(
type
==
SORT_
TABLE_MERGE_SCAN
)
{
if
(
type
==
SORT_
BLOCK_TS_MERGE
)
{
SBlockOrderInfo
*
pOrder
=
TARRAY_GET_ELEM
(
pSortInfo
,
0
);
pSortHandle
->
cmpParam
.
tsSlotId
=
pOrder
->
slotId
;
pSortHandle
->
cmpParam
.
order
=
pOrder
->
order
;
...
...
@@ -586,7 +590,7 @@ int32_t msortComparFn(const void* pLeft, const void* pRight, void* param) {
}
}
if
(
pParam
->
sortType
==
SORT_
TABLE_MERGE_SCAN
)
{
if
(
pParam
->
sortType
==
SORT_
BLOCK_TS_MERGE
)
{
SColumnInfoData
*
pLeftColInfoData
=
TARRAY_GET_ELEM
(
pLeftBlock
->
pDataBlock
,
pParam
->
tsSlotId
);
SColumnInfoData
*
pRightColInfoData
=
TARRAY_GET_ELEM
(
pRightBlock
->
pDataBlock
,
pParam
->
tsSlotId
);
int64_t
*
left1
=
(
int64_t
*
)(
pLeftColInfoData
->
pData
)
+
pLeftSource
->
src
.
rowIndex
;
...
...
@@ -709,18 +713,23 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
taosArrayDestroy
(
pResList
);
return
code
;
}
int
nRows
=
0
;
SArray
*
pPageIdList
=
taosArrayInit
(
4
,
sizeof
(
int32_t
));
while
(
1
)
{
if
(
tsortIsClosed
(
pHandle
))
{
code
=
terrno
=
TSDB_CODE_TSC_QUERY_CANCELLED
;
return
code
;
}
if
(
pHandle
->
mergeLimit
!=
-
1
&&
nRows
>=
pHandle
->
mergeLimit
)
{
break
;
}
SSDataBlock
*
pDataBlock
=
getSortedBlockDataInner
(
pHandle
,
&
pHandle
->
cmpParam
,
numOfRows
);
if
(
pDataBlock
==
NULL
)
{
break
;
}
nRows
+=
pDataBlock
->
info
.
rows
;
int32_t
pageId
=
-
1
;
void
*
pPage
=
getNewBufPage
(
pHandle
->
pBuf
,
&
pageId
);
...
...
@@ -740,7 +749,6 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) {
setBufPageDirty
(
pPage
,
true
);
releaseBufPage
(
pHandle
->
pBuf
,
pPage
);
blockDataCleanup
(
pDataBlock
);
}
...
...
@@ -846,7 +854,6 @@ static int32_t appendDataBlockToPageBuf(SSortHandle* pHandle, SSDataBlock* blk,
setBufPageDirty
(
pPage
,
true
);
releaseBufPage
(
pHandle
->
pBuf
,
pPage
);
blockDataCleanup
(
blk
);
return
0
;
}
...
...
@@ -913,6 +920,9 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
size_t
blkPgSz
=
pgHeaderSz
;
while
(
nRows
<
totalRows
)
{
if
(
pHandle
->
mergeLimit
!=
-
1
&&
nRows
>=
pHandle
->
mergeLimit
)
{
break
;
}
int32_t
minIdx
=
tMergeTreeGetChosenIndex
(
pTree
);
SSDataBlock
*
minBlk
=
taosArrayGetP
(
aBlk
,
minIdx
);
int32_t
minRow
=
sup
.
aRowIdx
[
minIdx
];
...
...
@@ -920,6 +930,7 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
if
(
blkPgSz
<=
pHandle
->
pageSize
&&
blkPgSz
+
bufInc
>
pHandle
->
pageSize
)
{
appendDataBlockToPageBuf
(
pHandle
,
pHandle
->
pDataBlock
,
aPgId
);
blockDataCleanup
(
pHandle
->
pDataBlock
);
blkPgSz
=
pgHeaderSz
;
bufInc
=
getPageBufIncForRow
(
minBlk
,
minRow
,
0
);
}
...
...
@@ -939,6 +950,7 @@ static int32_t sortBlocksToExtSource(SSortHandle* pHandle, SArray* aBlk, SBlockO
}
if
(
pHandle
->
pDataBlock
->
info
.
rows
>
0
)
{
appendDataBlockToPageBuf
(
pHandle
,
pHandle
->
pDataBlock
,
aPgId
);
blockDataCleanup
(
pHandle
->
pDataBlock
);
}
SSDataBlock
*
pMemSrcBlk
=
createOneDataBlock
(
pHandle
->
pDataBlock
,
false
);
doAddNewExternalMemSource
(
pHandle
->
pBuf
,
aExtSrc
,
pMemSrcBlk
,
&
pHandle
->
sourceId
,
aPgId
);
...
...
@@ -1061,7 +1073,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
int64_t
el
=
taosGetTimestampUs
()
-
p
;
pHandle
->
sortElapsed
+=
el
;
if
(
pHandle
->
maxRows
>
0
)
blockDataKeepFirstNRows
(
pHandle
->
pDataBlock
,
pHandle
->
m
axRows
);
if
(
pHandle
->
pqMaxRows
>
0
)
blockDataKeepFirstNRows
(
pHandle
->
pDataBlock
,
pHandle
->
pqM
axRows
);
code
=
doAddToBuf
(
pHandle
->
pDataBlock
,
pHandle
);
if
(
code
!=
TSDB_CODE_SUCCESS
)
{
return
code
;
...
...
@@ -1086,7 +1098,7 @@ static int32_t createBlocksQuickSortInitialSources(SSortHandle* pHandle) {
return
code
;
}
if
(
pHandle
->
maxRows
>
0
)
blockDataKeepFirstNRows
(
pHandle
->
pDataBlock
,
pHandle
->
m
axRows
);
if
(
pHandle
->
pqMaxRows
>
0
)
blockDataKeepFirstNRows
(
pHandle
->
pDataBlock
,
pHandle
->
pqM
axRows
);
int64_t
el
=
taosGetTimestampUs
()
-
p
;
pHandle
->
sortElapsed
+=
el
;
...
...
@@ -1111,7 +1123,7 @@ static int32_t createInitialSources(SSortHandle* pHandle) {
if
(
pHandle
->
type
==
SORT_SINGLESOURCE_SORT
)
{
code
=
createBlocksQuickSortInitialSources
(
pHandle
);
}
else
if
(
pHandle
->
type
==
SORT_
TABLE_MERGE_SCAN
)
{
}
else
if
(
pHandle
->
type
==
SORT_
BLOCK_TS_MERGE
)
{
code
=
createBlocksMergeSortInitialSources
(
pHandle
);
}
uInfo
(
"%zu sources created"
,
taosArrayGetSize
(
pHandle
->
pOrderedSource
));
...
...
@@ -1165,6 +1177,10 @@ void tsortSetClosed(SSortHandle* pHandle) {
atomic_store_8
(
&
pHandle
->
closed
,
2
);
}
void
tsortSetMergeLimit
(
SSortHandle
*
pHandle
,
int64_t
mergeLimit
)
{
pHandle
->
mergeLimit
=
mergeLimit
;
}
int32_t
tsortSetFetchRawDataFp
(
SSortHandle
*
pHandle
,
_sort_fetch_block_fn_t
fetchFp
,
void
(
*
fp
)(
SSDataBlock
*
,
void
*
),
void
*
param
)
{
pHandle
->
fetchfp
=
fetchFp
;
...
...
@@ -1244,8 +1260,8 @@ void tsortSetForceUsePQSort(SSortHandle* pHandle) {
static
bool
tsortIsPQSortApplicable
(
SSortHandle
*
pHandle
)
{
if
(
pHandle
->
type
!=
SORT_SINGLESOURCE_SORT
)
return
false
;
if
(
tsortIsForceUsePQSort
(
pHandle
))
return
true
;
uint64_t
maxRowsFitInMemory
=
pHandle
->
sortBufSize
/
(
pHandle
->
m
axTupleLength
+
sizeof
(
char
*
));
return
maxRowsFitInMemory
>
pHandle
->
m
axRows
;
uint64_t
maxRowsFitInMemory
=
pHandle
->
pqSortBufSize
/
(
pHandle
->
pqM
axTupleLength
+
sizeof
(
char
*
));
return
maxRowsFitInMemory
>
pHandle
->
pqM
axRows
;
}
static
bool
tsortPQCompFn
(
void
*
a
,
void
*
b
,
void
*
param
)
{
...
...
@@ -1291,7 +1307,7 @@ static int32_t tupleComparFn(const void* pLeft, const void* pRight, void* param)
}
static
int32_t
tsortOpenForPQSort
(
SSortHandle
*
pHandle
)
{
pHandle
->
pBoundedQueue
=
createBoundedQueue
(
pHandle
->
m
axRows
,
tsortPQCompFn
,
destroyTuple
,
pHandle
);
pHandle
->
pBoundedQueue
=
createBoundedQueue
(
pHandle
->
pqM
axRows
,
tsortPQCompFn
,
destroyTuple
,
pHandle
);
if
(
NULL
==
pHandle
->
pBoundedQueue
)
return
TSDB_CODE_OUT_OF_MEMORY
;
tsortSetComparFp
(
pHandle
,
tupleComparFn
);
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录