From 809a78b06eb1babf45e398ee23e48454f4977ff8 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Sun, 24 Apr 2022 18:05:28 +0800 Subject: [PATCH] enh: order by primary key optimize --- cmake/cmake.define | 2 +- include/libs/nodes/plannodes.h | 7 ++--- source/libs/command/src/explain.c | 27 ----------------- source/libs/executor/src/executorimpl.c | 6 ++-- source/libs/nodes/src/nodesCodeFuncs.c | 34 +++++++--------------- source/libs/planner/src/planLogicCreater.c | 3 +- source/libs/planner/src/planOptimizer.c | 13 ++++++--- source/libs/planner/src/planPhysiCreater.c | 5 +--- source/libs/planner/test/planOptTest.cpp | 32 ++++++++++++++++++++ 9 files changed, 60 insertions(+), 69 deletions(-) create mode 100644 source/libs/planner/test/planOptTest.cpp diff --git a/cmake/cmake.define b/cmake/cmake.define index c9a188600a..fb6ba8cc2e 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.16) -set(CMAKE_VERBOSE_MAKEFILE ON) +set(CMAKE_VERBOSE_MAKEFILE OFF) #set output directory SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index b4a290cbfc..829770ed1b 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -46,7 +46,7 @@ typedef struct SScanLogicNode { struct STableMeta* pMeta; SVgroupsInfo* pVgroupList; EScanType scanType; - uint8_t scanFlag; // denotes reversed scan of data or not + uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count STimeWindow scanRange; SName tableName; bool showRewrite; @@ -189,9 +189,6 @@ typedef struct SScanPhysiNode { SNodeList* pScanCols; uint64_t uid; // unique id of the table int8_t tableType; - int32_t order; // scan order: TSDB_ORDER_ASC|TSDB_ORDER_DESC - int32_t count; // repeat count - int32_t reverse; // reverse scan count SName tableName; } SScanPhysiNode; @@ -207,7 +204,7 @@ typedef struct SSystemTableScanPhysiNode { typedef struct STableScanPhysiNode { SScanPhysiNode scan; - uint8_t scanFlag; // denotes reversed scan of data or not + uint8_t scanSeq[2]; // first is scan count, and second is reverse scan count STimeWindow scanRange; double ratio; int32_t dataRequired; diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index 4853bb4eb3..75084e7610 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -344,11 +344,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTagScanNode->node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTagScanNode->count); - if (pTagScanNode->reverse) { - EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTagScanNode->reverse); - } EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -361,10 +356,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTagScanNode->order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - if (pResNode->pExecInfo) { QRY_ERR_RET(qExplainBufAppendVerboseExecInfo(pResNode->pExecInfo, tbuf, &tlen)); if (tlen) { @@ -388,11 +379,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pTblScanNode->scan.count); - if (pTblScanNode->scan.reverse) { - EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pTblScanNode->scan.reverse); - } EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -405,10 +391,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pTblScanNode->scan.order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_TIMERANGE_FORMAT, pTblScanNode->scanRange.skey, pTblScanNode->scanRange.ekey); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); @@ -434,11 +416,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); EXPLAIN_ROW_APPEND(EXPLAIN_WIDTH_FORMAT, pSTblScanNode->scan.node.pOutputDataBlockDesc->totalRowSize); EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_LOOPS_FORMAT, pSTblScanNode->scan.count); - if (pSTblScanNode->scan.reverse) { - EXPLAIN_ROW_APPEND(EXPLAIN_BLANK_FORMAT); - EXPLAIN_ROW_APPEND(EXPLAIN_REVERSE_FORMAT, pSTblScanNode->scan.reverse); - } EXPLAIN_ROW_APPEND(EXPLAIN_RIGHT_PARENTHESIS_FORMAT); EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level)); @@ -451,10 +428,6 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i EXPLAIN_ROW_END(); QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - EXPLAIN_ROW_NEW(level + 1, EXPLAIN_ORDER_FORMAT, EXPLAIN_ORDER_STRING(pSTblScanNode->scan.order)); - EXPLAIN_ROW_END(); - QRY_ERR_RET(qExplainResAppendRow(ctx, tbuf, tlen, level + 1)); - if (pSTblScanNode->scan.node.pConditions) { EXPLAIN_ROW_NEW(level + 1, EXPLAIN_FILTER_FORMAT); QRY_ERR_RET(nodesNodeToSQL(pSTblScanNode->scan.node.pConditions, tbuf + VARSTR_HEADER_SIZE, TSDB_EXPLAIN_RESULT_ROW_SIZE, &tlen)); diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 4a0206f55e..364637db89 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -6482,8 +6482,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo .offset = pTableScanNode->offset, }; - return createTableScanOperatorInfo(pDataReader, pScanPhyNode->order, numOfCols, pTableScanNode->dataRequired, - pScanPhyNode->count, pScanPhyNode->reverse, pColList, pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); + return createTableScanOperatorInfo(pDataReader, pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC, numOfCols, pTableScanNode->dataRequired, + pTableScanNode->scanSeq[0], pTableScanNode->scanSeq[1], pColList, pResBlock, pScanPhyNode->node.pConditions, &interval, pTableScanNode->ratio, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_EXCHANGE == type) { SExchangePhysiNode* pExchange = (SExchangePhysiNode*)pPhyNode; SSDataBlock* pResBlock = createResDataBlock(pExchange->node.pOutputDataBlockDesc); @@ -6624,7 +6624,7 @@ static tsdbReaderT createDataReaderImpl(STableScanPhysiNode* pTableScanNode, STa void* readHandle, uint64_t queryId, uint64_t taskId) { STsdbQueryCond cond = {.loadExternalRows = false}; - cond.order = pTableScanNode->scan.order; + cond.order = pTableScanNode->scanSeq[0] > 0 ? TSDB_ORDER_ASC : TSDB_ORDER_DESC; cond.numOfCols = LIST_LENGTH(pTableScanNode->scan.pScanCols); cond.colList = taosMemoryCalloc(cond.numOfCols, sizeof(SColumnInfo)); if (cond.colList == NULL) { diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 676304f9a8..046a5368d9 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -671,9 +671,6 @@ static int32_t jsonToName(const SJson* pJson, void* pObj) { static const char* jkScanPhysiPlanScanCols = "ScanCols"; static const char* jkScanPhysiPlanTableId = "TableId"; static const char* jkScanPhysiPlanTableType = "TableType"; -static const char* jkScanPhysiPlanScanOrder = "ScanOrder"; -static const char* jkScanPhysiPlanScanCount = "ScanCount"; -static const char* jkScanPhysiPlanReverseScanCount = "ReverseScanCount"; static const char* jkScanPhysiPlanTableName = "TableName"; static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { @@ -689,15 +686,6 @@ static int32_t physiScanNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanTableType, pNode->tableType); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanOrder, pNode->order); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanScanCount, pNode->count); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkScanPhysiPlanReverseScanCount, pNode->reverse); - } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkScanPhysiPlanTableName, nameToJson, &pNode->tableName); } @@ -718,15 +706,6 @@ static int32_t jsonToPhysiScanNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonGetTinyIntValue(pJson, jkScanPhysiPlanTableType, &pNode->tableType); } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkScanPhysiPlanScanOrder, &pNode->order); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkScanPhysiPlanScanCount, &pNode->count); - } - if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetIntValue(pJson, jkScanPhysiPlanReverseScanCount, &pNode->reverse); - } if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkScanPhysiPlanTableName, jsonToName, &pNode->tableName); } @@ -742,7 +721,8 @@ static int32_t jsonToPhysiTagScanNode(const SJson* pJson, void* pObj) { return jsonToPhysiScanNode(pJson, pObj); } -static const char* jkTableScanPhysiPlanScanFlag = "ScanFlag"; +static const char* jkTableScanPhysiPlanScanCount = "ScanCount"; +static const char* jkTableScanPhysiPlanReverseScanCount = "ReverseScanCount"; static const char* jkTableScanPhysiPlanStartKey = "StartKey"; static const char* jkTableScanPhysiPlanEndKey = "EndKey"; static const char* jkTableScanPhysiPlanRatio = "Ratio"; @@ -759,7 +739,10 @@ static int32_t physiTableScanNodeToJson(const void* pObj, SJson* pJson) { int32_t code = physiScanNodeToJson(pObj, pJson); if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanFlag, pNode->scanFlag); + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanScanCount, pNode->scanSeq[0]); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanReverseScanCount, pNode->scanSeq[1]); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkTableScanPhysiPlanStartKey, pNode->scanRange.skey); @@ -800,7 +783,10 @@ static int32_t jsonToPhysiTableScanNode(const SJson* pJson, void* pObj) { int32_t code = jsonToPhysiScanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { - code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanScanFlag, &pNode->scanFlag); + code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanScanCount, &pNode->scanSeq[0]); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetUTinyIntValue(pJson, jkTableScanPhysiPlanReverseScanCount, &pNode->scanSeq[1]); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkTableScanPhysiPlanStartKey, &pNode->scanRange.skey); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index 2c7ad27927..ac5188170a 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -199,7 +199,8 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect TSWAP(pScan->pMeta, pRealTable->pMeta, STableMeta*); TSWAP(pScan->pVgroupList, pRealTable->pVgroupList, SVgroupsInfo*); - pScan->scanFlag = MAIN_SCAN; + pScan->scanSeq[0] = 1; + pScan->scanSeq[1] = 0; pScan->scanRange = TSWINDOW_INITIALIZER; pScan->tableName.type = TSDB_TABLE_NAME_T; pScan->tableName.acctId = pCxt->pPlanCxt->acctId; diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 477a4b1bb6..0eb4c8435a 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -586,7 +586,7 @@ static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { if (1 != LIST_LENGTH(pSortKeys)) { return false; } - SNode* pNode = nodesListGetNode(pSortKeys, 0); + SNode* pNode = ((SOrderByExprNode*)nodesListGetNode(pSortKeys, 0))->pExpr; return (QUERY_NODE_COLUMN == nodeType(pNode) ? (PRIMARYKEY_TIMESTAMP_COL_ID == ((SColumnNode*)pNode)->colId) : false); } @@ -642,15 +642,20 @@ static EOrder opkGetPrimaryKeyOrder(SSortLogicNode* pSort) { static SNode* opkRewriteDownNode(SSortLogicNode* pSort) { SNode* pDownNode = nodesListGetNode(pSort->node.pChildren, 0); // todo + pSort->node.pChildren = NULL; return pDownNode; } static int32_t opkDoOptimized(SOptimizeContext* pCxt, SSortLogicNode* pSort, SNodeList* pScanNodes) { EOrder order = opkGetPrimaryKeyOrder(pSort); - SNode* pScan = NULL; - FOREACH(pScan, pScanNodes) { - ((SScanLogicNode*)pScan)->scanFlag = (ORDER_ASC == order ? MAIN_SCAN : REVERSE_SCAN); + if (ORDER_DESC == order) { + SNode* pScan = NULL; + FOREACH(pScan, pScanNodes) { + ((SScanLogicNode*)pScan)->scanSeq[0] = 0; + ((SScanLogicNode*)pScan)->scanSeq[1] = 1; + } } + if (NULL == pSort->node.pParent) { // todo return TSDB_CODE_SUCCESS; diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index d5e12608ac..28635ef940 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -398,9 +398,6 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SScanLogicNo if (TSDB_CODE_SUCCESS == code) { pScanPhysiNode->uid = pScanLogicNode->pMeta->uid; pScanPhysiNode->tableType = pScanLogicNode->pMeta->tableType; - pScanPhysiNode->order = TSDB_ORDER_ASC; - pScanPhysiNode->count = 1; - pScanPhysiNode->reverse = 0; memcpy(&pScanPhysiNode->tableName, &pScanLogicNode->tableName, sizeof(SName)); } @@ -432,7 +429,7 @@ static int32_t createTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubp return TSDB_CODE_OUT_OF_MEMORY; } - pTableScan->scanFlag = pScanLogicNode->scanFlag; + memcpy(pTableScan->scanSeq, pScanLogicNode->scanSeq, sizeof(pScanLogicNode->scanSeq)); pTableScan->scanRange = pScanLogicNode->scanRange; pTableScan->ratio = pScanLogicNode->ratio; vgroupInfoToNodeAddr(pScanLogicNode->pVgroupList->vgroups, &pSubplan->execNode); diff --git a/source/libs/planner/test/planOptTest.cpp b/source/libs/planner/test/planOptTest.cpp new file mode 100644 index 0000000000..4a682a5e17 --- /dev/null +++ b/source/libs/planner/test/planOptTest.cpp @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "planTestUtil.h" +#include "planner.h" + +using namespace std; + +class PlanOptimizeTest : public PlannerTestBase { + +}; + +TEST_F(PlanOptimizeTest, orderByPrimaryKey) { + useDb("root", "test"); + + run("select * from t1 order by ts"); + run("select * from t1 order by ts desc"); + run("select c1 from t1 order by ts"); + run("select c1 from t1 order by ts desc"); +} -- GitLab