From 317cc8fc13fcbd6e82ab4714e883f21bfff30840 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 10 Jun 2022 10:16:18 +0800 Subject: [PATCH] feat: sma index optimize --- include/libs/nodes/plannodes.h | 1 + include/libs/nodes/querynodes.h | 1 + include/util/tjson.h | 15 +- source/libs/nodes/src/nodesCodeFuncs.c | 82 ++++++++++ source/libs/parser/inc/parUtil.h | 3 +- source/libs/parser/src/parAstParser.c | 4 + source/libs/parser/src/parTranslater.c | 82 +++------- source/libs/parser/src/parUtil.c | 38 ++++- source/libs/parser/test/mockCatalog.cpp | 6 + .../libs/parser/test/mockCatalogService.cpp | 59 ++++++- source/libs/parser/test/mockCatalogService.h | 1 + source/libs/planner/src/planOptimizer.c | 144 ++++++++++++++++-- source/libs/planner/test/planOtherTest.cpp | 2 + source/libs/planner/test/planTestUtil.cpp | 3 + source/util/src/tjson.c | 53 ++++++- 15 files changed, 393 insertions(+), 101 deletions(-) diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 00380724b3..b62854db43 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -62,6 +62,7 @@ typedef struct SScanLogicNode { int64_t watermark; int16_t tsColId; double filesFactor; + SArray* pSmaIndexes; } SScanLogicNode; typedef struct SJoinLogicNode { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index c2aa86e89f..33cc4a9c86 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -144,6 +144,7 @@ typedef struct SRealTableNode { SVgroupsInfo* pVgroupList; char qualDbName[TSDB_DB_NAME_LEN]; // SHOW qualDbName.TABLES double ratio; + SArray* pSmaIndexes; } SRealTableNode; typedef struct STempTableNode { diff --git a/include/util/tjson.h b/include/util/tjson.h index 84f7b81726..df433227ca 100644 --- a/include/util/tjson.h +++ b/include/util/tjson.h @@ -17,16 +17,17 @@ #define _TD_UTIL_JSON_H_ #include "os.h" +#include "tarray.h" #ifdef __cplusplus extern "C" { #endif -#define tjsonGetNumberValue(pJson, pName, val, code) \ - do { \ - uint64_t _tmp = 0; \ +#define tjsonGetNumberValue(pJson, pName, val, code) \ + do { \ + uint64_t _tmp = 0; \ code = tjsonGetBigIntValue(pJson, pName, &_tmp); \ - val = _tmp; \ + val = _tmp; \ } while (0) typedef void SJson; @@ -66,18 +67,20 @@ typedef int32_t (*FToJson)(const void* pObj, SJson* pJson); int32_t tjsonAddObject(SJson* pJson, const char* pName, FToJson func, const void* pObj); int32_t tjsonAddItem(SJson* pJson, FToJson func, const void* pObj); int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* pArray, int32_t itemSize, int32_t num); +int32_t tjsonAddTArray(SJson* pJson, const char* pName, FToJson func, const SArray* pArray); typedef int32_t (*FToObject)(const SJson* pJson, void* pObj); int32_t tjsonToObject(const SJson* pJson, const char* pName, FToObject func, void* pObj); int32_t tjsonMakeObject(const SJson* pJson, const char* pName, FToObject func, void** pObj, int32_t objSize); int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void* pArray, int32_t itemSize); +int32_t tjsonToTArray(const SJson* pJson, const char* pName, FToObject func, SArray** pArray, int32_t itemSize); char* tjsonToString(const SJson* pJson); char* tjsonToUnformattedString(const SJson* pJson); -SJson* tjsonParse(const char* pStr); -bool tjsonValidateJson(const char* pJson); +SJson* tjsonParse(const char* pStr); +bool tjsonValidateJson(const char* pJson); const char* tjsonGetError(); #ifdef __cplusplus diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 0735092c1a..48840c4a8b 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2809,10 +2809,85 @@ static int32_t jsonToTableNode(const SJson* pJson, void* pObj) { return code; } +static const char* jkTableIndexInfoIntervalUnit = "IntervalUnit"; +static const char* jkTableIndexInfoSlidingUnit = "SlidingUnit"; +static const char* jkTableIndexInfoInterval = "Interval"; +static const char* jkTableIndexInfoOffset = "Offset"; +static const char* jkTableIndexInfoSliding = "Sliding"; +static const char* jkTableIndexInfoDstTbUid = "DstTbUid"; +static const char* jkTableIndexInfoDstVgId = "DstVgId"; +static const char* jkTableIndexInfoEpSet = "EpSet"; +static const char* jkTableIndexInfoExpr = "Expr"; + +static int32_t tableIndexInfoToJson(const void* pObj, SJson* pJson) { + const STableIndexInfo* pNode = (const STableIndexInfo*)pObj; + + int32_t code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoIntervalUnit, pNode->intervalUnit); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoSlidingUnit, pNode->slidingUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoInterval, pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoOffset, pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoSliding, pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoDstTbUid, pNode->dstTbUid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddIntegerToObject(pJson, jkTableIndexInfoDstVgId, pNode->dstVgId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkTableIndexInfoEpSet, epSetToJson, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddStringToObject(pJson, jkTableIndexInfoExpr, pNode->expr); + } + + return code; +} + +static int32_t jsonToTableIndexInfo(const SJson* pJson, void* pObj) { + STableIndexInfo* pNode = (STableIndexInfo*)pObj; + + int32_t code = tjsonGetTinyIntValue(pJson, jkTableIndexInfoIntervalUnit, &pNode->intervalUnit); + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetTinyIntValue(pJson, jkTableIndexInfoSlidingUnit, &pNode->slidingUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoInterval, &pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoOffset, &pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoSliding, &pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetBigIntValue(pJson, jkTableIndexInfoDstTbUid, &pNode->dstTbUid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonGetIntValue(pJson, jkTableIndexInfoDstVgId, &pNode->dstVgId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonToObject(pJson, jkTableIndexInfoEpSet, jsonToEpSet, &pNode->epSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonDupStringValue(pJson, jkTableIndexInfoExpr, &pNode->expr); + } + + return code; +} + static const char* jkRealTableMetaSize = "MetaSize"; static const char* jkRealTableMeta = "Meta"; static const char* jkRealTableVgroupsInfoSize = "VgroupsInfoSize"; static const char* jkRealTableVgroupsInfo = "VgroupsInfo"; +static const char* jkRealTableSmaIndexes = "SmaIndexes"; static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) { const SRealTableNode* pNode = (const SRealTableNode*)pObj; @@ -2830,6 +2905,9 @@ static int32_t realTableNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkRealTableVgroupsInfo, vgroupsInfoToJson, pNode->pVgroupList); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddTArray(pJson, jkRealTableSmaIndexes, tableIndexInfoToJson, pNode->pSmaIndexes); + } return code; } @@ -2851,6 +2929,10 @@ static int32_t jsonToRealTableNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = tjsonMakeObject(pJson, jkRealTableVgroupsInfo, jsonToVgroupsInfo, (void**)&pNode->pVgroupList, objSize); } + if (TSDB_CODE_SUCCESS == code) { + code = + tjsonToTArray(pJson, jkRealTableSmaIndexes, jsonToTableIndexInfo, &pNode->pSmaIndexes, sizeof(STableIndexInfo)); + } return code; } diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 42719d95bd..fb67a35368 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -46,7 +46,7 @@ typedef struct SParseMetaCache { SHashObj* pDbInfo; // key is tbFName, element is SDbInfo* SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass SHashObj* pUdf; // key is funcName, element is SFuncInfo* - SHashObj* pSmaIndex; // key is tbFName, element is SArray* + SHashObj* pTableIndex; // key is tbFName, element is SArray* } SParseMetaCache; int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); @@ -76,6 +76,7 @@ int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pD SParseMetaCache* pMetaCache); int32_t reserveUserAuthInCacheExt(const char* pUser, const SName* pName, AUTH_TYPE type, SParseMetaCache* pMetaCache); int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache); +int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta); int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo); int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup); diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 6555ec3a7d..6acc4575e7 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -129,6 +129,10 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, SRealTa if (TSDB_CODE_SUCCESS == code) { code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveTableIndexInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, + pCxt->pMetaCache); + } return code; } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 90da79266f..c2fa1ffd22 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -258,24 +258,19 @@ static int32_t getUdfInfo(STranslateContext* pCxt, SFunctionNode* pFunc) { return code; } -static int32_t getTableIndex(STranslateContext* pCxt, const char* pDbName, const char* pTableName, SArray** pIndexes) { +static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray** pIndexes) { SParseContext* pParCxt = pCxt->pParseCxt; - SName name; - toName(pParCxt->acctId, pDbName, pTableName, &name); - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = collectUseDatabase(pName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(pName, pCxt->pTables); + } if (pParCxt->async) { - code = getTableIndexFromCache(pCxt->pMetaCache, &name, pIndexes); + code = getTableIndexFromCache(pCxt->pMetaCache, pName, pIndexes); } else { - code = collectUseDatabase(&name, pCxt->pDbs); - if (TSDB_CODE_SUCCESS == code) { - code = collectUseTable(&name, pCxt->pTables); - } - if (TSDB_CODE_SUCCESS == code) { - code = catalogGetTableIndex(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, &name, pIndexes); - } + code = catalogGetTableIndex(pParCxt->pCatalog, pParCxt->pTransporter, &pParCxt->mgmtEpSet, pName, pIndexes); } if (TSDB_CODE_SUCCESS != code) { - parserError("getTableIndex error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pDbName, pTableName); + parserError("getTableIndex error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname, pName->tname); } return code; } @@ -853,9 +848,9 @@ static EDealRes translateComparisonOperator(STranslateContext* pCxt, SOperatorNo } if (OP_TYPE_IN == pOp->opType || OP_TYPE_NOT_IN == pOp->opType) { SNodeListNode* pRight = (SNodeListNode*)pOp->pRight; - bool first = true; - SDataType targetDt = {0}; - SNode* pNode = NULL; + bool first = true; + SDataType targetDt = {0}; + SNode* pNode = NULL; FOREACH(pNode, pRight->pNodeList) { SDataType dt = ((SExprNode*)pNode)->resType; if (first) { @@ -1409,6 +1404,9 @@ static int32_t translateTable(STranslateContext* pCxt, SNode* pTable) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_TABLE_NOT_EXIST, pRealTable->table.tableName); } code = setTableVgroupList(pCxt, &name, pRealTable); + if (TSDB_CODE_SUCCESS == code) { + code = getTableIndex(pCxt, &name, &pRealTable->pSmaIndexes); + } } pRealTable->table.precision = pRealTable->pMeta->tableInfo.precision; pRealTable->table.singleTable = isSingleTable(pRealTable); @@ -2018,33 +2016,8 @@ static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SSelectStmt* pSelect nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteTimelineFuncImpl, pCxt); return pCxt->errCode; } -#if 0 -static bool mayBeApplySmaIndex(SSelectStmt* pSelect) { - if (NULL == pSelect->pWindow || QUERY_NODE_INTERVAL_WINDOW != nodeType(pSelect->pWindow) || - NULL != ((SIntervalWindowNode*)pSelect->pWindow)->pFill || - QUERY_NODE_REAL_TABLE != nodeType(pSelect->pFromTable) || NULL != pSelect->pWhere || - NULL != pSelect->pPartitionByList || NULL != pSelect->pGroupByList || NULL != pSelect->pHaving) { - return false; - } - return true; -} -static bool equalIntervalWindow(SIntervalWindowNode* pInterval, SNode* pWhere, STableIndexInfo* pIndex) { - int64_t interval = ((SValueNode*)pInterval->pInterval)->datum.i; - int8_t intervalUnit = ((SValueNode*)pInterval->pInterval)->unit; - int64_t offset = (NULL != pInterval->pOffset ? ((SValueNode*)pInterval->pOffset)->datum.i : 0); - int64_t sliding = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->datum.i : interval); - int8_t slidingUnit = (NULL != pInterval->pSliding ? ((SValueNode*)pInterval->pSliding)->unit : intervalUnit); - if (interval != pIndex->interval || intervalUnit != pIndex->intervalUnit || offset != pIndex->offset || - sliding != pIndex->sliding || slidingUnit != pIndex->slidingUnit) { - return false; - } - // todo - if (NULL != pWhere) { - return false; - } - return true; -} +#if 0 typedef struct SSmaIndexMatchFuncsCxt { int32_t errCode; @@ -2056,10 +2029,10 @@ typedef struct SSmaIndexMatchFuncsCxt { bool match; } SSmaIndexMatchFuncsCxt; -static SColumnNode* createColumnFromSmaFunc(uint64_t tableId, int32_t index, SExprNode* pSmaFunc) { +static int32_t smaOptCreateSmaCol(SNode* pSmaFunc, int32_t index, SNode** pOutput) { SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { - return NULL; + return TSDB_CODE_SUCCESS; } pCol->tableId = tableId; pCol->tableType = TSDB_SUPER_TABLE; @@ -2070,7 +2043,7 @@ static SColumnNode* createColumnFromSmaFunc(uint64_t tableId, int32_t index, SEx strcpy(pCol->tableAlias, SMA_TABLE_NAME); pCol->node.resType = pSmaFunc->resType; strcpy(pCol->node.aliasName, pSmaFunc->aliasName); - return pCol; + return TSDB_CODE_SUCCESS; } static int32_t collectSmaFunc(SSmaIndexMatchFuncsCxt* pCxt, int32_t index, SNode* pSmaFunc) { @@ -2262,25 +2235,6 @@ static int32_t attemptApplySmaIndexImpl(STranslateContext* pCxt, SSelectStmt* pS return TSDB_CODE_SUCCESS; } -static void destroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } - -static int32_t attemptApplySmaIndex(STranslateContext* pCxt, SSelectStmt* pSelect) { - SRealTableNode* pRealTable = (SRealTableNode*)pSelect->pFromTable; - SArray* pIndexes = NULL; - int32_t code = getTableIndex(pCxt, pRealTable->table.dbName, pRealTable->table.tableName, &pIndexes); - if (TSDB_CODE_SUCCESS == code) { - code = attemptApplySmaIndexImpl(pCxt, pSelect, pIndexes); - } - taosArrayDestroyEx(pIndexes, destroySmaIndex); - return code; -} - -static int32_t attemptApplyIndex(STranslateContext* pCxt, SSelectStmt* pSelect) { - // if (mayBeApplySmaIndex(pSelect)) { - // return attemptApplySmaIndex(pCxt, pSelect); - // } - return TSDB_CODE_SUCCESS; -} #endif static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 41bb0481ad..5bde056b5c 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -807,6 +807,42 @@ int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFun return code; } +static void destroySmaIndex(void* p) { taosMemoryFree(((STableIndexInfo*)p)->expr); } + +static SArray* smaIndexesDup(SArray* pSrc) { + SArray* pDst = taosArrayDup(pSrc); + if (NULL == pDst) { + return NULL; + } + int32_t size = taosArrayGetSize(pDst); + for (int32_t i = 0; i < size; ++i) { + ((STableIndexInfo*)taosArrayGet(pDst, i))->expr = NULL; + } + for (int32_t i = 0; i < size; ++i) { + STableIndexInfo* pIndex = taosArrayGet(pDst, i); + pIndex->expr = taosMemoryStrDup(((STableIndexInfo*)taosArrayGet(pSrc, i))->expr); + if (NULL == pIndex->expr) { + taosArrayDestroyEx(pDst, destroySmaIndex); + return NULL; + } + } + return pDst; +} + +int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { + return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableIndex); +} + int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes) { - return TSDB_CODE_PAR_INTERNAL_ERROR; + char fullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pName, fullName); + SArray* pSmaIndexes = NULL; + int32_t code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableIndex, (void**)&pSmaIndexes); + if (TSDB_CODE_SUCCESS == code) { + *pIndexes = smaIndexesDup(pSmaIndexes); + if (NULL == *pIndexes) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + return code; } diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index c7f0990132..0b2aacf5ef 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -214,6 +214,11 @@ int32_t __catalogRefreshGetTableMeta(SCatalog* pCatalog, void* pTransporter, con int32_t __catalogRemoveTableMeta(SCatalog* pCtg, SName* pTableName) { return 0; } +int32_t __catalogGetTableIndex(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmtEps, const SName* pName, + SArray** pRes) { + return g_mockCatalogService->catalogGetTableIndex(pName, pRes); +} + void initMetaDataEnv() { g_mockCatalogService.reset(new MockCatalogService()); @@ -230,6 +235,7 @@ void initMetaDataEnv() { stub.set(catalogGetUdfInfo, __catalogGetUdfInfo); stub.set(catalogRefreshGetTableMeta, __catalogRefreshGetTableMeta); stub.set(catalogRemoveTableMeta, __catalogRemoveTableMeta); + stub.set(catalogGetTableIndex, __catalogGetTableIndex); // { // AddrAny any("libcatalog.so"); // std::map result; diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 0c37c875c0..c730653e5e 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -149,6 +149,20 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } + int32_t catalogGetTableIndex(const SName* pTableName, SArray** pIndexes) const { + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + tNameExtractFullName(pTableName, tbFName); + auto it = index_.find(tbFName); + if (index_.end() == it) { + return TSDB_CODE_SUCCESS; + } + *pIndexes = taosArrayInit(it->second.size(), sizeof(STableIndexInfo)); + for (const auto& index : it->second) { + taosArrayPush(*pIndexes, &index); + } + return TSDB_CODE_SUCCESS; + } + int32_t catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const { int32_t code = getAllTableMeta(pCatalogReq->pTableMeta, &pMetaData->pTableMeta); if (TSDB_CODE_SUCCESS == code) { @@ -176,7 +190,7 @@ class MockCatalogServiceImpl { int32_t numOfColumns, int32_t numOfTags) { builder_ = TableBuilder::createTableBuilder(tableType, numOfColumns, numOfTags); meta_[db][tbname] = builder_->table(); - meta_[db][tbname]->schema->uid = id_++; + meta_[db][tbname]->schema->uid = getNextId(); return *(builder_.get()); } @@ -187,14 +201,11 @@ class MockCatalogServiceImpl { } meta_[db][tbname].reset(new MockTableMeta()); meta_[db][tbname]->schema = table.release(); - meta_[db][tbname]->schema->uid = id_++; + meta_[db][tbname]->schema->uid = getNextId(); meta_[db][tbname]->schema->tableType = TSDB_CHILD_TABLE; SVgroupInfo vgroup = {vgid, 0, 0, {0}, 0}; - addEpIntoEpSet(&vgroup.epSet, "dnode_1", 6030); - addEpIntoEpSet(&vgroup.epSet, "dnode_2", 6030); - addEpIntoEpSet(&vgroup.epSet, "dnode_3", 6030); - vgroup.epSet.inUse = 0; + genEpSet(&vgroup.epSet); meta_[db][tbname]->vgs.emplace_back(vgroup); // super table @@ -268,10 +279,39 @@ class MockCatalogServiceImpl { udf_.insert(std::make_pair(func, info)); } + void createSmaIndex(const SMCreateSmaReq* pReq) { + STableIndexInfo info; + info.intervalUnit = pReq->intervalUnit; + info.slidingUnit = pReq->slidingUnit; + info.interval = pReq->interval; + info.offset = pReq->offset; + info.sliding = pReq->sliding; + info.dstTbUid = getNextId(); + info.dstVgId = pReq->dstVgId; + genEpSet(&info.epSet); + info.expr = strdup(pReq->expr); + auto it = index_.find(pReq->stb); + if (index_.end() == it) { + index_.insert(std::make_pair(std::string(pReq->stb), std::vector{info})); + } else { + it->second.push_back(info); + } + } + private: typedef std::map> TableMetaCache; typedef std::map DbMetaCache; typedef std::map> UdfMetaCache; + typedef std::map> IndexMetaCache; + + uint64_t getNextId() { return id_++; } + + void genEpSet(SEpSet* pEpSet) { + addEpIntoEpSet(pEpSet, "dnode_1", 6030); + addEpIntoEpSet(pEpSet, "dnode_2", 6030); + addEpIntoEpSet(pEpSet, "dnode_3", 6030); + pEpSet->inUse = 0; + } std::string toDbname(const std::string& dbFullName) const { std::string::size_type n = dbFullName.find("."); @@ -467,6 +507,7 @@ class MockCatalogServiceImpl { std::unique_ptr builder_; DbMetaCache meta_; UdfMetaCache udf_; + IndexMetaCache index_; }; MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) {} @@ -490,6 +531,8 @@ void MockCatalogService::createFunction(const std::string& func, int8_t funcType impl_->createFunction(func, funcType, outputType, outputLen, bufSize); } +void MockCatalogService::createSmaIndex(const SMCreateSmaReq* pReq) { impl_->createSmaIndex(pReq); } + int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { return impl_->catalogGetTableMeta(pTableName, pTableMeta); } @@ -510,6 +553,10 @@ int32_t MockCatalogService::catalogGetUdfInfo(const std::string& funcName, SFunc return impl_->catalogGetUdfInfo(funcName, pInfo); } +int32_t MockCatalogService::catalogGetTableIndex(const SName* pTableName, SArray** pIndexes) const { + return impl_->catalogGetTableIndex(pTableName, pIndexes); +} + int32_t MockCatalogService::catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const { return impl_->catalogGetAllMeta(pCatalogReq, pMetaData); } diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index fafad81e95..c4ab091b7a 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -64,6 +64,7 @@ class MockCatalogService { int32_t catalogGetTableDistVgInfo(const SName* pTableName, SArray** pVgList) const; int32_t catalogGetDBVgInfo(const char* pDbFName, SArray** pVgList) const; int32_t catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const; + int32_t catalogGetTableIndex(const SName* pTableName, SArray** pIndexes) const; int32_t catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const; private: diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index f252fe7265..76338eb491 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -32,8 +32,7 @@ typedef struct SOptimizeContext { bool optimized; } SOptimizeContext; -typedef int32_t (*FMatch)(SOptimizeContext* pCxt, SLogicNode* pLogicNode); -typedef int32_t (*FOptimize)(SOptimizeContext* pCxt, SLogicNode* pLogicNode); +typedef int32_t (*FOptimize)(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan); typedef struct SOptimizeRule { char* pName; @@ -109,7 +108,6 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { } if (QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode->pParent)) { return true; - // return (WINDOW_TYPE_INTERVAL == ((SWindowLogicNode*)pNode->pParent)->winType); } return !osdHaveNormalCol(((SAggLogicNode*)pNode->pParent)->pGroupKeys); } @@ -231,9 +229,9 @@ static void setScanWindowInfo(SScanLogicNode* pScan) { } } -static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { +static int32_t osdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SOsdInfo info = {0}; - int32_t code = osdMatch(pCxt, pLogicNode, &info); + int32_t code = osdMatch(pCxt, pLogicSubplan->pNode, &info); if (TSDB_CODE_SUCCESS == code && info.pScan) { setScanWindowInfo((SScanLogicNode*)info.pScan); } @@ -635,8 +633,8 @@ static int32_t cpdPushCondition(SOptimizeContext* pCxt, SLogicNode* pLogicNode) return code; } -static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { - return cpdPushCondition(pCxt, pLogicNode); +static int32_t cpdOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + return cpdPushCondition(pCxt, pLogicSubplan->pNode); } static bool opkIsPrimaryKeyOrderBy(SNodeList* pSortKeys) { @@ -745,26 +743,142 @@ static int32_t opkOptimizeImpl(SOptimizeContext* pCxt, SSortLogicNode* pSort) { return code; } -static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { - SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicNode, opkSortMayBeOptimized); +static int32_t opkOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { + SSortLogicNode* pSort = (SSortLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, opkSortMayBeOptimized); if (NULL == pSort) { return TSDB_CODE_SUCCESS; } return opkOptimizeImpl(pCxt, pSort); } -static const SOptimizeRule optimizeRuleSet[] = {{.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, - {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, - {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}}; +static int32_t smaOptFindSmaFunc(SNodeList* pSmaFuncs, SNode* pFunc, SNode** pCol, bool* pFound) { + int32_t index = 0; + SNode* pSmaFunc = NULL; + FOREACH(pSmaFunc, pSmaFuncs) { + if (nodesEqualNode(pSmaFunc, pFunc)) { + *pFound = true; + return collectSmaFunc(pCxt, index, pSmaFunc); + } + ++index; + } + return TSDB_CODE_SUCCESS; +} + +static bool smaOptCouldApplyIndex(SScanLogicNode* pScan, STableIndexInfo* pIndex) { + if (pScan->interval != pIndex->interval || pScan->intervalUnit != pIndex->intervalUnit || + pScan->offset != pIndex->offset || pScan->sliding != pIndex->sliding || + pScan->slidingUnit != pIndex->slidingUnit) { + return false; + } + // todo time range + SNodeList* pFuncs = NULL; + SNode* pFunc = NULL; + FOREACH(pFunc, ((SWindowLogicNode*)pScan->node.pParent)->pFuncs) {} + + return true; +} + +static bool smaOptMayBeOptimized(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) { + return false; + } + + SScanLogicNode* pScan = (SScanLogicNode*)pNode; + if (0 == pScan->interval || NULL == pScan->pSmaIndexes || NULL != pScan->node.pConditions) { + return false; + } + + int32_t size = taosArrayGetSize(pScan->pSmaIndexes); + for (int32_t i = 0; i < size; ++i) { + STableIndexInfo* pIndex = taosArrayGet(pScan->pSmaIndexes, i); + } + + return false; +} + +static int32_t smaOptCreateMerge(SNodeList* pTargets) { + SMergeLogicNode* pMerge = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_MERGE); + if (NULL == pMerge) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pMerge->node.precision = pPartChild->precision; + pMerge->pMergeKeys = pMergeKeys; + + int32_t code = TSDB_CODE_SUCCESS; + pMerge->pInputs = nodesCloneList(pPartChild->pTargets); + pMerge->node.pTargets = nodesCloneList(pTargets); + if (NULL == pMerge->node.pTargets || NULL == pMerge->pInputs) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + if (TSDB_CODE_SUCCESS == code) { + if (NULL == pSubplan) { + code = nodesListMakeAppend(&pSplitNode->pChildren, pMerge); + } else { + code = splReplaceLogicNode(pSubplan, pSplitNode, (SLogicNode*)pMerge); + } + } + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pMerge); + } + return code; +} + +static int32_t smaOptCreateSmaScan(SScanLogicNode* pScan, STableIndexInfo* pIndex, SNodeList* pCols, + SScanLogicNode** pOutput) { + SScanLogicNode* pSmaScan = nodesMakeNode(QUERY_NODE_LOGIC_PLAN_SCAN); + if (NULL == pSmaScan) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pSmaScan->pScanCols = pCols; + pSmaScan->tableType = TSDB_SUPER_TABLE; + pSmaScan->tableId = pIndex->dstTbUid; + pSmaScan->stableId = pIndex->dstTbUid; + pSmaScan->scanType = SCAN_TYPE_TABLE; + pSmaScan->scanSeq[0] = pScan->scanSeq[0]; + pSmaScan->scanSeq[1] = pScan->scanSeq[1]; + pSmaScan->scanRange = pScan->scanRange; + pSmaScan->dataRequired = FUNC_DATA_REQUIRED_DATA_LOAD; + + pSmaScan->pVgroupList = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo)); + if (NULL == pSmaScan->pVgroupList) { + nodesDestroyNode(pSmaScan); + return TSDB_CODE_OUT_OF_MEMORY; + } + pSmaScan->pVgroupList->numOfVgroups = 1; + pSmaScan->pVgroupList->vgroups[0].vgId = pIndex->dstVgId; + memcpy(&(pSmaScan->pVgroupList->vgroups[0].epSet), &pIndex->epSet, sizeof(SEpSet)); + + *pOutput = pSmaScan; + return TSDB_CODE_SUCCESS; +} + +static int32_t smaOptimizeImpl(SOptimizeContext* pCxt, SScanLogicNode* pScan) { return TSDB_CODE_SUCCESS; } + +static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicNode* pLogicNode) { + SScanLogicNode* pScan = (SScanLogicNode*)optFindPossibleNode(pLogicNode, opkSortMayBeOptimized); + if (NULL == pScan) { + return TSDB_CODE_SUCCESS; + } + return smaOptimizeImpl(pCxt, pScan); +} + +// clang-format off +static const SOptimizeRule optimizeRuleSet[] = { + {.pName = "OptimizeScanData", .optimizeFunc = osdOptimize}, + {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, + {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, + {.pName = "SmaIndex", .optimizeFunc = smaOptimize} +}; +// clang-format on static const int32_t optimizeRuleNum = (sizeof(optimizeRuleSet) / sizeof(SOptimizeRule)); -static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicNode* pLogicNode) { +static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { SOptimizeContext cxt = {.pPlanCxt = pCxt, .optimized = false}; do { cxt.optimized = false; for (int32_t i = 0; i < optimizeRuleNum; ++i) { - int32_t code = optimizeRuleSet[i].optimizeFunc(&cxt, pLogicNode); + int32_t code = optimizeRuleSet[i].optimizeFunc(&cxt, pLogicSubplan); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -774,5 +888,5 @@ static int32_t applyOptimizeRule(SPlanContext* pCxt, SLogicNode* pLogicNode) { } int32_t optimizeLogicPlan(SPlanContext* pCxt, SLogicSubplan* pLogicSubplan) { - return applyOptimizeRule(pCxt, pLogicSubplan->pNode); + return applyOptimizeRule(pCxt, pLogicSubplan); } diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index 04d76a8b91..85f8b7d9f6 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -43,6 +43,8 @@ TEST_F(PlanOtherTest, createSmaIndex) { useDb("root", "test"); run("CREATE SMA INDEX idx1 ON t1 FUNCTION(MAX(c1), MIN(c3 + 10), SUM(c4)) INTERVAL(10s)"); + + run("SELECT SUM(c4) FROM t1 INTERVAL(10s)"); } TEST_F(PlanOtherTest, explain) { diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 4e89f9d008..57d7cb6608 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -14,12 +14,14 @@ */ #include "planTestUtil.h" + #include #include #include #include "cmdnodes.h" +#include "mockCatalogService.h" #include "parser.h" #include "planInt.h" @@ -361,6 +363,7 @@ class PlannerTestBaseImpl { } else if (QUERY_NODE_CREATE_INDEX_STMT == nodeType(pQuery->pRoot)) { SMCreateSmaReq req = {0}; tDeserializeSMCreateSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req); + g_mockCatalogService->createSmaIndex(&req); nodesStringToNode(req.ast, &pCxt->pAstRoot); pCxt->streamQuery = true; } else if (QUERY_NODE_CREATE_STREAM_STMT == nodeType(pQuery->pRoot)) { diff --git a/source/util/src/tjson.c b/source/util/src/tjson.c index b15c188f04..45a2ffec77 100644 --- a/source/util/src/tjson.c +++ b/source/util/src/tjson.c @@ -14,6 +14,7 @@ */ #define _DEFAULT_SOURCE + #include "tjson.h" #include "cJSON.h" #include "taoserror.h" @@ -138,6 +139,23 @@ int32_t tjsonAddArray(SJson* pJson, const char* pName, FToJson func, const void* return TSDB_CODE_SUCCESS; } +int32_t tjsonAddTArray(SJson* pJson, const char* pName, FToJson func, const SArray* pArray) { + int32_t num = taosArrayGetSize(pArray); + if (num > 0) { + SJson* pJsonArray = tjsonAddArrayToObject(pJson, pName); + if (NULL == pJsonArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + for (int32_t i = 0; i < num; ++i) { + int32_t code = tjsonAddItem(pJsonArray, func, taosArrayGet(pArray, i)); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } + return TSDB_CODE_SUCCESS; +} + char* tjsonToString(const SJson* pJson) { return cJSON_Print((cJSON*)pJson); } char* tjsonToUnformattedString(const SJson* pJson) { return cJSON_PrintUnformatted((cJSON*)pJson); } @@ -184,7 +202,7 @@ int32_t tjsonGetBigIntValue(const SJson* pJson, const char* pName, int64_t* pVal return TSDB_CODE_FAILED; } #ifdef WINDOWS - sscanf(p,"%lld",pVal); + sscanf(p, "%lld", pVal); #else // sscanf(p,"%ld",pVal); *pVal = taosStr2Int64(p, NULL, 10); @@ -219,7 +237,7 @@ int32_t tjsonGetUBigIntValue(const SJson* pJson, const char* pName, uint64_t* pV return TSDB_CODE_FAILED; } #ifdef WINDOWS - sscanf(p,"%llu",pVal); + sscanf(p, "%llu", pVal); #else // sscanf(p,"%ld",pVal); *pVal = taosStr2UInt64(p, NULL, 10); @@ -299,24 +317,43 @@ int32_t tjsonToArray(const SJson* pJson, const char* pName, FToObject func, void return TSDB_CODE_SUCCESS; } +int32_t tjsonToTArray(const SJson* pJson, const char* pName, FToObject func, SArray** pArray, int32_t itemSize) { + const cJSON* jArray = tjsonGetObjectItem(pJson, pName); + int32_t size = tjsonGetArraySize(jArray); + if (size > 0) { + *pArray = taosArrayInit(size, itemSize); + if (NULL == *pArray) { + return TSDB_CODE_OUT_OF_MEMORY; + } + taosArraySetSize(*pArray, size); + for (int32_t i = 0; i < size; ++i) { + int32_t code = func(tjsonGetArrayItem(jArray, i), taosArrayGet(*pArray, i)); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + } + return TSDB_CODE_SUCCESS; +} + SJson* tjsonParse(const char* pStr) { return cJSON_Parse(pStr); } -bool tjsonValidateJson(const char *jIn) { - if (!jIn){ +bool tjsonValidateJson(const char* jIn) { + if (!jIn) { return false; } // set json real data - cJSON *root = cJSON_Parse(jIn); - if (root == NULL){ + cJSON* root = cJSON_Parse(jIn); + if (root == NULL) { return false; } - if(!cJSON_IsObject(root)){ + if (!cJSON_IsObject(root)) { return false; } int size = cJSON_GetArraySize(root); - for(int i = 0; i < size; i++) { + for (int i = 0; i < size; i++) { cJSON* item = cJSON_GetArrayItem(root, i); if (!item) { return false; -- GitLab