From 022f5bfc140ad969c681cb7cfe0f1a9fb5baf096 Mon Sep 17 00:00:00 2001 From: Xiaoyu Wang Date: Fri, 29 Apr 2022 20:06:26 +0800 Subject: [PATCH] feat: rewrite timeline function --- include/libs/nodes/querynodes.h | 1 + source/libs/function/src/builtins.c | 13 +- source/libs/function/src/functionMgt.c | 5 +- source/libs/parser/src/parAstCreater.c | 1 + source/libs/parser/src/parCalcConst.c | 2 +- source/libs/parser/src/parTranslater.c | 159 +++++++++++++------ source/libs/parser/test/parSelectTest.cpp | 12 ++ source/libs/planner/test/planBasicTest.cpp | 14 +- source/libs/planner/test/planGroupByTest.cpp | 8 + 9 files changed, 153 insertions(+), 62 deletions(-) diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index b32d288d43..c88ee65727 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -230,6 +230,7 @@ typedef struct SSelectStmt { uint8_t precision; bool isEmptyResult; bool hasAggFuncs; + bool isTimeOrderQuery; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 1e0b9b3130..8f3c88900d 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -221,7 +221,7 @@ static int32_t translateSpread(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = (SDataType) { .bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE }; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; return TSDB_CODE_SUCCESS; } @@ -256,8 +256,7 @@ static int32_t translateLength(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } - pFunc->node.resType = - (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; return TSDB_CODE_SUCCESS; } @@ -439,6 +438,7 @@ static int32_t translateToJson(SFunctionNode* pFunc, char* pErrBuf, int32_t len) return TSDB_CODE_SUCCESS; } +// clang-format off const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "count", @@ -568,7 +568,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "first", .type = FUNCTION_TYPE_FIRST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -578,7 +578,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "last", .type = FUNCTION_TYPE_LAST, - .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_MULTI_RES_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateFirstLast, .getEnvFunc = getFirstLastFuncEnv, .initFunc = functionSetup, @@ -588,7 +588,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "diff", .type = FUNCTION_TYPE_DIFF, - .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC, + .classification = FUNC_MGT_NONSTANDARD_SQL_FUNC | FUNC_MGT_TIMELINE_FUNC, .translateFunc = translateInOutNum, .getEnvFunc = getDiffFuncEnv, .initFunc = diffFunctionSetup, @@ -976,5 +976,6 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .finalizeFunc = NULL } }; +// clang-format on const int32_t funcMgtBuiltinsNum = (sizeof(funcMgtBuiltins) / sizeof(SBuiltinFuncDefinition)); diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 0113da94eb..b505f2e8ec 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -21,11 +21,8 @@ #include "taos.h" #include "taoserror.h" #include "thash.h" -#include "builtins.h" -#include "catalog.h" #include "tudf.h" - typedef struct SFuncMgtService { SHashObj* pFuncNameHashTable; } SFuncMgtService; @@ -148,6 +145,8 @@ bool fmIsAggFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MG bool fmIsScalarFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC); } +bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); } + bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); } bool fmIsScanPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCAN_PC_FUNC); } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 20e1bba77f..1701976f8d 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -586,6 +586,7 @@ SNode* createSelectStmt(SAstCreateContext* pCxt, bool isDistinct, SNodeList* pPr select->pProjectionList = pProjectionList; select->pFromTable = pTable; sprintf(select->stmtName, "%p", select); + select->isTimeOrderQuery = true; return (SNode*)select; } diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 0aa4a3e6cb..9c2bd10686 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -240,7 +240,7 @@ static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bo code = calcConstNode(&pSelect->pWindow); } if (TSDB_CODE_SUCCESS == code) { - code = calcConstList(pSelect->pGroupByList); + code = calcConstGroupBy(pCxt, pSelect); } if (TSDB_CODE_SUCCESS == code) { code = calcConstSelectCondition(pCxt, pSelect, &pSelect->pHaving); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 5421821f7c..645dbd7288 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -319,7 +319,7 @@ static void setColumnInfoByExpr(const STableNode* pTable, SExprNode* pExpr, SCol pCol->node.resType = pExpr->resType; } -static int32_t createColumnNodeByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) { +static int32_t createColumnsByTable(STranslateContext* pCxt, const STableNode* pTable, SNodeList* pList) { if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { const STableMeta* pMeta = ((SRealTableNode*)pTable)->pMeta; int32_t nums = @@ -603,6 +603,7 @@ static EDealRes translateFunction(STranslateContext* pCxt, SFunctionNode* pFunc) return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_AGG_FUNC_NESTING); } pCxt->pCurrStmt->hasAggFuncs = true; + pCxt->pCurrStmt->isTimeOrderQuery = false; } return DEAL_RES_CONTINUE; @@ -899,7 +900,7 @@ static int32_t createAllColumns(STranslateContext* pCxt, SNodeList** pCols) { size_t nums = taosArrayGetSize(pTables); for (size_t i = 0; i < nums; ++i) { STableNode* pTable = taosArrayGetP(pTables, i); - int32_t code = createColumnNodeByTable(pCxt, pTable, *pCols); + int32_t code = createColumnsByTable(pCxt, pTable, *pCols); if (TSDB_CODE_SUCCESS != code) { return code; } @@ -971,7 +972,7 @@ static int32_t createTableAllCols(STranslateContext* pCxt, SColumnNode* pCol, SN } } if (TSDB_CODE_SUCCESS == code) { - code = createColumnNodeByTable(pCxt, pTable, *pOutput); + code = createColumnsByTable(pCxt, pTable, *pOutput); } return code; } @@ -1053,52 +1054,33 @@ static int32_t createMultiResFuncsFromStar(STranslateContext* pCxt, SFunctionNod return code; } -static bool isCountStar(SNode* pNode) { - if (QUERY_NODE_FUNCTION != nodeType(pNode) || 1 != LIST_LENGTH(((SFunctionNode*)pNode)->pParameterList)) { - return false; - } - SNode* pPara = nodesListGetNode(((SFunctionNode*)pNode)->pParameterList, 0); - return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); -} - -static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { - SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); - STableNode* pTable = NULL; - int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); - if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { - setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); - } - return code; -} - static int32_t translateStar(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL == pSelect->pProjectionList) { // select * ... return createAllColumns(pCxt, &pSelect->pProjectionList); } else { SNode* pNode = NULL; WHERE_EACH(pNode, pSelect->pProjectionList) { + int32_t code = TSDB_CODE_SUCCESS; if (isMultiResFunc(pNode)) { SNodeList* pFuncs = NULL; - if (TSDB_CODE_SUCCESS != createMultiResFuncsFromStar(pCxt, (SFunctionNode*)pNode, &pFuncs)) { - return TSDB_CODE_OUT_OF_MEMORY; + code = createMultiResFuncsFromStar(pCxt, (SFunctionNode*)pNode, &pFuncs); + if (TSDB_CODE_SUCCESS == code) { + INSERT_LIST(pSelect->pProjectionList, pFuncs); + ERASE_NODE(pSelect->pProjectionList); + continue; } - INSERT_LIST(pSelect->pProjectionList, pFuncs); - ERASE_NODE(pSelect->pProjectionList); - continue; } else if (isTableStar(pNode)) { SNodeList* pCols = NULL; - if (TSDB_CODE_SUCCESS != createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols)) { - return TSDB_CODE_OUT_OF_MEMORY; - } - INSERT_LIST(pSelect->pProjectionList, pCols); - ERASE_NODE(pSelect->pProjectionList); - continue; - } else if (isCountStar(pNode)) { - int32_t code = rewriteCountStar(pCxt, (SFunctionNode*)pNode); - if (TSDB_CODE_SUCCESS != code) { - return code; + code = createTableAllCols(pCxt, (SColumnNode*)pNode, &pCols); + if (TSDB_CODE_SUCCESS == code) { + INSERT_LIST(pSelect->pProjectionList, pCols); + ERASE_NODE(pSelect->pProjectionList); + continue; } } + if (TSDB_CODE_SUCCESS != code) { + return code; + } WHERE_NEXT; } } @@ -1214,12 +1196,14 @@ static int32_t translateGroupBy(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL != pSelect->pGroupByList && NULL != pSelect->pWindow) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_WINDOW_COEXIST); } - pCxt->currClause = SQL_CLAUSE_GROUP_BY; - return translateExprList(pCxt, pSelect->pGroupByList); + if (NULL != pSelect->pGroupByList) { + pCxt->currClause = SQL_CLAUSE_GROUP_BY; + pSelect->isTimeOrderQuery = false; + return translateExprList(pCxt, pSelect->pGroupByList); + } + return TSDB_CODE_SUCCESS; } -static bool isValTimeUnit(char unit) { return ('n' == unit || 'y' == unit); } - static int64_t getMonthsFromTimeVal(int64_t val, int32_t fromPrecision, char unit) { int64_t days = convertTimeFromPrecisionToUnit(val, fromPrecision, 'd'); switch (unit) { @@ -1246,7 +1230,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* uint8_t precision = ((SColumnNode*)pInterval->pCol)->node.resType.precision; SValueNode* pInter = (SValueNode*)pInterval->pInterval; - bool valInter = isValTimeUnit(pInter->unit); + bool valInter = TIME_IS_VAR_DURATION(pInter->unit); if (pInter->datum.i <= 0 || (!valInter && convertTimePrecision(pInter->datum.i, precision, TSDB_TIME_PRECISION_MICRO) < tsMinIntervalTime)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_VALUE_TOO_SMALL, tsMinIntervalTime); @@ -1260,7 +1244,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* if (pInter->unit == 'n' && pOffset->unit == 'y') { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_OFFSET_UNIT); } - bool fixed = !isValTimeUnit(pOffset->unit) && !valInter; + bool fixed = !TIME_IS_VAR_DURATION(pOffset->unit) && !valInter; if ((fixed && pOffset->datum.i >= pInter->datum.i) || (!fixed && getMonthsFromTimeVal(pOffset->datum.i, precision, pOffset->unit) >= getMonthsFromTimeVal(pInter->datum.i, precision, pInter->unit))) { @@ -1272,7 +1256,7 @@ static int32_t checkIntervalWindow(STranslateContext* pCxt, SIntervalWindowNode* const static int32_t INTERVAL_SLIDING_FACTOR = 100; SValueNode* pSliding = (SValueNode*)pInterval->pSliding; - if (pInter->unit == 'n' || pInter->unit == 'y') { + if (TIME_IS_VAR_DURATION(pSliding->unit)) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INTER_SLIDING_UNIT); } if ((pSliding->datum.i < convertTimePrecision(tsMinSlidingTime, TSDB_TIME_PRECISION_MILLI, precision)) || @@ -1379,6 +1363,78 @@ static int32_t checkLimit(STranslateContext* pCxt, SSelectStmt* pSelect) { return TSDB_CODE_SUCCESS; } +static bool isCountStar(SFunctionNode* pFunc) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return false; + } + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + return (QUERY_NODE_COLUMN == nodeType(pPara) && 0 == strcmp(((SColumnNode*)pPara)->colName, "*")); +} + +// count(*) is rewritten as count(ts) for scannning optimization +static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) { + SColumnNode* pCol = nodesListGetNode(pCount->pParameterList, 0); + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, ('\0' == pCol->tableAlias[0] ? NULL : pCol->tableAlias), &pTable); + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + } + return code; +} + +static int32_t createPrimaryKeyColByTable(STranslateContext* pCxt, STableNode* pTable, SNode** pPrimaryKey) { + SColumnNode* pCol = nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + return TSDB_CODE_OUT_OF_MEMORY; + } + if (QUERY_NODE_REAL_TABLE == nodeType(pTable)) { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, false, pCol); + } else { + // todo + } + *pPrimaryKey = (SNode*)pCol; + return TSDB_CODE_SUCCESS; +} + +static int32_t createPrimaryKeyCol(STranslateContext* pCxt, SNode** pPrimaryKey) { + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, NULL, &pTable); + if (TSDB_CODE_SUCCESS == code) { + code = createPrimaryKeyColByTable(pCxt, pTable, pPrimaryKey); + } + return code; +} + +static int32_t rewriteTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + SNode* pPrimaryKey = NULL; + int32_t code = createPrimaryKeyCol(pCxt, &pPrimaryKey); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pFunc->pParameterList, pPrimaryKey); + } + return code; +} + +EDealRes rewriteFuncForSelectImpl(SNode* pNode, void* pContext) { + if (QUERY_NODE_FUNCTION == nodeType(pNode)) { + STranslateContext* pCxt = pContext; + SFunctionNode* pFunc = (SFunctionNode*)pNode; + if (isCountStar(pFunc)) { + pCxt->errCode = rewriteCountStar(pCxt, pFunc); + } else if (fmIsTimelineFunc(pFunc->funcId)) { + pCxt->errCode = rewriteTimelineFunc(pCxt, pFunc); + } + if (TSDB_CODE_SUCCESS != pCxt->errCode) { + return DEAL_RES_ERROR; + } + } + return DEAL_RES_CONTINUE; +} + +static int32_t rewriteFuncForSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { + nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, rewriteFuncForSelectImpl, pCxt); + return pCxt->errCode; +} + static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrStmt = pSelect; int32_t code = translateFrom(pCxt, pSelect); @@ -1409,6 +1465,9 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { if (TSDB_CODE_SUCCESS == code) { code = checkLimit(pCxt, pSelect); } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteFuncForSelect(pCxt, pSelect); + } return code; } @@ -1663,13 +1722,17 @@ static int32_t checkDbRetentionsOption(STranslateContext* pCxt, SNodeList* pRete return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_RETENTIONS_OPTION); } - SNode* pNode = NULL; - FOREACH(pNode, pRetentions) { - SNode* pVal = NULL; - FOREACH(pVal, ((SNodeListNode*)pNode)->pNodeList) { - if (DEAL_RES_ERROR == translateValue(pCxt, (SValueNode*)pVal)) { + SNode* pRetention = NULL; + FOREACH(pRetention, pRetentions) { + SNode* pNode = NULL; + FOREACH(pNode, ((SNodeListNode*)pRetention)->pNodeList) { + SValueNode* pVal = (SValueNode*)pNode; + if (DEAL_RES_ERROR == translateValue(pCxt, pVal)) { return pCxt->errCode; } + if (!TIME_IS_VAR_DURATION(pVal->unit)) { + pVal->datum.i = convertTimeFromPrecisionToUnit(pVal->datum.i, pVal->node.resType.precision, pVal->unit); + } } } diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 990c272612..4c3a4e8ab9 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -80,6 +80,18 @@ TEST_F(ParserSelectTest, multiResFunc) { run("select last(t2.*), first(t1.c1, t2.*), last_row(t1.*, t2.*) from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); } +TEST_F(ParserSelectTest, timelineFunc) { + useDb("root", "test"); + + run("select last(*), first(*) from t1"); + + run("select last(*), first(*) from t1 group by c1"); + + run("select last(*), first(*) from t1 interval(10s)"); + + run("select diff(c1) from t1"); +} + TEST_F(ParserSelectTest, clause) { useDb("root", "test"); diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index 26b1fb8ece..05a009d79d 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -23,9 +23,9 @@ class PlanBasicTest : public PlannerTestBase {}; TEST_F(PlanBasicTest, select) { useDb("root", "test"); - // run("select * from t1"); - // run("select 1 from t1"); - // run("select * from st1"); + run("select * from t1"); + run("select 1 from t1"); + run("select * from st1"); run("select 1 from st1"); } @@ -40,4 +40,10 @@ TEST_F(PlanBasicTest, join) { run("select t1.c1, t2.c2 from st1s1 t1, st1s2 t2 where t1.ts = t2.ts"); run("select t1.c1, t2.c2 from st1s1 t1 join st1s2 t2 on t1.ts = t2.ts"); -} \ No newline at end of file +} + +TEST_F(PlanBasicTest, func) { + useDb("root", "test"); + + run("select diff(c1) from t1"); +} diff --git a/source/libs/planner/test/planGroupByTest.cpp b/source/libs/planner/test/planGroupByTest.cpp index 73fcffd729..05ffe41fe7 100644 --- a/source/libs/planner/test/planGroupByTest.cpp +++ b/source/libs/planner/test/planGroupByTest.cpp @@ -42,3 +42,11 @@ TEST_F(PlanGroupByTest, withOrderBy) { // order by alias of aggfunc // run("select count(*), sum(c1) a from t1 order by a"); } + +TEST_F(PlanGroupByTest, aggFunc) { + useDb("root", "test"); + + run("select last(*), first(*) from t1"); + + run("select last(*), first(*) from t1 group by c1"); +} -- GitLab