Merge pull request #11069 from taosdata/feature/3.0_wxy

sort bugfix, and pseudo column implement

include/common/ttokendef.h

@@ -140,45 +140,53 @@
 #define TK_AS                             122
 #define TK_NK_BOOL                        123
 #define TK_NK_VARIABLE                    124
-#define TK_BETWEEN                        125
-#define TK_IS                             126
-#define TK_NULL                           127
-#define TK_NK_LT                          128
-#define TK_NK_GT                          129
-#define TK_NK_LE                          130
-#define TK_NK_GE                          131
-#define TK_NK_NE                          132
-#define TK_MATCH                          133
-#define TK_NMATCH                         134
-#define TK_IN                             135
-#define TK_JOIN                           136
-#define TK_INNER                          137
-#define TK_SELECT                         138
-#define TK_DISTINCT                       139
-#define TK_WHERE                          140
-#define TK_PARTITION                      141
-#define TK_BY                             142
-#define TK_SESSION                        143
-#define TK_STATE_WINDOW                   144
-#define TK_SLIDING                        145
-#define TK_FILL                           146
-#define TK_VALUE                          147
-#define TK_NONE                           148
-#define TK_PREV                           149
-#define TK_LINEAR                         150
-#define TK_NEXT                           151
-#define TK_GROUP                          152
-#define TK_HAVING                         153
-#define TK_ORDER                          154
-#define TK_SLIMIT                         155
-#define TK_SOFFSET                        156
-#define TK_LIMIT                          157
-#define TK_OFFSET                         158
-#define TK_ASC                            159
-#define TK_DESC                           160
-#define TK_NULLS                          161
-#define TK_FIRST                          162
-#define TK_LAST                           163
+#define TK_NK_UNDERLINE                   125
+#define TK_ROWTS                          126
+#define TK_TBNAME                         127
+#define TK_QSTARTTS                       128
+#define TK_QENDTS                         129
+#define TK_WSTARTTS                       130
+#define TK_WENDTS                         131
+#define TK_WDURATION                      132
+#define TK_BETWEEN                        133
+#define TK_IS                             134
+#define TK_NULL                           135
+#define TK_NK_LT                          136
+#define TK_NK_GT                          137
+#define TK_NK_LE                          138
+#define TK_NK_GE                          139
+#define TK_NK_NE                          140
+#define TK_MATCH                          141
+#define TK_NMATCH                         142
+#define TK_IN                             143
+#define TK_JOIN                           144
+#define TK_INNER                          145
+#define TK_SELECT                         146
+#define TK_DISTINCT                       147
+#define TK_WHERE                          148
+#define TK_PARTITION                      149
+#define TK_BY                             150
+#define TK_SESSION                        151
+#define TK_STATE_WINDOW                   152
+#define TK_SLIDING                        153
+#define TK_FILL                           154
+#define TK_VALUE                          155
+#define TK_NONE                           156
+#define TK_PREV                           157
+#define TK_LINEAR                         158
+#define TK_NEXT                           159
+#define TK_GROUP                          160
+#define TK_HAVING                         161
+#define TK_ORDER                          162
+#define TK_SLIMIT                         163
+#define TK_SOFFSET                        164
+#define TK_LIMIT                          165
+#define TK_OFFSET                         166
+#define TK_ASC                            167
+#define TK_DESC                           168
+#define TK_NULLS                          169
+#define TK_FIRST                          170
+#define TK_LAST                           171
 
 #define TK_NK_SPACE                       300
 #define TK_NK_COMMENT                     301

include/libs/function/functionMgt.h

@@ -96,7 +96,16 @@ typedef enum EFunctionType {
   FUNCTION_TYPE_SERVER_SERSION,
   FUNCTION_TYPE_SERVER_STATUS,
   FUNCTION_TYPE_CURRENT_USER,
-  FUNCTION_TYPE_USER
+  FUNCTION_TYPE_USER,
+
+  // pseudo column function
+  FUNCTION_TYPE_ROWTS = 3500,
+  FUNCTION_TYPE_TBNAME,
+  FUNCTION_TYPE_QSTARTTS,
+  FUNCTION_TYPE_QENDTS,
+  FUNCTION_TYPE_WSTARTTS,
+  FUNCTION_TYPE_WENDTS,
+  FUNCTION_TYPE_WDURATION
 } EFunctionType;
 
 struct SqlFunctionCtx;
@@ -118,6 +127,8 @@ bool fmIsStringFunc(int32_t funcId);
 bool fmIsDatetimeFunc(int32_t funcId);
 bool fmIsTimelineFunc(int32_t funcId);
 bool fmIsTimeorderFunc(int32_t funcId);
+bool fmIsWindowPseudoColumnFunc(int32_t funcId);
+bool fmIsWindowClauseFunc(int32_t funcId);
 
 int32_t fmFuncScanType(int32_t funcId);
 

source/libs/function/inc/builtins.h

@@ -33,6 +33,8 @@ extern "C" {
 #define FUNC_MGT_DATETIME_FUNC        FUNC_MGT_FUNC_CLASSIFICATION_MASK(4)
 #define FUNC_MGT_TIMELINE_FUNC        FUNC_MGT_FUNC_CLASSIFICATION_MASK(5)
 #define FUNC_MGT_TIMEORDER_FUNC       FUNC_MGT_FUNC_CLASSIFICATION_MASK(6)
+#define FUNC_MGT_PSEUDO_COLUMN_FUNC   FUNC_MGT_FUNC_CLASSIFICATION_MASK(7)
+#define FUNC_MGT_WINDOW_PC_FUNC       FUNC_MGT_FUNC_CLASSIFICATION_MASK(8)
 
 #define FUNC_MGT_TEST_MASK(val, mask) (((val) & (mask)) != 0)
 

source/libs/function/src/builtins.c

@@ -291,6 +291,76 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = {
     .initFunc     = NULL,
     .sprocessFunc = NULL,
     .finalizeFunc = NULL
+  },
+  {
+    .name = "_rowts",
+    .type = FUNCTION_TYPE_ROWTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "tbname",
+    .type = FUNCTION_TYPE_TBNAME,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "_qstartts",
+    .type = FUNCTION_TYPE_QSTARTTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "_qendts",
+    .type = FUNCTION_TYPE_QENDTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "_wstartts",
+    .type = FUNCTION_TYPE_QSTARTTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "_wendts",
+    .type = FUNCTION_TYPE_QENDTS,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
+  },
+  {
+    .name = "_wduration",
+    .type = FUNCTION_TYPE_WDURATION,
+    .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_WINDOW_PC_FUNC,
+    .checkFunc    = stubCheckAndGetResultType,
+    .getEnvFunc   = NULL,
+    .initFunc     = NULL,
+    .sprocessFunc = NULL,
+    .finalizeFunc = NULL
   }
 };
 
@@ -329,6 +399,13 @@ int32_t stubCheckAndGetResultType(SFunctionNode* pFunc) {
       break;
     }
     case FUNCTION_TYPE_CONCAT:
+    case FUNCTION_TYPE_ROWTS:
+    case FUNCTION_TYPE_TBNAME:
+    case FUNCTION_TYPE_QSTARTTS:
+    case FUNCTION_TYPE_QENDTS:
+    case FUNCTION_TYPE_WSTARTTS:
+    case FUNCTION_TYPE_WENDTS:
+    case FUNCTION_TYPE_WDURATION:
       // todo
       break;
 

source/libs/function/src/functionMgt.c

@@ -44,6 +44,13 @@ static void doInitFunctionHashTable() {
   }
 }
 
+static bool isSpecificClassifyFunc(int32_t funcId, uint64_t classification) {
+  if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
+    return false;
+  }
+  return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, classification);
+}
+
 int32_t fmFuncMgtInit() {
   taosThreadOnce(&functionHashTableInit, doInitFunctionHashTable);
   return initFunctionCode;
@@ -89,10 +96,19 @@ int32_t fmGetScalarFuncExecFuncs(int32_t funcId, SScalarFuncExecFuncs* pFpSet) {
 }
 
 bool fmIsAggFunc(int32_t funcId) {
-  if (funcId < 0 || funcId >= funcMgtBuiltinsNum) {
-    return false;
-  }
-  return FUNC_MGT_TEST_MASK(funcMgtBuiltins[funcId].classification, FUNC_MGT_AGG_FUNC);
+  return isSpecificClassifyFunc(funcId, FUNC_MGT_AGG_FUNC);
+}
+
+bool fmIsScalarFunc(int32_t funcId) {
+  return isSpecificClassifyFunc(funcId, FUNC_MGT_SCALAR_FUNC);
+}
+
+bool fmIsWindowPseudoColumnFunc(int32_t funcId) {
+  return isSpecificClassifyFunc(funcId, FUNC_MGT_WINDOW_PC_FUNC);
+}
+
+bool fmIsWindowClauseFunc(int32_t funcId) {
+  return fmIsAggFunc(funcId) || fmIsWindowPseudoColumnFunc(funcId);
 }
 
 void fmFuncMgtDestroy() {

source/libs/parser/inc/sql.y

@@ -15,6 +15,7 @@
 #include <assert.h>
 #include <stdbool.h>
 
+#include "functionMgt.h"
 #include "nodes.h"
 #include "parToken.h"
 #include "ttokendef.h"
@@ -417,7 +418,7 @@ topic_name(A) ::= NK_ID(B).
 /************************************************ expression **********************************************************/
 expression(A) ::= literal(B).                                                     { A = B; }
 //expression(A) ::= NK_QUESTION(B).                                                 { A = B; }
-//expression(A) ::= pseudo_column(B).                                               { A = B; }
+expression(A) ::= pseudo_column(B).                                               { A = B; }
 expression(A) ::= column_reference(B).                                            { A = B; }
 expression(A) ::= function_name(B) NK_LP expression_list(C) NK_RP(D).             { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, C)); }
 expression(A) ::= function_name(B) NK_LP NK_STAR(C) NK_RP(D).                     { A = createRawExprNodeExt(pCxt, &B, &D, createFunctionNode(pCxt, &B, createNodeList(pCxt, createColumnNode(pCxt, NULL, &C)))); }
@@ -467,7 +468,38 @@ expression_list(A) ::= expression_list(B) NK_COMMA expression(C).
 column_reference(A) ::= column_name(B).                                           { A = createRawExprNode(pCxt, &B, createColumnNode(pCxt, NULL, &B)); }
 column_reference(A) ::= table_name(B) NK_DOT column_name(C).                      { A = createRawExprNodeExt(pCxt, &B, &C, createColumnNode(pCxt, &B, &C)); }
 
-//pseudo_column(A) ::= NK_NOW.                                                      { A = createFunctionNode(pCxt, NULL, NULL); }
+//pseudo_column(A) ::= NK_NOW.                                                    { A = createFunctionNode(pCxt, NULL, NULL); }
+pseudo_column(A) ::=  NK_UNDERLINE(B) ROWTS(C).                                   {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
+pseudo_column(A) ::=  TBNAME(B).                                                  { A = createRawExprNode(pCxt, &B, createFunctionNode(pCxt, &B, NULL)); }
+pseudo_column(A) ::=  NK_UNDERLINE(B) QSTARTTS(C).                                {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
+pseudo_column(A) ::=  NK_UNDERLINE(B) QENDTS(C).                                  {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
+pseudo_column(A) ::=  NK_UNDERLINE(B) WSTARTTS(C).                                {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
+pseudo_column(A) ::=  NK_UNDERLINE(B) WENDTS(C).                                  {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
+pseudo_column(A) ::=  NK_UNDERLINE(B) WDURATION(C).                               {
+                                                                                    SToken t = B;
+                                                                                    t.n = (C.z + C.n) - B.z;
+                                                                                    A = createRawExprNode(pCxt, &t, createFunctionNode(pCxt, &t, NULL));
+                                                                                  }
 
 /************************************************ predicate ***********************************************************/
 predicate(A) ::= expression(B) compare_op(C) expression(D).                       {

source/libs/parser/src/parTokenizer.c

@@ -104,12 +104,15 @@ static SKeyword keywordTable[] = {
     {"PRECISION",     TK_PRECISION},
     {"PRIVILEGE",     TK_PRIVILEGE},
     {"PREV",          TK_PREV},
+    {"QENDTS",        TK_QENDTS},
     {"QNODE",         TK_QNODE},
     {"QNODES",        TK_QNODES},
+    {"QSTARTTS",      TK_QSTARTTS},
     {"QUORUM",        TK_QUORUM},
     {"REPLICA",       TK_REPLICA},
     {"RETENTIONS",    TK_RETENTIONS},
     {"ROLLUP",        TK_ROLLUP},
+    {"ROWTS",         TK_ROWTS},
     {"SELECT",        TK_SELECT},
     {"SESSION",       TK_SESSION},
     {"SHOW",          TK_SHOW},
@@ -127,6 +130,7 @@ static SKeyword keywordTable[] = {
     {"TABLE",         TK_TABLE},
     {"TABLES",        TK_TABLES},
     {"TAGS",          TK_TAGS},
+    {"TBNAME",        TK_TBNAME},
     {"TIMESTAMP",     TK_TIMESTAMP},
     {"TINYINT",       TK_TINYINT},
     {"TOPIC",         TK_TOPIC},
@@ -141,7 +145,10 @@ static SKeyword keywordTable[] = {
     {"VARCHAR",       TK_VARCHAR},
     {"VGROUPS",       TK_VGROUPS},
     {"WAL",           TK_WAL},
+    {"WDURATION",     TK_WDURATION},
+    {"WENDTS",        TK_WENDTS},
     {"WHERE",         TK_WHERE},
+    {"WSTARTTS",      TK_WSTARTTS},
     // {"ID",           TK_ID},
     // {"STRING",       TK_STRING},
     // {"EQ",           TK_EQ},
@@ -233,7 +240,6 @@ static SKeyword keywordTable[] = {
     // {"TRIGGER",      TK_TRIGGER},
     // {"VIEW",         TK_VIEW},
     // {"SEMI",         TK_SEMI},
-    // {"TBNAME",       TK_TBNAME},
     // {"VNODES",       TK_VNODES},
 //    {"PARTITIONS",   TK_PARTITIONS},
     // {"TOPICS",       TK_TOPICS},
@@ -427,6 +433,10 @@ uint32_t tGetToken(const char* z, uint32_t* tokenId) {
       *tokenId = TK_NK_QUESTION;
       return 1;
     }
+    case '_': {
+      *tokenId = TK_NK_UNDERLINE;
+      return 1;
+    }
     case '`':
     case '\'':
     case '"': {

source/libs/parser/test/parserAstTest.cpp

@@ -206,6 +206,13 @@ TEST_F(ParserTest, selectExpression) {
   ASSERT_TRUE(run());
 }
 
+TEST_F(ParserTest, selectPseudoColumn) {
+  setDatabase("root", "test");
+
+  bind("SELECT _wstartts, _wendts, count(*) FROM t1 interval(10s)");
+  ASSERT_TRUE(run());
+}
+
 TEST_F(ParserTest, selectClause) {
   setDatabase("root", "test");
 

source/libs/planner/src/planLogicCreater.c

@@ -411,7 +411,7 @@ static int32_t createAggLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect,
 }
 
 static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SWindowLogicNode* pWindow, SLogicNode** pLogicNode) {
-  int32_t code = nodesCollectFuncs(pSelect, fmIsAggFunc, &pWindow->pFuncs);
+  int32_t code = nodesCollectFuncs(pSelect, fmIsWindowClauseFunc, &pWindow->pFuncs);
 
   if (TSDB_CODE_SUCCESS == code) {
     code = rewriteExpr(pWindow->pFuncs, pSelect, SQL_CLAUSE_WINDOW);

source/libs/planner/src/planPhysiCreater.c

@@ -36,10 +36,6 @@ typedef struct SPhysiPlanContext {
 } SPhysiPlanContext;
 
 static int32_t getSlotKey(SNode* pNode, const char* pStmtName, char* pKey) {
-  if (QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode)) {
-    return getSlotKey(((SOrderByExprNode*)pNode)->pExpr, pStmtName, pKey);
-  }
-
   if (QUERY_NODE_COLUMN == nodeType(pNode)) {
     SColumnNode* pCol = (SColumnNode*)pNode;
     if (NULL != pStmtName) {
@@ -184,15 +180,16 @@ static int32_t addDataBlockSlotsImpl(SPhysiPlanContext* pCxt, SNodeList* pList,
   int16_t nextSlotId = taosHashGetSize(pHash), slotId = 0;
   SNode* pNode = NULL;
   FOREACH(pNode, pList) {
+    SNode* pExpr = QUERY_NODE_ORDER_BY_EXPR == nodeType(pNode) ? ((SOrderByExprNode*)pNode)->pExpr : pNode;
     char name[TSDB_TABLE_NAME_LEN + TSDB_COL_NAME_LEN] = {0};
-    int32_t len = getSlotKey(pNode, pStmtName, name);
+    int32_t len = getSlotKey(pExpr, pStmtName, name);
     SSlotIndex* pIndex = taosHashGet(pHash, name, len);
     if (NULL == pIndex) {
-      code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pNode, nextSlotId, output));
+      code = nodesListStrictAppend(pDataBlockDesc->pSlots, createSlotDesc(pCxt, pExpr, nextSlotId, output));
       if (TSDB_CODE_SUCCESS == code) {
         code = putSlotToHashImpl(pDataBlockDesc->dataBlockId, nextSlotId, name, len, pHash);
       }
-      pDataBlockDesc->resultRowSize += ((SExprNode*)pNode)->resType.bytes;
+      pDataBlockDesc->resultRowSize += ((SExprNode*)pExpr)->resType.bytes;
       slotId = nextSlotId;
       ++nextSlotId;
     } else {
@@ -600,7 +597,7 @@ static EDealRes doRewritePrecalcExprs(SNode** pNode, void* pContext) {
       return collectAndRewrite(pContext, pNode);
     }
     case QUERY_NODE_FUNCTION: {
-      if (!fmIsAggFunc(((SFunctionNode*)(*pNode))->funcId)) {
+      if (fmIsScalarFunc(((SFunctionNode*)(*pNode))->funcId)) {
         return collectAndRewrite(pContext, pNode);
       }
     }

source/libs/planner/test/plannerTest.cpp

@@ -192,6 +192,9 @@ TEST_F(PlannerTest, interval) {
 
   bind("SELECT count(*) FROM t1 interval(10s)");
   ASSERT_TRUE(run());
+
+  bind("SELECT _wstartts, _wduration, _wendts, count(*) FROM t1 interval(10s)");
+  ASSERT_TRUE(run());
 }
 
 TEST_F(PlannerTest, sessionWindow) {