refactor some codes. fix bugs in selectivity+tags/ts query.

src/client/src/tscSQLParser.c

@@ -121,7 +121,6 @@ static int32_t doFunctionsCompatibleCheck(SSqlObj* pSql);
 static int32_t tscQueryOnlyMetricTags(SSqlCmd* pCmd, bool* queryOnMetricTags) {
   assert(QUERY_IS_STABLE_QUERY(pCmd->type));
 
-  // here colIdx == -1 means the special column tbname that is the name of each table
   *queryOnMetricTags = true;
   for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) {
     SSqlExpr* pExpr = tscSqlExprGet(pCmd, i);
@@ -1151,7 +1150,7 @@ int32_t parseIntervalClause(SSqlCmd* pCmd, SQuerySQL* pQuerySql) {
   }
 
   // check the invalid sql expresssion: select count(tbname)/count(tag1)/count(tag2) from super_table interval(1d);
-  for(int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) {
+  for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) {
     SSqlExpr* pExpr = tscSqlExprGet(pCmd, i);
     if (pExpr->functionId == TSDB_FUNC_COUNT && TSDB_COL_IS_TAG(pExpr->colInfo.flag)) {
       setErrMsg(pCmd, msg1);
@@ -2791,10 +2790,14 @@ static bool functionCompatibleCheck(SSqlCmd* pCmd) {
   // diff function cannot be executed with other function
   // arithmetic function can be executed with other arithmetic functions
   for (int32_t i = startIdx + 1; i < pCmd->fieldsInfo.numOfOutputCols; ++i) {
-    int16_t functionId = tscSqlExprGet(pCmd, i)->functionId;
-    if (functionId == TSDB_FUNC_TAGPRJ ||
-        functionId == TSDB_FUNC_TAG ||
-        functionId == TSDB_FUNC_TS) {
+    SSqlExpr* pExpr = tscSqlExprGet(pCmd, i);
+
+    int16_t functionId = pExpr->functionId;
+    if (functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TS) {
+      continue;
+    }
+
+    if (functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
       continue;
     }
 
@@ -2809,9 +2812,7 @@ static bool functionCompatibleCheck(SSqlCmd* pCmd) {
     for (int32_t i = 0; i < pCmd->fieldsInfo.numOfOutputCols; ++i) {
       int16_t functionId = tscSqlExprGet(pCmd, i)->functionId;
 
-      if (functionId == TSDB_FUNC_PRJ ||
-          functionId == TSDB_FUNC_TAGPRJ ||
-          functionId == TSDB_FUNC_TS ||
+      if (functionId == TSDB_FUNC_PRJ || functionId == TSDB_FUNC_TAGPRJ || functionId == TSDB_FUNC_TS ||
           functionId == TSDB_FUNC_ARITHM) {
         continue;
       }
@@ -4986,11 +4987,29 @@ int32_t validateSqlFunctionInStreamSql(SSqlCmd* pCmd) {
 
 int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd) {
   bool        isProjectionFunction = false;
-  const char* msg = "column projection is not compatible with interval";
+  const char* msg1 = "column projection is not compatible with interval";
+  const char* msg2 = "interval not allowed for tag queries";
   
   // multi-output set/ todo refactor
   for (int32_t k = 0; k < pCmd->fieldsInfo.numOfOutputCols; ++k) {
     SSqlExpr* pExpr = tscSqlExprGet(pCmd, k);
+    
+    // projection query on primary timestamp, the selectivity function needs to be present.
+    if (pExpr->functionId == TSDB_FUNC_PRJ && pExpr->colInfo.colId == PRIMARYKEY_TIMESTAMP_COL_INDEX) {
+      bool hasSelectivity = false;
+      for(int32_t j = 0; j < pCmd->fieldsInfo.numOfOutputCols; ++j) {
+        SSqlExpr* pEx = tscSqlExprGet(pCmd, j);
+        if ((aAggs[pEx->functionId].nStatus & TSDB_FUNCSTATE_SELECTIVITY) == TSDB_FUNCSTATE_SELECTIVITY) {
+          hasSelectivity = true;
+          break;
+        }
+      }
+      
+      if (hasSelectivity) {
+        continue;
+      }
+    }
+    
     if (pExpr->functionId == TSDB_FUNC_PRJ || pExpr->functionId == TSDB_FUNC_DIFF ||
        pExpr->functionId == TSDB_FUNC_ARITHM) {
       isProjectionFunction = true;
@@ -4998,7 +5017,7 @@ int32_t validateFunctionsInIntervalOrGroupbyQuery(SSqlCmd* pCmd) {
   }
 
   if (isProjectionFunction) {
-    setErrMsg(pCmd, msg);
+    setErrMsg(pCmd, msg1);
   }
 
   return isProjectionFunction == true ? TSDB_CODE_INVALID_SQL : TSDB_CODE_SUCCESS;
@@ -5164,8 +5183,7 @@ int32_t parseLimitClause(SSqlObj* pSql, SQuerySQL* pQuerySql) {
 
   if (UTIL_METER_IS_METRIC(pMeterMetaInfo)) {
     bool    queryOnTags = false;
-    int32_t ret = tscQueryOnlyMetricTags(pCmd, &queryOnTags);
-    if (ret != TSDB_CODE_SUCCESS) {
+    if (tscQueryOnlyMetricTags(pCmd, &queryOnTags) != TSDB_CODE_SUCCESS) {
       return TSDB_CODE_INVALID_SQL;
     }
 
@@ -5493,7 +5511,7 @@ static int32_t checkUpdateTagPrjFunctions(SSqlCmd* pCmd) {
 
     // When the tag projection function on tag column that is not in the group by clause, aggregation function and
     // selectivity function exist in select clause is not allowed.
-    if(numOfAggregation > 0) {
+    if (numOfAggregation > 0) {
       setErrMsg(pCmd, msg1);
       return TSDB_CODE_INVALID_SQL;
     }
@@ -5598,14 +5616,14 @@ int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) {
   const char* msg2 = "interval not allowed in group by normal column";
   const char* msg3 = "group by not allowed on projection query";
   const char* msg4 = "tags retrieve not compatible with group by";
-  const char* msg5 = "retrieve tags not compatible with group by ";
+  const char* msg5 = "retrieve tags not compatible with group by or interval query";
 
   SSqlCmd*        pCmd = &pSql->cmd;
   SMeterMetaInfo* pMeterMetaInfo = tscGetMeterMetaInfo(pCmd, 0);
 
   // only retrieve tags, group by is not supportted
   if (pCmd->command == TSDB_SQL_RETRIEVE_TAGS) {
-    if (pCmd->groupbyExpr.numOfGroupCols > 0) {
+    if (pCmd->groupbyExpr.numOfGroupCols > 0 || pCmd->nAggTimeInterval > 0) {
       setErrMsg(pCmd, msg5);
       return TSDB_CODE_INVALID_SQL;
     } else {
@@ -5634,7 +5652,7 @@ int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) {
        * group by normal columns.
        * Check if the column projection is identical to the group by column or not
        */
-      if (functId == TSDB_FUNC_PRJ) {
+      if (functId == TSDB_FUNC_PRJ && pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX) {
         bool qualified = false;
         for (int32_t j = 0; j < pCmd->groupbyExpr.numOfGroupCols; ++j) {
           SColIndexEx* pColIndex = &pCmd->groupbyExpr.columnInfo[j];
@@ -5650,7 +5668,8 @@ int32_t doFunctionsCompatibleCheck(SSqlObj* pSql) {
       }
 
       if (IS_MULTIOUTPUT(aAggs[functId].nStatus) && functId != TSDB_FUNC_TOP && functId != TSDB_FUNC_BOTTOM &&
-          functId != TSDB_FUNC_TAGPRJ) {
+          functId != TSDB_FUNC_TAGPRJ &&
+          (functId == TSDB_FUNC_PRJ && pExpr->colInfo.colId != PRIMARYKEY_TIMESTAMP_COL_INDEX)) {
         setErrMsg(pCmd, msg1);
         return TSDB_CODE_INVALID_SQL;
       }

src/client/src/tscUtil.c

@@ -142,7 +142,6 @@ bool tscIsSelectivityWithTagQuery(SSqlCmd* pCmd) {
   return false;
 }
 
-
 void tscGetDBInfoFromMeterId(char* meterId, char* db) {
   char* st = strstr(meterId, TS_PATH_DELIMITER);
   if (st != NULL) {
@@ -265,7 +264,7 @@ bool tscIsPointInterpQuery(SSqlCmd* pCmd) {
 }
 
 bool tscIsTWAQuery(SSqlCmd* pCmd) {
-  for(int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) {
+  for (int32_t i = 0; i < pCmd->exprsInfo.numOfExprs; ++i) {
     SSqlExpr* pExpr = tscSqlExprGet(pCmd, i);
     if (pExpr == NULL) {
       continue;
@@ -450,7 +449,8 @@ void tscDestroyDataBlock(STableDataBlocks* pDataBlock) {
   tfree(pDataBlock);
 }
 
-SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes, uint32_t offset) {
+SParamInfo* tscAddParamToDataBlock(STableDataBlocks* pDataBlock, char type, uint8_t timePrec, short bytes,
+                                   uint32_t offset) {
   uint32_t needed = pDataBlock->numOfParams + 1;
   if (needed > pDataBlock->numOfAllocedParams) {
     needed *= 2;
@@ -490,13 +490,13 @@ SDataBlockList* tscCreateBlockArrayList() {
   return pDataBlockArrayList;
 }
 
-void tscAppendDataBlock(SDataBlockList *pList, STableDataBlocks *pBlocks) {
+void tscAppendDataBlock(SDataBlockList* pList, STableDataBlocks* pBlocks) {
   if (pList->nSize >= pList->nAlloc) {
     pList->nAlloc = pList->nAlloc << 1;
-    pList->pData = realloc(pList->pData, sizeof(void *) * (size_t)pList->nAlloc);
+    pList->pData = realloc(pList->pData, sizeof(void*) * (size_t)pList->nAlloc);
 
     // reset allocated memory
-    memset(pList->pData + pList->nSize, 0, sizeof(void *) * (pList->nAlloc - pList->nSize));
+    memset(pList->pData + pList->nSize, 0, sizeof(void*) * (pList->nAlloc - pList->nSize));
   }
 
   pList->pData[pList->nSize++] = pBlocks;
@@ -553,7 +553,7 @@ void tscFreeUnusedDataBlocks(SDataBlockList* pList) {
 }
 
 STableDataBlocks* tscCreateDataBlockEx(size_t size, int32_t rowSize, int32_t startOffset, char* name) {
-  STableDataBlocks *dataBuf = tscCreateDataBlock(size);
+  STableDataBlocks* dataBuf = tscCreateDataBlock(size);
 
   dataBuf->rowSize = rowSize;
   dataBuf->size = startOffset;
@@ -573,7 +573,7 @@ STableDataBlocks* tscGetDataBlockFromList(void* pHashList, SDataBlockList* pData
   }
 
   if (dataBuf == NULL) {
-    dataBuf = tscCreateDataBlockEx((size_t) size, rowSize, startOffset, tableId);
+    dataBuf = tscCreateDataBlockEx((size_t)size, rowSize, startOffset, tableId);
     dataBuf = *(STableDataBlocks**)taosAddIntHash(pHashList, id, (char*)&dataBuf);
     tscAppendDataBlock(pDataBlockList, dataBuf);
   }
@@ -869,11 +869,11 @@ void tscClearFieldInfo(SFieldInfo* pFieldInfo) {
 
 static void _exprCheckSpace(SSqlExprInfo* pExprInfo, int32_t size) {
   if (size > pExprInfo->numOfAlloc) {
-    int32_t oldSize = pExprInfo->numOfAlloc;
+    uint32_t oldSize = pExprInfo->numOfAlloc;
 
-    int32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1);
+    uint32_t newSize = (oldSize <= 0) ? 8 : (oldSize << 1U);
     while (newSize < size) {
-      newSize = (newSize << 1);
+      newSize = (newSize << 1U);
     }
 
     if (newSize > TSDB_MAX_COLUMNS) {
@@ -1161,7 +1161,7 @@ void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo) {
   assert(pColumnBaseInfo->numOfCols <= TSDB_MAX_COLUMNS);
 
   for (int32_t i = 0; i < pColumnBaseInfo->numOfCols; ++i) {
-    SColumnBase *pColBase = &(pColumnBaseInfo->pColList[i]);
+    SColumnBase* pColBase = &(pColumnBaseInfo->pColList[i]);
 
     if (pColBase->numOfFilters > 0) {
       for (int32_t j = 0; j < pColBase->numOfFilters; ++j) {
@@ -1179,8 +1179,9 @@ void tscColumnBaseInfoDestroy(SColumnBaseInfo* pColumnBaseInfo) {
   tfree(pColumnBaseInfo->pColList);
 }
 
-
-void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size) { _cf_ensureSpace(pColumnBaseInfo, size); }
+void tscColumnBaseInfoReserve(SColumnBaseInfo* pColumnBaseInfo, int32_t size) {
+  _cf_ensureSpace(pColumnBaseInfo, size);
+}
 
 /*
  * 1. normal name, not a keyword or number
@@ -1228,7 +1229,7 @@ int32_t tscValidateName(SSQLToken* pToken) {
       int len = tSQLGetToken(pToken->z, &pToken->type);
 
       // single token, validate it
-      if (len == pToken->n){
+      if (len == pToken->n) {
         return validateQuoteToken(pToken);
       } else {
         sep = strnchr(pToken->z, TS_PATH_DELIMITER[0], pToken->n, true);
@@ -1652,7 +1653,6 @@ int32_t SStringEnsureRemain(SString* pStr, int32_t size) {
 
   char* tmp = realloc(pStr->z, newsize);
   if (tmp == NULL) {
-
 #ifdef WINDOWS
     LPVOID lpMsgBuf;
     FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER | FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS, NULL,
@@ -1771,8 +1771,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int32_t vnodeIndex, int16_t tableIndex
                                      pMeterMetaInfo->tagColumnIndex);
   } else {
     SMeterMetaInfo* pPrevInfo = tscGetMeterMetaInfo(&pPrevSql->cmd, 0);
-    pFinalInfo = tscAddMeterMetaInfo(&pNew->cmd, name, pPrevInfo->pMeterMeta, pPrevInfo->pMetricMeta, pMeterMetaInfo->numOfTags,
-                        pMeterMetaInfo->tagColumnIndex);
+    pFinalInfo = tscAddMeterMetaInfo(&pNew->cmd, name, pPrevInfo->pMeterMeta, pPrevInfo->pMetricMeta,
+                                     pMeterMetaInfo->numOfTags, pMeterMetaInfo->tagColumnIndex);
 
     pPrevInfo->pMeterMeta = NULL;
     pPrevInfo->pMetricMeta = NULL;
@@ -1783,7 +1783,8 @@ SSqlObj* createSubqueryObj(SSqlObj* pSql, int32_t vnodeIndex, int16_t tableIndex
     assert(pFinalInfo->pMetricMeta != NULL);
   }
 
-  tscTrace("%p new subquery %p, vnodeIdx:%d, tableIndex:%d, type:%d", pSql, pNew, vnodeIndex, tableIndex, pNew->cmd.type);
+  tscTrace("%p new subquery %p, vnodeIdx:%d, tableIndex:%d, type:%d", pSql, pNew, vnodeIndex, tableIndex,
+           pNew->cmd.type);
   return pNew;
 }
 

src/system/detail/src/vnodeQueryImpl.c

@@ -3122,9 +3122,11 @@ static bool onlyOneQueryType(SQuery *pQuery, int32_t functId, int32_t functIdDst
   for (int32_t i = 0; i < pQuery->numOfOutputCols; ++i) {
     int32_t functionId = pQuery->pSelectExpr[i].pBase.functionId;
 
-    if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY || functionId == TSDB_FUNC_TAG) {
+    if (functionId == TSDB_FUNC_TS || functionId == TSDB_FUNC_TS_DUMMY ||
+      functionId == TSDB_FUNC_TAG || functionId == TSDB_FUNC_TAG_DUMMY) {
       continue;
     }
+
     if (functionId != functId && functionId != functIdDst) {
       return false;
     }
@@ -3137,10 +3139,9 @@ static bool onlyFirstQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSD
 
 static bool onlyLastQuery(SQuery *pQuery) { return onlyOneQueryType(pQuery, TSDB_FUNC_LAST, TSDB_FUNC_LAST_DST); }
 
-static void rewriteExecOrder(SQuery *pQuery, bool metricQuery) {
+static void changeExecuteScanOrder(SQuery *pQuery, bool metricQuery) {
   // in case of point-interpolation query, use asc order scan
-  char msg[] =
-      "QInfo:%p scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%lld-%lld, "
+  char msg[] = "QInfo:%p scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%lld-%lld, "
       "new qrange:%lld-%lld";
 
   // descending order query
@@ -3614,7 +3615,7 @@ int32_t vnodeQuerySingleMeterPrepare(SQInfo *pQInfo, SMeterObj *pMeterObj, SMete
   }
 
   setScanLimitationByResultBuffer(pQuery);
-  rewriteExecOrder(pQuery, false);
+  changeExecuteScanOrder(pQuery, false);
 
   pQInfo->over = 0;
   pQInfo->pointsRead = 0;
@@ -3790,7 +3791,7 @@ int32_t vnodeMultiMeterQueryPrepare(SQInfo *pQInfo, SQuery *pQuery, void *param)
   pQInfo->pointsRead = 0;
   pQuery->pointsRead = 0;
 
-  rewriteExecOrder(pQuery, true);
+  changeExecuteScanOrder(pQuery, true);
 
   vnodeInitDataBlockInfo(&pSupporter->runtimeEnv.loadBlockInfo);
   vnodeInitLoadCompBlockInfo(&pSupporter->runtimeEnv.loadCompBlockInfo);

src/system/detail/src/vnodeUtil.c

@@ -289,7 +289,7 @@ SSqlFunctionExpr* vnodeCreateSqlFunctionExpr(SQueryMeterMsg* pQueryMsg, int32_t*
       return NULL;
     }
 
-    if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY) {
+    if (pExprs[i].pBase.functionId == TSDB_FUNC_TAG_DUMMY || pExprs[i].pBase.functionId == TSDB_FUNC_TS_DUMMY) {
       tagLen += pExprs[i].resBytes;
     }
     assert(isValidDataType(pExprs[i].resType, pExprs[i].resBytes));