Merge pull request #10351 from taosdata/feature/TD-10987

Feature/TD-10987 add mode function

src/client/inc/tscUtil.h

@@ -231,15 +231,15 @@ void addExprParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes)
 int32_t   tscGetResRowLength(SArray* pExprList);
 
 SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-    int16_t size, int16_t resColId, int16_t interSize, bool isTagCol);
+    int16_t size, int16_t resColId, int32_t interSize, bool isTagCol);
 
 SExprInfo* tscExprCreate(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-                         int16_t size, int16_t resColId, int16_t interSize, int32_t colType);
+                         int16_t size, int16_t resColId, int32_t interSize, int32_t colType);
 
 void tscExprAddParams(SSqlExpr* pExpr, char* argument, int32_t type, int32_t bytes);
 
 SExprInfo* tscExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-                           int16_t size, int16_t resColId, int16_t interSize, bool isTagCol);
+                           int16_t size, int16_t resColId, int32_t interSize, bool isTagCol);
 
 SExprInfo* tscExprUpdate(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, int16_t srcColumnIndex, int16_t type,
                            int32_t size);

src/client/src/tscGlobalmerge.c

@@ -615,11 +615,9 @@ static void doMergeResultImpl(SOperatorInfo* pInfo, SQLFunctionCtx *pCtx, int32_
       aAggs[functionId].mergeFunc(&pCtx[j]);
     }
 
-    if (functionId == TSDB_FUNC_UNIQUE &&
-        (GET_RES_INFO(&(pCtx[j]))->numOfRes > MAX_UNIQUE_RESULT_ROWS || GET_RES_INFO(&(pCtx[j]))->numOfRes == -1)){
-      tscError("Unique result num is too large. num: %d, limit: %d",
-             GET_RES_INFO(&(pCtx[j]))->numOfRes, MAX_UNIQUE_RESULT_ROWS);
-      longjmp(pInfo->pRuntimeEnv->env, TSDB_CODE_QRY_UNIQUE_RESULT_TOO_LARGE);
+    if (GET_RES_INFO(&(pCtx[j]))->numOfRes == -1){
+      tscError("result num is too large.");
+      longjmp(pInfo->pRuntimeEnv->env, TSDB_CODE_QRY_RESULT_TOO_LARGE);
     }
   }
 }

src/client/src/tscSQLParser.c

@@ -2693,7 +2693,7 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
   const char* msg26 = "start param cannot be 0 with 'log_bin'";
   const char* msg27 = "factor param cannot be negative or equal to 0/1";
   const char* msg28 = "the second paramter of diff should be 0 or 1";
-  const char* msg29 = "key timestamp column cannot be used to unique function";
+  const char* msg29 = "key timestamp column cannot be used to unique/mode function";
 
   switch (functionId) {
     case TSDB_FUNC_COUNT: {
@@ -2791,7 +2791,8 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
     case TSDB_FUNC_CSUM:
     case TSDB_FUNC_STDDEV:
     case TSDB_FUNC_LEASTSQR:
-    case TSDB_FUNC_ELAPSED: {
+    case TSDB_FUNC_ELAPSED:
+    case TSDB_FUNC_MODE: {
       // 1. valid the number of parameters
       int32_t numOfParams =
           (pItem->pNode->Expr.paramList == NULL) ? 0 : (int32_t)taosArrayGetSize(pItem->pNode->Expr.paramList);
@@ -2852,7 +2853,9 @@ int32_t addExprAndResultField(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, int32_t col
       // 2. check if sql function can be applied on this column data type
       SSchema* pSchema = tscGetTableColumnSchema(pTableMetaInfo->pTableMeta, index.columnIndex);
 
-      if (!IS_NUMERIC_TYPE(pSchema->type) && (functionId != TSDB_FUNC_ELAPSED)) {
+      if (functionId == TSDB_FUNC_MODE && pColumnSchema->colId == PRIMARYKEY_TIMESTAMP_COL_INDEX ){
+        return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg29);
+      } else if (!IS_NUMERIC_TYPE(pSchema->type) && (functionId != TSDB_FUNC_ELAPSED) && (functionId != TSDB_FUNC_MODE)) {
         return invalidOperationMsg(tscGetErrorMsgPayload(pCmd), msg1);
       } else if (IS_UNSIGNED_NUMERIC_TYPE(pSchema->type) &&
                  (functionId == TSDB_FUNC_DIFF || functionId == TSDB_FUNC_DERIVATIVE)) {
@@ -4010,7 +4013,8 @@ int32_t tscTansformFuncForSTableQuery(SQueryInfo* pQueryInfo) {
         (functionId == TSDB_FUNC_SAMPLE) ||
         (functionId == TSDB_FUNC_ELAPSED) ||
         (functionId == TSDB_FUNC_HISTOGRAM) ||
-        (functionId == TSDB_FUNC_UNIQUE)) {
+        (functionId == TSDB_FUNC_UNIQUE) ||
+        (functionId == TSDB_FUNC_MODE)) {
       if (getResultDataInfo(pSrcSchema->type, pSrcSchema->bytes, functionId, (int32_t)pExpr->base.param[0].i64, &type, &bytes,
                             &interBytes, 0, true, NULL) != TSDB_CODE_SUCCESS) {
         return TSDB_CODE_TSC_INVALID_OPERATION;

src/client/src/tscUtil.c

@@ -2535,7 +2535,7 @@ void tscFieldInfoCopy(SFieldInfo* pFieldInfo, const SFieldInfo* pSrc, const SArr
 
 
 SExprInfo* tscExprCreate(STableMetaInfo* pTableMetaInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-                         int16_t size, int16_t resColId, int16_t interSize, int32_t colType) {
+                         int16_t size, int16_t resColId, int32_t interSize, int32_t colType) {
   SExprInfo* pExpr = calloc(1, sizeof(SExprInfo));
   if (pExpr == NULL) {
     return NULL;
@@ -2592,7 +2592,7 @@ SExprInfo* tscExprCreate(STableMetaInfo* pTableMetaInfo, int16_t functionId, SCo
 }
 
 SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-                           int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) {
+                           int16_t size, int16_t resColId, int32_t interSize, bool isTagCol) {
   int32_t num = (int32_t)taosArrayGetSize(pQueryInfo->exprList);
   if (index == num) {
     return tscExprAppend(pQueryInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
@@ -2605,7 +2605,7 @@ SExprInfo* tscExprInsert(SQueryInfo* pQueryInfo, int32_t index, int16_t function
 }
 
 SExprInfo* tscExprAppend(SQueryInfo* pQueryInfo, int16_t functionId, SColumnIndex* pColIndex, int16_t type,
-                           int16_t size, int16_t resColId, int16_t interSize, bool isTagCol) {
+                           int16_t size, int16_t resColId, int32_t interSize, bool isTagCol) {
   STableMetaInfo* pTableMetaInfo = tscGetMetaInfo(pQueryInfo, pColIndex->tableIndex);
   SExprInfo* pExpr = tscExprCreate(pTableMetaInfo, functionId, pColIndex, type, size, resColId, interSize, isTagCol);
   taosArrayPush(pQueryInfo->exprList, &pExpr);
@@ -4937,7 +4937,8 @@ static int32_t createGlobalAggregateExpr(SQueryAttr* pQueryAttr, SQueryInfo* pQu
     pse->colInfo.colIndex = i;
 
     pse->colType = pExpr->base.resType;
-    if(pExpr->base.resBytes > INT16_MAX && pExpr->base.functionId == TSDB_FUNC_UNIQUE){
+    if(pExpr->base.resBytes > INT16_MAX &&
+        (pExpr->base.functionId == TSDB_FUNC_UNIQUE || pExpr->base.functionId == TSDB_FUNC_MODE)){
       pQueryAttr->interBytesForGlobal = pExpr->base.resBytes;
     }else{
       pse->colBytes = pExpr->base.resBytes;

src/inc/taoserror.h

@@ -293,7 +293,7 @@ int32_t* taosGetErrno();
 #define TSDB_CODE_QRY_SYS_ERROR                 TAOS_DEF_ERROR_CODE(0, 0x070D)  //"System error")
 #define TSDB_CODE_QRY_INVALID_TIME_CONDITION    TAOS_DEF_ERROR_CODE(0, 0x070E)  //"invalid time condition")
 #define TSDB_CODE_QRY_INVALID_SCHEMA_VERSION    TAOS_DEF_ERROR_CODE(0, 0x0710)  //"invalid schema version")
-#define TSDB_CODE_QRY_UNIQUE_RESULT_TOO_LARGE   TAOS_DEF_ERROR_CODE(0, 0x0711)  //"unique result num is too large")
+#define TSDB_CODE_QRY_RESULT_TOO_LARGE          TAOS_DEF_ERROR_CODE(0, 0x0711)  //"result num is too large")
 
 // grant
 #define TSDB_CODE_GRANT_EXPIRED                 TAOS_DEF_ERROR_CODE(0, 0x0800)  //"License expired"

src/query/inc/qAggMain.h

@@ -79,8 +79,9 @@ extern "C" {
 #define TSDB_FUNC_ELAPSED      37
 #define TSDB_FUNC_HISTOGRAM    38
 #define TSDB_FUNC_UNIQUE       39
+#define TSDB_FUNC_MODE         40
 
-#define TSDB_FUNC_MAX_NUM    40
+#define TSDB_FUNC_MAX_NUM    41
 
 #define TSDB_FUNCSTATE_SO           0x1u    // single output
 #define TSDB_FUNCSTATE_MO           0x2u    // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM
@@ -148,7 +149,7 @@ typedef struct SResultRowCellInfo {
   int8_t   hasResult;       // result generated, not NULL value
   bool     initialized;     // output buffer has been initialized
   bool     complete;        // query has completed
-  uint32_t numOfRes;        // num of output result in current buffer
+  int32_t  numOfRes;        // num of output result in current buffer
 } SResultRowCellInfo;
 
 typedef struct SPoint1 {
@@ -203,6 +204,7 @@ typedef struct SQLFunctionCtx {
   SPoint1      end;
 
   SHashObj     **pUniqueSet;   // for unique function
+  SHashObj     **pModeSet;     // for mode function
 } SQLFunctionCtx;
 
 typedef struct SAggFunctionInfo {

src/query/inc/qExecutor.h

@@ -91,6 +91,7 @@ typedef struct SResultRow {
   STimeWindow   win;
   char         *key;               // start key of current result row
   SHashObj     *uniqueHash;  // for unique function
+  SHashObj     *modeHash;  // for unique function
 } SResultRow;
 
 typedef struct SResultRowCell {

src/query/inc/qResultbuf.h

@@ -80,6 +80,8 @@ typedef struct SDiskbasedResultBuf {
 #define PAGE_INFO_INITIALIZER         (SPageDiskInfo){-1, -1}
 #define MAX_UNIQUE_RESULT_ROWS (1000)
 #define MAX_UNIQUE_RESULT_SIZE (1024*1024*1)
+#define MAX_MODE_INNER_RESULT_ROWS (1000000)
+#define MAX_MODE_INNER_RESULT_SIZE (1024*1024*10)
 /**
  * create disk-based result buffer
  * @param pResultBuf

src/query/src/qExecutor.c

@@ -362,7 +362,7 @@ SSDataBlock* createOutputBuf(SExprInfo* pExpr, int32_t numOfOutput, int32_t numO
       qError("size is too large, failed to allocate column buffer for output buffer");
       tmp = 128*1024*1024;
     }
-    int32_t size = MAX(tmp, minSize);
+    int32_t size = (int32_t)MAX(tmp, minSize);
     idata.pData = calloc(1, size);  // at least to hold a pointer on x64 platform
     if (idata.pData == NULL) {
       qError("failed to allocate column buffer for output buffer");
@@ -1009,11 +1009,9 @@ static void doApplyFunctions(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx* pCtx
       }
     }
 
-    if (functionId == TSDB_FUNC_UNIQUE &&
-        (GET_RES_INFO(&(pCtx[k]))->numOfRes > MAX_UNIQUE_RESULT_ROWS || GET_RES_INFO(&(pCtx[k]))->numOfRes == -1)){
-      qError("Unique result num is too large. num: %d, limit: %d",
-             GET_RES_INFO(&(pCtx[k]))->numOfRes, MAX_UNIQUE_RESULT_ROWS);
-      longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_UNIQUE_RESULT_TOO_LARGE);
+    if (GET_RES_INFO(&(pCtx[k]))->numOfRes == -1){
+      qError("result num is too large.");
+      longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_RESULT_TOO_LARGE);
     }
 
     // restore it
@@ -1276,11 +1274,9 @@ static void doAggregateImpl(SOperatorInfo* pOperator, TSKEY startTs, SQLFunction
         assert(0);
       }
 
-      if (functionId == TSDB_FUNC_UNIQUE &&
-          (GET_RES_INFO(&(pCtx[k]))->numOfRes > MAX_UNIQUE_RESULT_ROWS || GET_RES_INFO(&(pCtx[k]))->numOfRes == -1)){
-        qError("Unique result num is too large. num: %d, limit: %d",
-               GET_RES_INFO(&(pCtx[k]))->numOfRes, MAX_UNIQUE_RESULT_ROWS);
-        longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_UNIQUE_RESULT_TOO_LARGE);
+      if (GET_RES_INFO(&(pCtx[k]))->numOfRes == -1){
+        qError("Mode inner result num is too large");
+        longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_RESULT_TOO_LARGE);
       }
     }
   }
@@ -1965,7 +1961,8 @@ static SQLFunctionCtx* createSQLFunctionCtx(SQueryRuntimeEnv* pRuntimeEnv, SExpr
     }
 
     pCtx->inputType  = pSqlExpr->colType;
-    if (pRuntimeEnv->pQueryAttr->interBytesForGlobal > INT16_MAX && pSqlExpr->functionId == TSDB_FUNC_UNIQUE){
+    if (pRuntimeEnv->pQueryAttr->interBytesForGlobal > INT16_MAX &&
+        (pSqlExpr->functionId == TSDB_FUNC_UNIQUE || pSqlExpr->functionId == TSDB_FUNC_MODE)){
       pCtx->inputBytes  = pRuntimeEnv->pQueryAttr->interBytesForGlobal;
     }else{
       pCtx->inputBytes = pSqlExpr->colBytes;
@@ -3690,6 +3687,8 @@ void setDefaultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SOptrBasicInfo *pInfo, i
     pCtx[i].resultInfo   = pCellInfo;
     if (pCtx[i].functionId == TSDB_FUNC_UNIQUE) {
       pCtx[i].pUniqueSet = &pRow->uniqueHash;
+    }else if (pCtx[i].functionId == TSDB_FUNC_MODE) {
+      pCtx[i].pModeSet = &pRow->modeHash;
     }
     pCtx[i].pOutput      = pData->pData;
     pCtx[i].currentStage = stage;
@@ -4027,6 +4026,8 @@ void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pRe
     pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset);
     if (pCtx[i].functionId == TSDB_FUNC_UNIQUE){
       pCtx[i].pUniqueSet = &pResult->uniqueHash;
+    }else if (pCtx[i].functionId == TSDB_FUNC_MODE){
+      pCtx[i].pModeSet = &pResult->modeHash;
     }
 
     SResultRowCellInfo* pResInfo = pCtx[i].resultInfo;
@@ -4123,6 +4124,8 @@ void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult, SQLF
     pCtx[i].resultInfo = getResultCell(pResult, i, rowCellInfoOffset);
     if (pCtx[i].functionId == TSDB_FUNC_UNIQUE) {
       pCtx[i].pUniqueSet = &pResult->uniqueHash;
+    }else if (pCtx[i].functionId == TSDB_FUNC_MODE) {
+      pCtx[i].pModeSet = &pResult->modeHash;
     }
   }
 }

src/query/src/qUtil.c

@@ -92,6 +92,10 @@ void cleanupResultRowInfo(SResultRowInfo *pResultRowInfo) {
         taosHashCleanup(pResultRowInfo->pResult[i]->uniqueHash);
         pResultRowInfo->pResult[i]->uniqueHash = NULL;
       }
+      if (pResultRowInfo->pResult[i]->modeHash){
+        taosHashCleanup(pResultRowInfo->pResult[i]->modeHash);
+        pResultRowInfo->pResult[i]->modeHash = NULL;
+      }
     }
   }
   
@@ -205,7 +209,7 @@ SResultRowPool* initResultRowPool(size_t size) {
     qError("ResultRow blockSize is too large:%" PRId64, tmp);
     tmp = 128*1024*1024;
   }
-  p->blockSize = tmp;
+  p->blockSize = (int32_t)tmp;
   p->position.pos = 0;
 
   p->pData = taosArrayInit(8, POINTER_BYTES);

src/util/src/terror.c

@@ -299,7 +299,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_QRY_NOT_ENOUGH_BUFFER,        "Query buffer limit ha
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INCONSISTAN,              "File inconsistance in replica")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_TIME_CONDITION,   "One valid time range condition expected")
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_SYS_ERROR,                "System error")
-TAOS_DEFINE_ERROR(TSDB_CODE_QRY_UNIQUE_RESULT_TOO_LARGE,  "Unique result num is too large")
+TAOS_DEFINE_ERROR(TSDB_CODE_QRY_RESULT_TOO_LARGE,         "result num is too large")
 
 // grant
 TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_EXPIRED,                "License expired")