Merge pull request #21455 from taosdata/enh/add_varchar_sma

fix: fix count var type error

source/common/src/tdataformat.c

@@ -2441,7 +2441,7 @@ _exit:
 int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap,
                                     char *data) {
   int32_t code = 0;
-  if(data == NULL){
+  if (data == NULL) {
     for (int32_t i = 0; i < nRows; ++i) {
       code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0);
     }
@@ -2455,8 +2455,9 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt
         code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0);
         if (code) goto _exit;
       } else {
-        if(ASSERT(varDataTLen(data + offset) <= bytes)){
-          uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes);
+        if (ASSERT(varDataTLen(data + offset) <= bytes)) {
+          uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset),
+                 bytes);
           code = TSDB_CODE_INVALID_PARA;
           goto _exit;
         }
@@ -2508,7 +2509,7 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) {
   if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) {
     ASSERT(pColData->type == pBind->buffer_type);
   }
-  
+
   if (IS_VAR_DATA_TYPE(pColData->type)) {  // var-length data type
     for (int32_t i = 0; i < pBind->num; ++i) {
       if (pBind->is_null && pBind->is_null[i]) {
@@ -3521,6 +3522,43 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum
   }
 }
 
+static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min,
+                                                int16_t *numOfNull) {
+  *(uint64_t *)sum = 0;
+  *(uint64_t *)max = 0;
+  *(uint64_t *)min = 0;
+  *numOfNull = 0;
+
+  switch (pColData->flag) {
+    case HAS_NONE:
+    case HAS_NULL:
+    case (HAS_NONE | HAS_NULL):
+      *numOfNull = pColData->nVal;
+      break;
+    case HAS_VALUE:
+      *numOfNull = 0;
+      break;
+    case (HAS_VALUE | HAS_NULL):
+    case (HAS_VALUE | HAS_NONE):
+      for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+        if (GET_BIT1(pColData->pBitMap, iVal) == 0) {
+          (*numOfNull)++;
+        }
+      }
+      break;
+    case (HAS_VALUE | HAS_NONE | HAS_NULL):
+      for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+        if (GET_BIT2(pColData->pBitMap, iVal) != 2) {
+          (*numOfNull)++;
+        }
+      }
+      break;
+    default:
+      ASSERT(0);
+      break;
+  }
+}
+
 void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = {
     NULL,
     tColDataCalcSMABool,           // TSDB_DATA_TYPE_BOOL
@@ -3530,14 +3568,14 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_
     tColDataCalcSMABigInt,         // TSDB_DATA_TYPE_BIGINT
     tColDataCalcSMAFloat,          // TSDB_DATA_TYPE_FLOAT
     tColDataCalcSMADouble,         // TSDB_DATA_TYPE_DOUBLE
-    NULL,                          // TSDB_DATA_TYPE_VARCHAR
+    tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_VARCHAR
     tColDataCalcSMABigInt,         // TSDB_DATA_TYPE_TIMESTAMP
-    NULL,                          // TSDB_DATA_TYPE_NCHAR
+    tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_NCHAR
     tColDataCalcSMAUTinyInt,       // TSDB_DATA_TYPE_UTINYINT
     tColDataCalcSMATinyUSmallInt,  // TSDB_DATA_TYPE_USMALLINT
     tColDataCalcSMAUInt,           // TSDB_DATA_TYPE_UINT
     tColDataCalcSMAUBigInt,        // TSDB_DATA_TYPE_UBIGINT
-    NULL,                          // TSDB_DATA_TYPE_JSON
+    tColDataCalcSMAVarType,        // TSDB_DATA_TYPE_JSON
     NULL,                          // TSDB_DATA_TYPE_VARBINARY
     NULL,                          // TSDB_DATA_TYPE_DECIMAL
     NULL,                          // TSDB_DATA_TYPE_BLOB

source/dnode/vnode/inc/vnode.h

@@ -184,7 +184,7 @@ int32_t      tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pT
                             SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables);
 void         tsdbReaderClose(STsdbReader *pReader);
 int32_t      tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext);
-int32_t      tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave);
+int32_t      tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA);
 void         tsdbReleaseDataBlock(STsdbReader *pReader);
 SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList);
 int32_t      tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond);

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -4973,7 +4973,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) {
   return code;
 }
 
-static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) {
+static bool doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) {
+  bool hasNullSMA = false;
   // do fill all null column value SMA info
   int32_t i = 0, j = 0;
   int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg);
@@ -4993,6 +4994,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_
         taosArrayInsert(pSup->pColAgg, i, &nullColAgg);
         i += 1;
         size++;
+        hasNullSMA = true;
       }
       j += 1;
     }
@@ -5003,12 +5005,15 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_
       SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows};
       taosArrayInsert(pSup->pColAgg, i, &nullColAgg);
       i += 1;
+      hasNullSMA = true;
     }
     j++;
   }
+
+  return hasNullSMA;
 }
 
-int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) {
+int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool *hasNullSMA) {
   SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg;
 
   int32_t code = 0;
@@ -5072,7 +5077,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock,
   }
 
   // do fill all null column value SMA info
-  doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg);
+  if (doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg)) {
+    *hasNullSMA = true;
+    return TSDB_CODE_SUCCESS;
+  }
   size_t size = taosArrayGetSize(pSup->pColAgg);
 
   int32_t i = 0, j = 0;

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -523,7 +523,7 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData,
   for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) {
     SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
 
-    if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type) || ((pColData->flag & HAS_VALUE) == 0)) continue;
+    if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue;
 
     SColumnDataAgg sma = {.colId = pColData->cid};
     tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull);

source/libs/executor/src/scanoperator.c

@@ -228,12 +228,13 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA
 
 static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) {
   bool    allColumnsHaveAgg = true;
-  int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg);
+  bool    hasNullSMA = false;
+  int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg, &hasNullSMA);
   if (code != TSDB_CODE_SUCCESS) {
     T_LONG_JMP(pTaskInfo->env, code);
   }
 
-  if (!allColumnsHaveAgg) {
+  if (!allColumnsHaveAgg || hasNullSMA) {
     return false;
   }
   return true;

source/libs/function/src/builtinsimpl.c

@@ -500,7 +500,7 @@ static int64_t getNumOfElems(SqlFunctionCtx* pCtx) {
    */
   SInputColumnInfoData* pInput = &pCtx->input;
   SColumnInfoData*      pInputCol = pInput->pData[0];
-  if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) {
+  if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) {
     numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull;
   } else {
     if (pInputCol->hasNull) {

tests/system-test/2-query/count_null.py

+import taos
+import sys
+
+from util.log import *
+from util.sql import *
+from util.cases import *
+
+
+
+class TDTestCase:
+
+    def init(self, conn, logSql, replicaVar=1):
+        self.replicaVar = int(replicaVar)
+        tdLog.debug(f"start to excute {__file__}")
+        #tdSql.init(conn.cursor())
+        tdSql.init(conn.cursor(), logSql)  # output sql.txt file
+
+    def check_results(self):
+        tdSql.query(f"select count(*) from tb1")
+        tdSql.checkData(0, 0, 20000)
+        tdSql.query(f"select count(c1) from tb1")
+        tdSql.checkData(0, 0, 0)
+        tdSql.query(f"select count(c2) from tb1")
+        tdSql.checkData(0, 0, 0)
+        tdSql.query(f"select count(c3) from tb1")
+        tdSql.checkData(0, 0, 0)
+        tdSql.query(f"select count(c4) from tb1")
+        tdSql.checkData(0, 0, 0)
+        tdSql.query(f"select count(c5) from tb1")
+        tdSql.checkData(0, 0, 0)
+        tdSql.query(f"select count(c6) from tb1")
+        tdSql.checkData(0, 0, 0)
+        tdSql.query(f"select count(c7) from tb1")
+        tdSql.checkData(0, 0, 0)
+        tdSql.query(f"select count(c8) from tb1")
+        tdSql.checkData(0, 0, 0)
+
+        tdSql.query(f"select count(*) from tb2")
+        tdSql.checkData(0, 0, 20000)
+        tdSql.query(f"select count(c1) from tb2")
+        tdSql.checkData(0, 0, 20000)
+        tdSql.query(f"select count(c2) from tb2")
+        tdSql.checkData(0, 0, 20000)
+        tdSql.query(f"select count(c3) from tb2")
+        tdSql.checkData(0, 0, 20000)
+        tdSql.query(f"select count(c4) from tb2")
+        tdSql.checkData(0, 0, 20000)
+        tdSql.query(f"select count(c5) from tb2")
+        tdSql.checkData(0, 0, 20000)
+        tdSql.query(f"select count(c6) from tb2")
+        tdSql.checkData(0, 0, 20000)
+        tdSql.query(f"select count(c7) from tb2")
+        tdSql.checkData(0, 0, 0)
+        tdSql.query(f"select count(c8) from tb2")
+        tdSql.checkData(0, 0, 0)
+
+        for i in range (3, 6):
+            tdSql.query(f"select count(*) from tb{i}")
+            tdSql.checkData(0, 0, 20000)
+            tdSql.query(f"select count(c1) from tb{i}")
+            tdSql.checkData(0, 0, 10000)
+            tdSql.query(f"select count(c2) from tb{i}")
+            tdSql.checkData(0, 0, 10000)
+            tdSql.query(f"select count(c3) from tb{i}")
+            tdSql.checkData(0, 0, 10000)
+            tdSql.query(f"select count(c4) from tb{i}")
+            tdSql.checkData(0, 0, 10000)
+            tdSql.query(f"select count(c5) from tb{i}")
+            tdSql.checkData(0, 0, 10000)
+            tdSql.query(f"select count(c6) from tb{i}")
+            tdSql.checkData(0, 0, 10000)
+            tdSql.query(f"select count(c7) from tb{i}")
+            tdSql.checkData(0, 0, 10000)
+            tdSql.query(f"select count(c8) from tb{i}")
+            tdSql.checkData(0, 0, 10000)
+
+
+    def run(self):
+        dbname = 'db'
+        tbnames = ['tb1', 'tb2', 'tb3', 'tb4', 'tb5', 'tb6']
+        num_rows = 20000
+        num_tables = 6
+        ts_base = 1685548800000
+
+        tdSql.prepare()
+
+        tdLog.printNoPrefix("==========step1:create table")
+
+        for i in range (num_tables):
+            tdSql.execute(
+                f'''create table if not exists {dbname}.{tbnames[i]}
+                (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10))
+
+                '''
+            )
+
+
+        tdLog.printNoPrefix("==========step2:insert data")
+
+        for i in range(num_rows):
+            tdSql.execute(f"insert into {dbname}.{tbnames[0]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)")
+
+        for i in range(num_rows):
+            tdSql.execute(f"insert into {dbname}.{tbnames[1]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)")
+
+        for i in range(num_rows):
+            if i % 2 == 0:
+                tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)")
+            else:
+                tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')")
+
+        for i in range(num_rows):
+            if i % 2 == 0:
+                tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, null, null, null, null, null, null, null, 'binary', 'nchar')")
+            else:
+                tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)")
+
+        for i in range(num_rows):
+            if i < num_rows / 2:
+                tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)")
+            else:
+                tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')")
+
+        for i in range(num_rows):
+            if i >= num_rows / 2:
+                tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)")
+            else:
+                tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')")
+
+
+        tdLog.printNoPrefix("==========step3:check result in memory")
+        self.check_results();
+
+        tdLog.printNoPrefix("==========step3:check result from disk")
+        tdSql.execute(f"flush database db")
+        self.check_results();
+
+
+    def stop(self):
+        tdSql.close()
+        tdLog.success(f"{__file__} successfully executed")
+
+tdCases.addLinux(__file__, TDTestCase())
+tdCases.addWindows(__file__, TDTestCase())