diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index b18bd882ae49d97cf0658d7463528fadbb949eb7..55204045ba574da9da9167ca32c622d076b6b6d3 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -2441,7 +2441,7 @@ _exit: int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t bytes, int32_t nRows, char *lengthOrbitmap, char *data) { int32_t code = 0; - if(data == NULL){ + if (data == NULL) { for (int32_t i = 0; i < nRows; ++i) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NONE](pColData, NULL, 0); } @@ -2455,8 +2455,9 @@ int32_t tColDataAddValueByDataBlock(SColData *pColData, int8_t type, int32_t byt code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); if (code) goto _exit; } else { - if(ASSERT(varDataTLen(data + offset) <= bytes)){ - uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), bytes); + if (ASSERT(varDataTLen(data + offset) <= bytes)) { + uError("var data length invalid, varDataTLen(data + offset):%d <= bytes:%d", (int)varDataTLen(data + offset), + bytes); code = TSDB_CODE_INVALID_PARA; goto _exit; } @@ -2508,7 +2509,7 @@ int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { ASSERT(pColData->type == pBind->buffer_type); } - + if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { @@ -3521,6 +3522,43 @@ static FORCE_INLINE void tColDataCalcSMAUBigInt(SColData *pColData, int64_t *sum } } +static FORCE_INLINE void tColDataCalcSMAVarType(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, + int16_t *numOfNull) { + *(uint64_t *)sum = 0; + *(uint64_t *)max = 0; + *(uint64_t *)min = 0; + *numOfNull = 0; + + switch (pColData->flag) { + case HAS_NONE: + case HAS_NULL: + case (HAS_NONE | HAS_NULL): + *numOfNull = pColData->nVal; + break; + case HAS_VALUE: + *numOfNull = 0; + break; + case (HAS_VALUE | HAS_NULL): + case (HAS_VALUE | HAS_NONE): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT1(pColData->pBitMap, iVal) == 0) { + (*numOfNull)++; + } + } + break; + case (HAS_VALUE | HAS_NONE | HAS_NULL): + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + if (GET_BIT2(pColData->pBitMap, iVal) != 2) { + (*numOfNull)++; + } + } + break; + default: + ASSERT(0); + break; + } +} + void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_t *min, int16_t *numOfNull) = { NULL, tColDataCalcSMABool, // TSDB_DATA_TYPE_BOOL @@ -3530,14 +3568,14 @@ void (*tColDataCalcSMA[])(SColData *pColData, int64_t *sum, int64_t *max, int64_ tColDataCalcSMABigInt, // TSDB_DATA_TYPE_BIGINT tColDataCalcSMAFloat, // TSDB_DATA_TYPE_FLOAT tColDataCalcSMADouble, // TSDB_DATA_TYPE_DOUBLE - NULL, // TSDB_DATA_TYPE_VARCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_VARCHAR tColDataCalcSMABigInt, // TSDB_DATA_TYPE_TIMESTAMP - NULL, // TSDB_DATA_TYPE_NCHAR + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_NCHAR tColDataCalcSMAUTinyInt, // TSDB_DATA_TYPE_UTINYINT tColDataCalcSMATinyUSmallInt, // TSDB_DATA_TYPE_USMALLINT tColDataCalcSMAUInt, // TSDB_DATA_TYPE_UINT tColDataCalcSMAUBigInt, // TSDB_DATA_TYPE_UBIGINT - NULL, // TSDB_DATA_TYPE_JSON + tColDataCalcSMAVarType, // TSDB_DATA_TYPE_JSON NULL, // TSDB_DATA_TYPE_VARBINARY NULL, // TSDB_DATA_TYPE_DECIMAL NULL, // TSDB_DATA_TYPE_BLOB diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index cb83a561d24aed55c4fde3f824139d8d4dbd26e6..5a7bebebff6a85c2468d9eaa5187015211235d3e 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -184,7 +184,7 @@ int32_t tsdbReaderOpen(SVnode *pVnode, SQueryTableDataCond *pCond, void *pT SSDataBlock *pResBlock, STsdbReader **ppReader, const char *idstr, bool countOnly, SHashObj** pIgnoreTables); void tsdbReaderClose(STsdbReader *pReader); int32_t tsdbNextDataBlock(STsdbReader *pReader, bool *hasNext); -int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave); +int32_t tsdbRetrieveDatablockSMA(STsdbReader *pReader, SSDataBlock *pDataBlock, bool *allHave, bool *hasNullSMA); void tsdbReleaseDataBlock(STsdbReader *pReader); SSDataBlock *tsdbRetrieveDataBlock(STsdbReader *pTsdbReadHandle, SArray *pColumnIdList); int32_t tsdbReaderReset(STsdbReader *pReader, SQueryTableDataCond *pCond); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 4acc784aee7dfba25edf578c92a2acafd724e2ff..5d970ce6c3b30cb55770253856398dcda1ec1e51 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -4973,7 +4973,8 @@ int32_t tsdbNextDataBlock(STsdbReader* pReader, bool* hasNext) { return code; } -static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { +static bool doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_t numOfCols, SColumnDataAgg* pTsAgg) { + bool hasNullSMA = false; // do fill all null column value SMA info int32_t i = 0, j = 0; int32_t size = (int32_t)taosArrayGetSize(pSup->pColAgg); @@ -4993,6 +4994,7 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; size++; + hasNullSMA = true; } j += 1; } @@ -5003,12 +5005,15 @@ static void doFillNullColSMA(SBlockLoadSuppInfo* pSup, int32_t numOfRows, int32_ SColumnDataAgg nullColAgg = {.colId = pSup->colId[j], .numOfNull = numOfRows}; taosArrayInsert(pSup->pColAgg, i, &nullColAgg); i += 1; + hasNullSMA = true; } j++; } + + return hasNullSMA; } -int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave) { +int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, bool* allHave, bool *hasNullSMA) { SColumnDataAgg*** pBlockSMA = &pDataBlock->pBlockAgg; int32_t code = 0; @@ -5072,7 +5077,10 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SSDataBlock* pDataBlock, } // do fill all null column value SMA info - doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg); + if (doFillNullColSMA(pSup, pBlock->nRow, numOfCols, pTsAgg)) { + *hasNullSMA = true; + return TSDB_CODE_SUCCESS; + } size_t size = taosArrayGetSize(pSup->pColAgg); int32_t i = 0, j = 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index 50fd9d7aa7035adbfe8d17fde078b301483fe015..b25ab393da5af4c3656306afd58cab522cbdad23 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -523,7 +523,7 @@ static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, for (int32_t iColData = 0; iColData < pBlockData->nColData; iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type) || ((pColData->flag & HAS_VALUE) == 0)) continue; + if ((!pColData->smaOn) || ((pColData->flag & HAS_VALUE) == 0)) continue; SColumnDataAgg sma = {.colId = pColData->cid}; tColDataCalcSMA[pColData->type](pColData, &sma.sum, &sma.max, &sma.min, &sma.numOfNull); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index a001d99408a82d8e180d778409397658d0fbc8a0..c1430c4ce0ac541eef8ddc38a1a29106537ee2d9 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -228,12 +228,13 @@ static bool doFilterByBlockSMA(SFilterInfo* pFilterInfo, SColumnDataAgg** pColsA static bool doLoadBlockSMA(STableScanBase* pTableScanInfo, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { bool allColumnsHaveAgg = true; - int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg); + bool hasNullSMA = false; + int32_t code = tsdbRetrieveDatablockSMA(pTableScanInfo->dataReader, pBlock, &allColumnsHaveAgg, &hasNullSMA); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } - if (!allColumnsHaveAgg) { + if (!allColumnsHaveAgg || hasNullSMA) { return false; } return true; diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 4c019b3e710961065e0f4439f2f4f4baeb9de990..cdecc8b8ccdc13eb8a2eea4f161989e2effa1ebc 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -500,7 +500,7 @@ static int64_t getNumOfElems(SqlFunctionCtx* pCtx) { */ SInputColumnInfoData* pInput = &pCtx->input; SColumnInfoData* pInputCol = pInput->pData[0]; - if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows && !IS_VAR_DATA_TYPE(pInputCol->info.type)) { + if (pInput->colDataSMAIsSet && pInput->totalRows == pInput->numOfRows) { numOfElem = pInput->numOfRows - pInput->pColumnDataAgg[0]->numOfNull; } else { if (pInputCol->hasNull) { diff --git a/tests/system-test/2-query/count_null.py b/tests/system-test/2-query/count_null.py new file mode 100644 index 0000000000000000000000000000000000000000..6d2c8db8d6b4bd33ae75213ab838279ff9e65503 --- /dev/null +++ b/tests/system-test/2-query/count_null.py @@ -0,0 +1,144 @@ +import taos +import sys + +from util.log import * +from util.sql import * +from util.cases import * + + + +class TDTestCase: + + def init(self, conn, logSql, replicaVar=1): + self.replicaVar = int(replicaVar) + tdLog.debug(f"start to excute {__file__}") + #tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def check_results(self): + tdSql.query(f"select count(*) from tb1") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c2) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c3) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c4) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c5) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c6) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c7) from tb1") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb1") + tdSql.checkData(0, 0, 0) + + tdSql.query(f"select count(*) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c2) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c3) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c4) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c5) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c6) from tb2") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c7) from tb2") + tdSql.checkData(0, 0, 0) + tdSql.query(f"select count(c8) from tb2") + tdSql.checkData(0, 0, 0) + + for i in range (3, 6): + tdSql.query(f"select count(*) from tb{i}") + tdSql.checkData(0, 0, 20000) + tdSql.query(f"select count(c1) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c2) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c3) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c4) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c5) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c6) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c7) from tb{i}") + tdSql.checkData(0, 0, 10000) + tdSql.query(f"select count(c8) from tb{i}") + tdSql.checkData(0, 0, 10000) + + + def run(self): + dbname = 'db' + tbnames = ['tb1', 'tb2', 'tb3', 'tb4', 'tb5', 'tb6'] + num_rows = 20000 + num_tables = 6 + ts_base = 1685548800000 + + tdSql.prepare() + + tdLog.printNoPrefix("==========step1:create table") + + for i in range (num_tables): + tdSql.execute( + f'''create table if not exists {dbname}.{tbnames[i]} + (ts timestamp, c0 tinyint, c1 smallint, c2 int, c3 bigint, c4 double, c5 float, c6 bool, c7 varchar(10), c8 nchar(10)) + + ''' + ) + + + tdLog.printNoPrefix("==========step2:insert data") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[0]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + + for i in range(num_rows): + tdSql.execute(f"insert into {dbname}.{tbnames[1]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[2]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i % 2 == 0: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, null, null, null, null, null, null, null, 'binary', 'nchar')") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[3]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, null, null)") + + for i in range(num_rows): + if i < num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[4]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + for i in range(num_rows): + if i >= num_rows / 2: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, null, null, null, null, null, null, null, null, null)") + else: + tdSql.execute(f"insert into {dbname}.{tbnames[5]} values ({ts_base + i}, 1, 1, 1, 1, 1, 1, 1, 'binary', 'nchar')") + + + tdLog.printNoPrefix("==========step3:check result in memory") + self.check_results(); + + tdLog.printNoPrefix("==========step3:check result from disk") + tdSql.execute(f"flush database db") + self.check_results(); + + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase())