diff --git a/include/common/trow.h b/include/common/trow.h index 282921087c949ac7c405d6123a14177819996b4f..9a09bd3c8f8041d01b7dceb50598a5648632f5a9 100644 --- a/include/common/trow.h +++ b/include/common/trow.h @@ -222,8 +222,10 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, T static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType); static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode); int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode); -static FORCE_INLINE int32_t tdGetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT *pValType); -static FORCE_INLINE int32_t tdGetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT *pValType); +static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); +static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType); +static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode); +static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx); int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints, int8_t bitmapMode); static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val, @@ -325,7 +327,16 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, return TSDB_CODE_SUCCESS; } -static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { +static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx) { + TDRowValT valType = 0; + tdGetBitmapValTypeII(pBitmap, idx, &valType); + if (tdValTypeIsNorm(valType)) { + return true; + } + return false; +} + +static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) { switch (bitmapMode) { case 0: tdGetBitmapValTypeII(pBitmap, colIdx, pValType); @@ -350,7 +361,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD * @param pValType * @return FORCE_INLINE */ -static FORCE_INLINE int32_t tdGetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT *pValType) { +static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { TASSERT(0); terrno = TSDB_CODE_INVALID_PARA; @@ -449,7 +460,7 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, T * @param pValType * @return FORCE_INLINE */ -static FORCE_INLINE int32_t tdGetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT *pValType) { +static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) { if (!pBitmap || colIdx < 0) { TASSERT(0); terrno = TSDB_CODE_INVALID_PARA; diff --git a/include/common/ttypes.h b/include/common/ttypes.h index 23b260849dc382ffaca7466b782d21698dd8b93d..1a09368bd342800c09902b1fc39785ee7dfbbc23 100644 --- a/include/common/ttypes.h +++ b/include/common/ttypes.h @@ -248,7 +248,7 @@ typedef struct tDataTypeDescriptor { int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output, int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize); - void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex, + void (*statisFunc)(const void* pBitmap, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex, int16_t *maxindex, int16_t *numofnull); } tDataTypeDescriptor; diff --git a/source/common/src/ttypes.c b/source/common/src/ttypes.c index cb8746c963574cec0c9d5db4e6d8d190fd7ac4bd..d4371e11f724a7cbc36d955fc38bc1d6e5207caa 100644 --- a/source/common/src/ttypes.c +++ b/source/common/src/ttypes.c @@ -16,6 +16,7 @@ #define _DEFAULT_SOURCE #include "ttypes.h" #include "tcompression.h" +#include "trow.h" const int32_t TYPE_BYTES[15] = { -1, // TSDB_DATA_TYPE_NULL @@ -49,7 +50,7 @@ const int32_t TYPE_BYTES[15] = { } \ } while (0) -static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int8_t *data = (int8_t *)pData; *min = INT64_MAX; @@ -60,7 +61,8 @@ static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, i assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (data[i] == TSDB_DATA_BOOL_NULL) { + // if (data[i] == TSDB_DATA_BOOL_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -69,7 +71,7 @@ static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, i } } -static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int8_t *data = (int8_t *)pData; *min = INT64_MAX; @@ -80,7 +82,8 @@ static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) { + // if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -89,7 +92,7 @@ static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int } } -static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint8_t *data = (uint8_t *)pData; uint64_t _min = UINT64_MAX; @@ -102,7 +105,8 @@ static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) { + // if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -115,7 +119,7 @@ static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int *sum = _sum; } -static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int16_t *data = (int16_t *)pData; *min = INT64_MAX; @@ -126,7 +130,8 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) { + // if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -135,7 +140,7 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in } } -static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint16_t *data = (uint16_t *)pData; uint64_t _min = UINT64_MAX; @@ -148,7 +153,8 @@ static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, in assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) { + // if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -161,7 +167,7 @@ static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, in *sum = _sum; } -static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int32_t *data = (int32_t *)pData; *min = INT64_MAX; @@ -172,7 +178,8 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) { + // if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -181,7 +188,7 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in } } -static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint32_t *data = (uint32_t *)pData; uint64_t _min = UINT64_MAX; @@ -194,7 +201,8 @@ static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, in assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) { + // if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -207,7 +215,7 @@ static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, in *sum = _sum; } -static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { int64_t *data = (int64_t *)pData; *min = INT64_MAX; @@ -218,7 +226,8 @@ static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, in assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) { + // if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -227,7 +236,7 @@ static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, in } } -static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { uint64_t *data = (uint64_t *)pData; uint64_t _min = UINT64_MAX; @@ -240,7 +249,8 @@ static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, in assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) { + // if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -253,7 +263,7 @@ static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, in *sum = _sum; } -static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { float *data = (float *)pData; float fmin = FLT_MAX; @@ -265,7 +275,8 @@ static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int6 assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) { + // if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -289,7 +300,7 @@ static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int6 SET_DOUBLE_VAL(min, fmin); } -static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { double *data = (double *)pData; double dmin = DBL_MAX; @@ -301,7 +312,8 @@ static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int6 assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { + // if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; continue; } @@ -325,13 +337,14 @@ static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int6 SET_DOUBLE_PTR(min, &dmin); } -static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { const char *data = pData; assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(data, TSDB_DATA_TYPE_BINARY)) { + // if (isNull(data, TSDB_DATA_TYPE_BINARY)) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; } @@ -345,13 +358,14 @@ static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, in *maxIndex = 0; } -static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, +static void getStatics_nchr(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) { const char *data = pData; assert(numOfRow <= INT16_MAX); for (int32_t i = 0; i < numOfRow; ++i) { - if (isNull(data, TSDB_DATA_TYPE_NCHAR)) { + // if (isNull(data, TSDB_DATA_TYPE_NCHAR)) { + if (!tdIsBitmapValTypeNormII(pBitmap, i)) { (*numOfNull) += 1; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index a1edb7cd9c32148ccfda3ccef7597b4f20544304..28fb2a65488f9ce101966138771e28b041447b03 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -1250,7 +1250,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex), &(pBlockCol->numOfNull)); #endif - (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max), + (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pBitmap, pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max), &(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex), &(pAggrBlkCol->numOfNull));