Merge branch 'feature/3.0_liaohj' of github.com:taosdata/tdengine into feature/3.0_liaohj

include/common/trow.h

@@ -222,8 +222,10 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, T
 static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT valType);
 static FORCE_INLINE int32_t tdSetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT valType, int8_t bitmapMode);
 int32_t                     tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType, int8_t bitmapMode);
-static FORCE_INLINE int32_t tdGetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT *pValType);
-static FORCE_INLINE int32_t tdGetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT *pValType);
+static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType);
+static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType);
+static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode);
+static FORCE_INLINE bool    tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx);
 int32_t tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int32_t numOfRows, int32_t maxPoints,
                              int8_t bitmapMode);
 static FORCE_INLINE int32_t tdAppendColValToTpRow(SRowBuilder *pBuilder, TDRowValT valType, const void *val,
@@ -325,7 +327,16 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeII(void *pBitmap, int16_t colIdx,
   return TSDB_CODE_SUCCESS;
 }
 
-static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
+static FORCE_INLINE bool tdIsBitmapValTypeNormII(const void *pBitmap, int16_t idx) {
+  TDRowValT valType = 0;
+  tdGetBitmapValTypeII(pBitmap, idx, &valType);
+  if (tdValTypeIsNorm(valType)) {
+    return true;
+  }
+  return false;
+}
+
+static FORCE_INLINE int32_t tdGetBitmapValType(const void *pBitmap, int16_t colIdx, TDRowValT *pValType, int8_t bitmapMode) {
   switch (bitmapMode) {
     case 0:
       tdGetBitmapValTypeII(pBitmap, colIdx, pValType);
@@ -350,7 +361,7 @@ static FORCE_INLINE int32_t tdGetBitmapValType(void *pBitmap, int16_t colIdx, TD
  * @param pValType
  * @return FORCE_INLINE
  */
-static FORCE_INLINE int32_t tdGetBitmapValTypeII(void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
+static FORCE_INLINE int32_t tdGetBitmapValTypeII(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
   if (!pBitmap || colIdx < 0) {
     TASSERT(0);
     terrno = TSDB_CODE_INVALID_PARA;
@@ -449,7 +460,7 @@ static FORCE_INLINE int32_t tdSetBitmapValTypeI(void *pBitmap, int16_t colIdx, T
  * @param pValType
  * @return FORCE_INLINE
  */
-static FORCE_INLINE int32_t tdGetBitmapValTypeI(void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
+static FORCE_INLINE int32_t tdGetBitmapValTypeI(const void *pBitmap, int16_t colIdx, TDRowValT *pValType) {
   if (!pBitmap || colIdx < 0) {
     TASSERT(0);
     terrno = TSDB_CODE_INVALID_PARA;

include/common/ttypes.h

@@ -248,7 +248,7 @@ typedef struct tDataTypeDescriptor {
                       int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
   int32_t (*decompFunc)(const char *const input, int32_t compressedSize, const int32_t nelements, char *const output,
                         int32_t outputSize, char algorithm, char *const buffer, int32_t bufferSize);
-  void (*statisFunc)(const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex,
+  void (*statisFunc)(const void* pBitmap, const void *pData, int32_t numofrow, int64_t *min, int64_t *max, int64_t *sum, int16_t *minindex,
                      int16_t *maxindex, int16_t *numofnull);
 } tDataTypeDescriptor;
 

source/common/src/ttypes.c

@@ -16,6 +16,7 @@
 #define _DEFAULT_SOURCE
 #include "ttypes.h"
 #include "tcompression.h"
+#include "trow.h"
 
 const int32_t TYPE_BYTES[15] = {
     -1,                      // TSDB_DATA_TYPE_NULL
@@ -49,7 +50,7 @@ const int32_t TYPE_BYTES[15] = {
     }                                                                          \
   } while (0)
 
-static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_bool(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                             int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int8_t *data = (int8_t *)pData;
   *min = INT64_MAX;
@@ -60,7 +61,8 @@ static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, i
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (data[i] == TSDB_DATA_BOOL_NULL) {
+    // if (data[i] == TSDB_DATA_BOOL_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -69,7 +71,7 @@ static void getStatics_bool(const void *pData, int32_t numOfRow, int64_t *min, i
   }
 }
 
-static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_i8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int8_t *data = (int8_t *)pData;
   *min = INT64_MAX;
@@ -80,7 +82,8 @@ static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) {
+    // if (((uint8_t)data[i]) == TSDB_DATA_TINYINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -89,7 +92,7 @@ static void getStatics_i8(const void *pData, int32_t numOfRow, int64_t *min, int
   }
 }
 
-static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_u8(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                           int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint8_t *data = (uint8_t *)pData;
   uint64_t _min = UINT64_MAX;
@@ -102,7 +105,8 @@ static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) {
+    // if (((uint8_t)data[i]) == TSDB_DATA_UTINYINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -115,7 +119,7 @@ static void getStatics_u8(const void *pData, int32_t numOfRow, int64_t *min, int
   *sum = _sum;
 }
 
-static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_i16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int16_t *data = (int16_t *)pData;
   *min = INT64_MAX;
@@ -126,7 +130,8 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) {
+    // if (((uint16_t)data[i]) == TSDB_DATA_SMALLINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -135,7 +140,7 @@ static void getStatics_i16(const void *pData, int32_t numOfRow, int64_t *min, in
   }
 }
 
-static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_u16(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint16_t *data = (uint16_t *)pData;
   uint64_t  _min = UINT64_MAX;
@@ -148,7 +153,8 @@ static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) {
+    // if (((uint16_t)data[i]) == TSDB_DATA_USMALLINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -161,7 +167,7 @@ static void getStatics_u16(const void *pData, int32_t numOfRow, int64_t *min, in
   *sum = _sum;
 }
 
-static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_i32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int32_t *data = (int32_t *)pData;
   *min = INT64_MAX;
@@ -172,7 +178,8 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) {
+    // if (((uint32_t)data[i]) == TSDB_DATA_INT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -181,7 +188,7 @@ static void getStatics_i32(const void *pData, int32_t numOfRow, int64_t *min, in
   }
 }
 
-static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_u32(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint32_t *data = (uint32_t *)pData;
   uint64_t  _min = UINT64_MAX;
@@ -194,7 +201,8 @@ static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) {
+    // if (((uint32_t)data[i]) == TSDB_DATA_UINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -207,7 +215,7 @@ static void getStatics_u32(const void *pData, int32_t numOfRow, int64_t *min, in
   *sum = _sum;
 }
 
-static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_i64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   int64_t *data = (int64_t *)pData;
   *min = INT64_MAX;
@@ -218,7 +226,8 @@ static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) {
+    // if (((uint64_t)data[i]) == TSDB_DATA_BIGINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -227,7 +236,7 @@ static void getStatics_i64(const void *pData, int32_t numOfRow, int64_t *min, in
   }
 }
 
-static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_u64(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   uint64_t *data = (uint64_t *)pData;
   uint64_t  _min = UINT64_MAX;
@@ -240,7 +249,8 @@ static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, in
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) {
+    // if (((uint64_t)data[i]) == TSDB_DATA_UBIGINT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -253,7 +263,7 @@ static void getStatics_u64(const void *pData, int32_t numOfRow, int64_t *min, in
   *sum = _sum;
 }
 
-static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_f(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                          int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   float *data = (float *)pData;
   float  fmin = FLT_MAX;
@@ -265,7 +275,8 @@ static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int6
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) {
+    // if ((*(uint32_t *)&(data[i])) == TSDB_DATA_FLOAT_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -289,7 +300,7 @@ static void getStatics_f(const void *pData, int32_t numOfRow, int64_t *min, int6
   SET_DOUBLE_VAL(min, fmin);
 }
 
-static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_d(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                          int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   double *data = (double *)pData;
   double  dmin = DBL_MAX;
@@ -301,7 +312,8 @@ static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int6
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) {
+    // if ((*(uint64_t *)&(data[i])) == TSDB_DATA_DOUBLE_NULL) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
       continue;
     }
@@ -325,13 +337,14 @@ static void getStatics_d(const void *pData, int32_t numOfRow, int64_t *min, int6
   SET_DOUBLE_PTR(min, &dmin);
 }
 
-static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_bin(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                            int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   const char *data = pData;
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull(data, TSDB_DATA_TYPE_BINARY)) {
+    // if (isNull(data, TSDB_DATA_TYPE_BINARY)) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
     }
 
@@ -345,13 +358,14 @@ static void getStatics_bin(const void *pData, int32_t numOfRow, int64_t *min, in
   *maxIndex = 0;
 }
 
-static void getStatics_nchr(const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
+static void getStatics_nchr(const void *pBitmap, const void *pData, int32_t numOfRow, int64_t *min, int64_t *max, int64_t *sum,
                             int16_t *minIndex, int16_t *maxIndex, int16_t *numOfNull) {
   const char *data = pData;
   assert(numOfRow <= INT16_MAX);
 
   for (int32_t i = 0; i < numOfRow; ++i) {
-    if (isNull(data, TSDB_DATA_TYPE_NCHAR)) {
+    // if (isNull(data, TSDB_DATA_TYPE_NCHAR)) {
+    if (!tdIsBitmapValTypeNormII(pBitmap, i)) {
       (*numOfNull) += 1;
     }
 

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -1250,7 +1250,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDFile *pDF
                                                &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex),
                                                &(pBlockCol->numOfNull));
 #endif
-      (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max),
+      (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pBitmap, pDataCol->pData, rowsToWrite, &(pAggrBlkCol->min), &(pAggrBlkCol->max),
                                                &(pAggrBlkCol->sum), &(pAggrBlkCol->minIndex), &(pAggrBlkCol->maxIndex),
                                                &(pAggrBlkCol->numOfNull));