Merge pull request #16847 from taosdata/refact/new_data_format

refact: new data format

include/common/tdataformat.h

@@ -36,8 +36,13 @@ typedef struct STSRow2       STSRow2;
 typedef struct STSRowBuilder STSRowBuilder;
 typedef struct STagVal       STagVal;
 typedef struct STag          STag;
+typedef struct SColData      SColData;
 
-// bitmap
+#define HAS_NONE  ((uint8_t)0x1)
+#define HAS_NULL  ((uint8_t)0x2)
+#define HAS_VALUE ((uint8_t)0x4)
+
+// bitmap ================================
 const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0},
                                        {0b00000000, 0b00000100, 0b00001000, 2},
                                        {0b00000000, 0b00010000, 0b00100000, 4},
@@ -51,21 +56,21 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0},
 #define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)])
 #define GET_BIT2(p, i)    (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3))
 
-// STSchema
+// STSchema ================================
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
 void    tTSchemaDestroy(STSchema *pTSchema);
 
-// SValue
+// SValue ================================
 int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type);
 int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type);
 int     tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type);
 
-// SColVal
+// SColVal ================================
 #define COL_VAL_NONE(CID, TYPE)     ((SColVal){.cid = (CID), .type = (TYPE), .isNone = 1})
 #define COL_VAL_NULL(CID, TYPE)     ((SColVal){.cid = (CID), .type = (TYPE), .isNull = 1})
 #define COL_VAL_VALUE(CID, TYPE, V) ((SColVal){.cid = (CID), .type = (TYPE), .value = (V)})
 
-// STSRow2
+// STSRow2 ================================
 #define TSROW_LEN(PROW, V)  tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL)
 #define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL)
 
@@ -77,7 +82,7 @@ int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray);
 int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow);
 int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow);
 
-// STSRowBuilder
+// STSRowBuilder ================================
 #define tsRowBuilderInit() ((STSRowBuilder){0})
 #define tsRowBuilderClear(B)     \
   do {                           \
@@ -86,7 +91,7 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow);
     }                            \
   } while (0)
 
-// STag
+// STag ================================
 int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag);
 void    tTagFree(STag *pTag);
 bool    tTagIsJson(const void *pTag);
@@ -100,7 +105,16 @@ void    tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid);
 void    debugPrintSTag(STag *pTag, const char *tag, int32_t ln);  // TODO: remove
 int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf);
 
-// STRUCT =================
+// SColData ================================
+void    tColDataDestroy(void *ph);
+void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
+void    tColDataClear(SColData *pColData);
+int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
+void    tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal);
+uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal);
+int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
+
+// STRUCT ================================
 struct STColumn {
   col_id_t colId;
   int8_t   type;
@@ -166,6 +180,18 @@ struct SColVal {
   SValue  value;
 };
 
+struct SColData {
+  int16_t  cid;
+  int8_t   type;
+  int8_t   smaOn;
+  int32_t  nVal;
+  uint8_t  flag;
+  uint8_t *pBitMap;
+  int32_t *aOffset;
+  int32_t  nData;
+  uint8_t *pData;
+};
+
 #pragma pack(push, 1)
 struct STagVal {
   //  char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta

source/common/src/tdataformat.c

@@ -15,6 +15,7 @@
 
 #define _DEFAULT_SOURCE
 #include "tdataformat.h"
+#include "tRealloc.h"
 #include "tcoding.h"
 #include "tdatablock.h"
 #include "tlog.h"
@@ -680,7 +681,7 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) {
   return n;
 }
 
-// STSchema
+// STSchema ========================================
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) {
   *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols);
   if (*ppTSchema == NULL) {
@@ -720,9 +721,7 @@ void tTSchemaDestroy(STSchema *pTSchema) {
   if (pTSchema) taosMemoryFree(pTSchema);
 }
 
-// STSRowBuilder
-
-// STag
+// STag ========================================
 static int tTagValCmprFn(const void *p1, const void *p2) {
   if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) {
     return -1;
@@ -1172,4 +1171,495 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
   return pSchema;
 }
 
-#endif
+#endif
\ No newline at end of file
+
+// SColData ========================================
+void tColDataDestroy(void *ph) {
+  SColData *pColData = (SColData *)ph;
+
+  tFree(pColData->pBitMap);
+  tFree((uint8_t *)pColData->aOffset);
+  tFree(pColData->pData);
+}
+
+void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
+  pColData->cid = cid;
+  pColData->type = type;
+  pColData->smaOn = smaOn;
+  tColDataClear(pColData);
+}
+
+void tColDataClear(SColData *pColData) {
+  pColData->nVal = 0;
+  pColData->flag = 0;
+  pColData->nData = 0;
+}
+
+static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVal) {
+  int32_t code = 0;
+
+  if (IS_VAR_DATA_TYPE(pColData->type)) {
+    code = tRealloc((uint8_t **)(&pColData->aOffset), sizeof(int32_t) * (pColData->nVal + 1));
+    if (code) goto _exit;
+    pColData->aOffset[pColData->nVal] = pColData->nData;
+
+    if (pColVal->value.nData) {
+      code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
+      if (code) goto _exit;
+      memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
+      pColData->nData += pColVal->value.nData;
+    }
+  } else {
+    ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal);
+    code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes);
+    if (code) goto _exit;
+    pColData->nData += tPutValue(pColData->pData + pColData->nData, &pColVal->value, pColVal->type);
+  }
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) {  // 0
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+    pColData->flag = HAS_NONE;
+  } else if (pColVal->isNull) {
+    pColData->flag = HAS_NULL;
+  } else {
+    pColData->flag = HAS_VALUE;
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) {  // HAS_NONE
+  int32_t code = 0;
+
+  if (!pColVal->isNone) {
+    int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+
+    code = tRealloc(&pColData->pBitMap, nBit);
+    if (code) goto _exit;
+
+    memset(pColData->pBitMap, 0, nBit);
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+    if (pColVal->isNull) {
+      pColData->flag |= HAS_NULL;
+    } else {
+      pColData->flag |= HAS_VALUE;
+
+      if (pColData->nVal) {
+        if (IS_VAR_DATA_TYPE(pColData->type)) {
+          int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+          code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+          if (code) goto _exit;
+          memset(pColData->aOffset, 0, nOffset);
+        } else {
+          pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+          code = tRealloc(&pColData->pData, pColData->nData);
+          if (code) goto _exit;
+          memset(pColData->pData, 0, pColData->nData);
+        }
+      }
+
+      code = tColDataPutValue(pColData, pColVal);
+      if (code) goto _exit;
+    }
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) {  // HAS_NULL
+  int32_t code = 0;
+
+  if (!pColVal->isNull) {
+    int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+    code = tRealloc(&pColData->pBitMap, nBit);
+    if (code) goto _exit;
+
+    if (pColVal->isNone) {
+      pColData->flag |= HAS_NONE;
+
+      memset(pColData->pBitMap, 255, nBit);
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+    } else {
+      pColData->flag |= HAS_VALUE;
+
+      memset(pColData->pBitMap, 0, nBit);
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+
+      if (pColData->nVal) {
+        if (IS_VAR_DATA_TYPE(pColData->type)) {
+          int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+          code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+          if (code) goto _exit;
+          memset(pColData->aOffset, 0, nOffset);
+        } else {
+          pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+          code = tRealloc(&pColData->pData, pColData->nData);
+          if (code) goto _exit;
+          memset(pColData->pData, 0, pColData->nData);
+        }
+      }
+
+      code = tColDataPutValue(pColData, pColVal);
+      if (code) goto _exit;
+    }
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) {  // HAS_NULL|HAS_NONE
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+  } else if (pColVal->isNull) {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+  } else {
+    pColData->flag |= HAS_VALUE;
+
+    uint8_t *pBitMap = NULL;
+    code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+      SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal));
+    }
+    SET_BIT2(pBitMap, pColData->nVal, 2);
+
+    tFree(pColData->pBitMap);
+    pColData->pBitMap = pBitMap;
+
+    if (pColData->nVal) {
+      if (IS_VAR_DATA_TYPE(pColData->type)) {
+        int32_t nOffset = sizeof(int32_t) * pColData->nVal;
+        code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset);
+        if (code) goto _exit;
+        memset(pColData->aOffset, 0, nOffset);
+      } else {
+        pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal;
+        code = tRealloc(&pColData->pData, pColData->nData);
+        if (code) goto _exit;
+        memset(pColData->pData, 0, pColData->nData);
+      }
+    }
+
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE
+  int32_t code = 0;
+
+  if (pColVal->isNone || pColVal->isNull) {
+    if (pColVal->isNone) {
+      pColData->flag |= HAS_NONE;
+    } else {
+      pColData->flag |= HAS_NULL;
+    }
+
+    int32_t nBit = BIT1_SIZE(pColData->nVal + 1);
+    code = tRealloc(&pColData->pBitMap, nBit);
+    if (code) goto _exit;
+
+    memset(pColData->pBitMap, 255, nBit);
+    SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
+  } else {
+    code = tColDataPutValue(pColData, pColVal);
+    if (code) goto _exit;
+  }
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE|HAS_NONE
+  int32_t code = 0;
+
+  if (pColVal->isNull) {
+    pColData->flag |= HAS_NULL;
+
+    uint8_t *pBitMap = NULL;
+    code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+      SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0);
+    }
+    SET_BIT2(pBitMap, pColData->nVal, 1);
+
+    tFree(pColData->pBitMap);
+    pColData->pBitMap = pBitMap;
+  } else {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    if (pColVal->isNone) {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+    } else {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+    }
+  }
+  code = tColDataPutValue(pColData, pColVal);
+  if (code) goto _exit;
+
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) {  // HAS_VALUE|HAS_NULL
+  int32_t code = 0;
+
+  if (pColVal->isNone) {
+    pColData->flag |= HAS_NONE;
+
+    uint8_t *pBitMap = NULL;
+    code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
+      SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1);
+    }
+    SET_BIT2(pBitMap, pColData->nVal, 0);
+
+    tFree(pColData->pBitMap);
+    pColData->pBitMap = pBitMap;
+  } else {
+    code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1));
+    if (code) goto _exit;
+
+    if (pColVal->isNull) {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 0);
+    } else {
+      SET_BIT1(pColData->pBitMap, pColData->nVal, 1);
+    }
+  }
+  code = tColDataPutValue(pColData, pColVal);
+  if (code) goto _exit;
+
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData,
+                                                 SColVal  *pColVal) {  // HAS_VALUE|HAS_NULL|HAS_NONE
+  int32_t code = 0;
+
+  code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1));
+  if (code) goto _exit;
+
+  if (pColVal->isNone) {
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
+  } else if (pColVal->isNull) {
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
+  } else {
+    SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
+  }
+  code = tColDataPutValue(pColData, pColVal);
+  if (code) goto _exit;
+
+  pColData->nVal++;
+
+_exit:
+  return code;
+}
+static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = {
+    tColDataAppendValue0,  // 0
+    tColDataAppendValue1,  // HAS_NONE
+    tColDataAppendValue2,  // HAS_NULL
+    tColDataAppendValue3,  // HAS_NULL|HAS_NONE
+    tColDataAppendValue4,  // HAS_VALUE
+    tColDataAppendValue5,  // HAS_VALUE|HAS_NONE
+    tColDataAppendValue6,  // HAS_VALUE|HAS_NULL
+    tColDataAppendValue7   // HAS_VALUE|HAS_NULL|HAS_NONE
+};
+int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
+  ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type);
+  return tColDataAppendValueImpl[pColData->flag](pColData, pColVal);
+}
+
+static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_NONE
+  *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+}
+static FORCE_INLINE void tColDataGetValue2(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_NULL
+  *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+}
+static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_NULL|HAS_NONE
+  switch (GET_BIT1(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+      break;
+    case 1:
+      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SColVal *pColVal) {  // HAS_VALUE
+  SValue value;
+  if (IS_VAR_DATA_TYPE(pColData->type)) {
+    if (iVal + 1 < pColData->nVal) {
+      value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal];
+    } else {
+      value.nData = pColData->nData - pColData->aOffset[iVal];
+    }
+    value.pData = pColData->pData + pColData->aOffset[iVal];
+  } else {
+    tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type);
+  }
+  *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value);
+}
+static FORCE_INLINE void tColDataGetValue5(SColData *pColData, int32_t iVal,
+                                           SColVal *pColVal) {  // HAS_VALUE|HAS_NONE
+  switch (GET_BIT1(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+      break;
+    case 1:
+      tColDataGetValue4(pColData, iVal, pColVal);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static FORCE_INLINE void tColDataGetValue6(SColData *pColData, int32_t iVal,
+                                           SColVal *pColVal) {  // HAS_VALUE|HAS_NULL
+  switch (GET_BIT1(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+      break;
+    case 1:
+      tColDataGetValue4(pColData, iVal, pColVal);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static FORCE_INLINE void tColDataGetValue7(SColData *pColData, int32_t iVal,
+                                           SColVal *pColVal) {  // HAS_VALUE|HAS_NULL|HAS_NONE
+  switch (GET_BIT2(pColData->pBitMap, iVal)) {
+    case 0:
+      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
+      break;
+    case 1:
+      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
+      break;
+    case 2:
+      tColDataGetValue4(pColData, iVal, pColVal);
+      break;
+    default:
+      ASSERT(0);
+  }
+}
+static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal *pColVal) = {
+    NULL,               // 0
+    tColDataGetValue1,  // HAS_NONE
+    tColDataGetValue2,  // HAS_NULL
+    tColDataGetValue3,  // HAS_NULL | HAS_NONE
+    tColDataGetValue4,  // HAS_VALUE
+    tColDataGetValue5,  // HAS_VALUE | HAS_NONE
+    tColDataGetValue6,  // HAS_VALUE | HAS_NULL
+    tColDataGetValue7   // HAS_VALUE | HAS_NULL | HAS_NONE
+};
+void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
+  ASSERT(iVal >= 0 && iVal < pColData->nVal && pColData->flag);
+  tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal);
+}
+
+uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal) {
+  uint8_t v;
+  switch (pColData->flag) {
+    case HAS_NONE:
+      v = 0;
+      break;
+    case HAS_NULL:
+      v = 1;
+      break;
+    case (HAS_NULL | HAS_NONE):
+      v = GET_BIT1(pColData->pBitMap, iVal);
+      break;
+    case HAS_VALUE:
+      v = 2;
+      break;
+    case (HAS_VALUE | HAS_NONE):
+      v = GET_BIT1(pColData->pBitMap, iVal);
+      if (v) v = 2;
+      break;
+    case (HAS_VALUE | HAS_NULL):
+      v = GET_BIT1(pColData->pBitMap, iVal) + 1;
+      break;
+    case (HAS_VALUE | HAS_NULL | HAS_NONE):
+      v = GET_BIT2(pColData->pBitMap, iVal);
+      break;
+    default:
+      ASSERT(0);
+      break;
+  }
+  return v;
+}
+
+int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
+  int32_t code = 0;
+  int32_t size;
+
+  ASSERT(pColDataSrc->nVal > 0);
+  ASSERT(pColDataDest->cid = pColDataSrc->cid);
+  ASSERT(pColDataDest->type = pColDataSrc->type);
+
+  pColDataDest->smaOn = pColDataSrc->smaOn;
+  pColDataDest->nVal = pColDataSrc->nVal;
+  pColDataDest->flag = pColDataSrc->flag;
+
+  // bitmap
+  if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
+    size = BIT2_SIZE(pColDataSrc->nVal);
+    code = tRealloc(&pColDataDest->pBitMap, size);
+    if (code) goto _exit;
+    memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
+  }
+
+  // offset
+  if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
+    size = sizeof(int32_t) * pColDataSrc->nVal;
+
+    code = tRealloc((uint8_t **)&pColDataDest->aOffset, size);
+    if (code) goto _exit;
+
+    memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size);
+  }
+
+  // value
+  pColDataDest->nData = pColDataSrc->nData;
+  code = tRealloc(&pColDataDest->pData, pColDataSrc->nData);
+  if (code) goto _exit;
+  memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData);
+
+_exit:
+  return code;
+}
\ No newline at end of file

source/dnode/vnode/src/inc/tsdb.h

@@ -44,7 +44,6 @@ typedef struct SMapData      SMapData;
 typedef struct SBlockIdx     SBlockIdx;
 typedef struct SDataBlk      SDataBlk;
 typedef struct SSttBlk       SSttBlk;
-typedef struct SColData      SColData;
 typedef struct SDiskDataHdr  SDiskDataHdr;
 typedef struct SBlockData    SBlockData;
 typedef struct SDelFile      SDelFile;
@@ -71,10 +70,6 @@ typedef struct SLDataIter    SLDataIter;
 #define TSDB_MAX_SUBBLOCKS 8
 #define TSDB_FHDR_SIZE     512
 
-#define HAS_NONE  ((int8_t)0x1)
-#define HAS_NULL  ((int8_t)0x2)
-#define HAS_VALUE ((int8_t)0x4)
-
 #define VERSION_MIN 0
 #define VERSION_MAX INT64_MAX
 
@@ -148,15 +143,6 @@ int32_t tPutBlockIdx(uint8_t *p, void *ph);
 int32_t tGetBlockIdx(uint8_t *p, void *ph);
 int32_t tCmprBlockIdx(void const *lhs, void const *rhs);
 int32_t tCmprBlockL(void const *lhs, void const *rhs);
-// SColdata
-void    tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn);
-void    tColDataReset(SColData *pColData);
-void    tColDataClear(void *ph);
-int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal);
-int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal);
-int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest);
-int32_t tPutColData(uint8_t *p, SColData *pColData);
-int32_t tGetColData(uint8_t *p, SColData *pColData);
 // SBlockData
 #define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0)
 #define tBlockDataLastRow(PBLOCKDATA)  tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1)
@@ -470,18 +456,6 @@ struct SSttBlk {
   SBlockInfo bInfo;
 };
 
-struct SColData {
-  int16_t  cid;
-  int8_t   type;
-  int8_t   smaOn;
-  int32_t  nVal;
-  uint8_t  flag;
-  uint8_t *pBitMap;
-  int32_t *aOffset;
-  int32_t  nData;
-  uint8_t *pData;
-};
-
 // (SBlockData){.suid = 0, .uid = 0}: block data not initialized
 // (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file
 // (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -903,7 +903,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn
           // null value exists, check one-by-one
           if (pData->flag != HAS_VALUE) {
             for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step, rowIndex++) {
-              uint8_t v = GET_BIT2(pData->pBitMap, j);
+              uint8_t v = tColDataGetBitValue(pData, j);
               if (v == 0 || v == 1) {
                 colDataSetNull_f(pColData->nullbitmap, rowIndex);
               }
@@ -4078,4 +4078,4 @@ void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap) {
   }
 
   tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode));
-}
+}
\ No newline at end of file

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -909,248 +909,6 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
   return code;
 }
 
-// SColData ========================================
-void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
-  pColData->cid = cid;
-  pColData->type = type;
-  pColData->smaOn = smaOn;
-  tColDataReset(pColData);
-}
-
-void tColDataReset(SColData *pColData) {
-  pColData->nVal = 0;
-  pColData->flag = 0;
-  pColData->nData = 0;
-}
-
-void tColDataClear(void *ph) {
-  SColData *pColData = (SColData *)ph;
-
-  tFree(pColData->pBitMap);
-  tFree((uint8_t *)pColData->aOffset);
-  tFree(pColData->pData);
-}
-
-int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
-  int32_t code = 0;
-  int64_t size;
-  SValue  value = {0};
-  SValue *pValue = &value;
-
-  ASSERT(pColVal->cid == pColData->cid);
-  ASSERT(pColVal->type == pColData->type);
-
-  // realloc bitmap
-  size = BIT2_SIZE(pColData->nVal + 1);
-  code = tRealloc(&pColData->pBitMap, size);
-  if (code) goto _exit;
-  if ((pColData->nVal & 3) == 0) {
-    pColData->pBitMap[pColData->nVal >> 2] = 0;
-  }
-
-  // put value
-  if (pColVal->isNone) {
-    pColData->flag |= HAS_NONE;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
-  } else if (pColVal->isNull) {
-    pColData->flag |= HAS_NULL;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
-  } else {
-    pColData->flag |= HAS_VALUE;
-    SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
-    pValue = &pColVal->value;
-  }
-
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    // offset
-    code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1));
-    if (code) goto _exit;
-    pColData->aOffset[pColData->nVal] = pColData->nData;
-
-    // value
-    if ((!pColVal->isNone) && (!pColVal->isNull)) {
-      code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
-      if (code) goto _exit;
-      memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
-      pColData->nData += pColVal->value.nData;
-    }
-  } else {
-    code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type));
-    if (code) goto _exit;
-    pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type);
-  }
-
-  pColData->nVal++;
-
-_exit:
-  return code;
-}
-
-int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
-  int32_t code = 0;
-  int32_t size;
-
-  ASSERT(pColDataSrc->nVal > 0);
-  ASSERT(pColDataDest->cid = pColDataSrc->cid);
-  ASSERT(pColDataDest->type = pColDataSrc->type);
-
-  pColDataDest->smaOn = pColDataSrc->smaOn;
-  pColDataDest->nVal = pColDataSrc->nVal;
-  pColDataDest->flag = pColDataSrc->flag;
-
-  // bitmap
-  if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
-    size = BIT2_SIZE(pColDataSrc->nVal);
-    code = tRealloc(&pColDataDest->pBitMap, size);
-    if (code) goto _exit;
-    memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
-  }
-
-  // offset
-  if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
-    size = sizeof(int32_t) * pColDataSrc->nVal;
-
-    code = tRealloc((uint8_t **)&pColDataDest->aOffset, size);
-    if (code) goto _exit;
-
-    memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size);
-  }
-
-  // value
-  pColDataDest->nData = pColDataSrc->nData;
-  code = tRealloc(&pColDataDest->pData, pColDataSrc->nData);
-  if (code) goto _exit;
-  memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData);
-
-_exit:
-  return code;
-}
-
-int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
-  int32_t code = 0;
-
-  ASSERT(iVal < pColData->nVal);
-  ASSERT(pColData->flag);
-
-  if (pColData->flag == HAS_NONE) {
-    *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
-    goto _exit;
-  } else if (pColData->flag == HAS_NULL) {
-    *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
-    goto _exit;
-  } else if (pColData->flag != HAS_VALUE) {
-    uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
-    if (v == 0) {
-      *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
-      goto _exit;
-    } else if (v == 1) {
-      *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
-      goto _exit;
-    }
-  }
-
-  // get value
-  SValue value;
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    if (iVal + 1 < pColData->nVal) {
-      value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal];
-    } else {
-      value.nData = pColData->nData - pColData->aOffset[iVal];
-    }
-
-    value.pData = pColData->pData + pColData->aOffset[iVal];
-  } else {
-    tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type);
-  }
-  *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value);
-
-_exit:
-  return code;
-}
-
-int32_t tPutColData(uint8_t *p, SColData *pColData) {
-  int32_t n = 0;
-
-  n += tPutI16v(p ? p + n : p, pColData->cid);
-  n += tPutI8(p ? p + n : p, pColData->type);
-  n += tPutI8(p ? p + n : p, pColData->smaOn);
-  n += tPutI32v(p ? p + n : p, pColData->nVal);
-  n += tPutU8(p ? p + n : p, pColData->flag);
-
-  if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
-  if (pColData->flag != HAS_VALUE) {
-    // bitmap
-
-    int32_t size = BIT2_SIZE(pColData->nVal);
-    if (p) {
-      memcpy(p + n, pColData->pBitMap, size);
-    }
-    n += size;
-  }
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    // offset
-
-    int32_t size = sizeof(int32_t) * pColData->nVal;
-    if (p) {
-      memcpy(p + n, pColData->aOffset, size);
-    }
-    n += size;
-  }
-  n += tPutI32v(p ? p + n : p, pColData->nData);
-  if (p) {
-    memcpy(p + n, pColData->pData, pColData->nData);
-  }
-  n += pColData->nData;
-
-_exit:
-  return n;
-}
-
-int32_t tGetColData(uint8_t *p, SColData *pColData) {
-  int32_t n = 0;
-
-  n += tGetI16v(p + n, &pColData->cid);
-  n += tGetI8(p + n, &pColData->type);
-  n += tGetI8(p + n, &pColData->smaOn);
-  n += tGetI32v(p + n, &pColData->nVal);
-  n += tGetU8(p + n, &pColData->flag);
-
-  if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
-  if (pColData->flag != HAS_VALUE) {
-    // bitmap
-
-    int32_t size = BIT2_SIZE(pColData->nVal);
-    pColData->pBitMap = p + n;
-    n += size;
-  }
-  if (IS_VAR_DATA_TYPE(pColData->type)) {
-    // offset
-
-    int32_t size = sizeof(int32_t) * pColData->nVal;
-    pColData->aOffset = (int32_t *)(p + n);
-    n += size;
-  }
-  n += tGetI32v(p + n, &pColData->nData);
-  pColData->pData = p + n;
-  n += pColData->nData;
-
-_exit:
-  return n;
-}
-
-static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) {
-  SColData *pColData1 = (SColData *)p1;
-  SColData *pColData2 = (SColData *)p2;
-
-  if (pColData1->cid < pColData2->cid) {
-    return -1;
-  } else if (pColData1->cid > pColData2->cid) {
-    return 1;
-  }
-
-  return 0;
-}
-
 // SBlockData ======================================================
 int32_t tBlockDataCreate(SBlockData *pBlockData) {
   int32_t code = 0;
@@ -1182,7 +940,7 @@ void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) {
   tFree((uint8_t *)pBlockData->aVersion);
   tFree((uint8_t *)pBlockData->aTSKEY);
   taosArrayDestroy(pBlockData->aIdx);
-  taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL);
+  taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL);
   pBlockData->aUid = NULL;
   pBlockData->aVersion = NULL;
   pBlockData->aTSKEY = NULL;
@@ -1251,7 +1009,7 @@ void tBlockDataClear(SBlockData *pBlockData) {
   pBlockData->nRow = 0;
   for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
     SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
-    tColDataReset(pColData);
+    tColDataClear(pColData);
   }
 }
 
@@ -1501,7 +1259,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD
   while (lidx <= ridx) {
     int32_t   midx = (lidx + ridx) / 2;
     SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx);
-    int32_t   c = tColDataCmprFn(pColData, &(SColData){.cid = cid});
+    int32_t   c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1);
 
     if (c == 0) {
       *ppColData = pColData;
@@ -1986,47 +1744,16 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
   int32_t size = 0;
   // bitmap
   if (pColData->flag != HAS_VALUE) {
-    uint8_t *pBitMap = pColData->pBitMap;
-    int32_t  szBitMap = BIT2_SIZE(pColData->nVal);
-
-    // BIT2 to BIT1
-    if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+    int32_t szBitMap;
+    if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+      szBitMap = BIT2_SIZE(pColData->nVal);
+    } else {
       szBitMap = BIT1_SIZE(pColData->nVal);
-      pBitMap = taosMemoryCalloc(1, szBitMap);
-      if (pBitMap == NULL) {
-        code = TSDB_CODE_OUT_OF_MEMORY;
-        goto _exit;
-      }
-
-      for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
-        uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
-        switch (pColData->flag) {
-          case (HAS_NULL | HAS_NONE):
-            SET_BIT1(pBitMap, iVal, v);
-            break;
-          case (HAS_VALUE | HAS_NONE):
-            if (v) {
-              SET_BIT1(pBitMap, iVal, 1);
-            } else {
-              SET_BIT1(pBitMap, iVal, 0);
-            }
-            break;
-          case (HAS_VALUE | HAS_NULL):
-            SET_BIT1(pBitMap, iVal, v - 1);
-            break;
-          default:
-            ASSERT(0);
-        }
-      }
     }
 
-    code = tsdbCmprData(pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, &pBlockCol->szBitmap,
-                        ppBuf);
+    code = tsdbCmprData(pColData->pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size,
+                        &pBlockCol->szBitmap, ppBuf);
     if (code) goto _exit;
-
-    if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
-      taosMemoryFree(pBitMap);
-    }
   }
   size += pBlockCol->szBitmap;
 
@@ -2064,46 +1791,15 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
   uint8_t *p = pIn;
   // bitmap
   if (pBlockCol->szBitmap) {
-    if (pBlockCol->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
-      uint8_t *pBitMap = NULL;
-      code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pBitMap,
-                            BIT1_SIZE(pColData->nVal), ppBuf);
-      if (code) goto _exit;
-
-      code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal));
-      if (code) {
-        tFree(pBitMap);
-        goto _exit;
-      }
-
-      // BIT1 to BIT2
-      for (int32_t iVal = 0; iVal < nVal; iVal++) {
-        uint8_t v = GET_BIT1(pBitMap, iVal);
-        switch (pBlockCol->flag) {
-          case (HAS_NULL | HAS_NONE):
-            SET_BIT2(pColData->pBitMap, iVal, v);
-            break;
-          case (HAS_VALUE | HAS_NONE):
-            if (v) {
-              SET_BIT2(pColData->pBitMap, iVal, 2);
-            } else {
-              SET_BIT2(pColData->pBitMap, iVal, 0);
-            }
-            break;
-          case (HAS_VALUE | HAS_NULL):
-            SET_BIT2(pColData->pBitMap, iVal, v + 1);
-            break;
-          default:
-            ASSERT(0);
-        }
-      }
-
-      tFree(pBitMap);
+    int32_t szBitMap;
+    if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+      szBitMap = BIT2_SIZE(pColData->nVal);
     } else {
-      code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap,
-                            BIT2_SIZE(pColData->nVal), ppBuf);
-      if (code) goto _exit;
+      szBitMap = BIT1_SIZE(pColData->nVal);
     }
+
+    code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, szBitMap, ppBuf);
+    if (code) goto _exit;
   }
   p += pBlockCol->szBitmap;