Merge pull request #10152 from taosdata/feature/TD-11381-3.0

Feature/td 11381 3.0 - STSRow refactor

2.0/src/client/inc/tsclient.h

@@ -113,8 +113,9 @@ typedef struct SParsedDataColInfo {
   int16_t        numOfCols;
   int16_t        numOfBound;
   uint16_t       flen;        // TODO: get from STSchema
-  uint16_t       allNullLen;  // TODO: get from STSchema
+  uint16_t       allNullLen;  // TODO: get from STSchema(base on SDataRow)
   uint16_t       extendedVarLen;
+  uint16_t       boundNullLen;    // bound column len with all NULL value(without VarDataOffsetT/SColIdx part)
   int32_t *      boundedColumns;  // bound column idx according to schema
   SBoundColumn * cols;
   SBoundIdxInfo *colIdxInfo;
@@ -124,25 +125,13 @@ typedef struct SParsedDataColInfo {
 #define IS_DATA_COL_ORDERED(spd) ((spd->orderStatus) == (int8_t)ORDER_STATUS_ORDERED)
 
 typedef struct {
-  int32_t dataLen;  // len of SDataRow
-  int32_t kvLen;    // len of SKVRow
-} SMemRowInfo;
-typedef struct {
-  uint8_t      memRowType;   // default is 0, that is SDataRow 
-  uint8_t      compareStat;  // 0 no need, 1 need compare
-  TDRowTLenT   kvRowInitLen;
-  SMemRowInfo *rowInfo;
+  uint8_t memRowType;  // default is 0, that is SDataRow
+  int32_t rowSize;
 } SMemRowBuilder;
 
-typedef enum {
-  ROW_COMPARE_NO_NEED = 0,
-  ROW_COMPARE_NEED = 1,
-} ERowCompareStat;
-
 int tsParseTime(SStrToken *pToken, int64_t *time, char **next, char *error, int16_t timePrec);
 
-int  initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols,
-                       int32_t allNullLen);
+int  initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo);
 void destroyMemRowBuilder(SMemRowBuilder *pBuilder);
 
 /**
@@ -175,38 +164,6 @@ static FORCE_INLINE void tscGetMemRowAppendInfo(SSchema *pSchema, uint8_t memRow
   *colId = pSchema[schemaIdx].colId;
 }
 
-/**
- * @brief Applicable to consume by multi-columns
- *
- * @param row
- * @param value
- * @param isCopyVarData In some scenario, the varVal is copied to row directly before calling tdAppend***ColVal()
- * @param colId
- * @param colType
- * @param idx index in SSchema
- * @param pBuilder
- * @param spd
- * @return FORCE_INLINE
- */
-static FORCE_INLINE void tscAppendMemRowColVal(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
-                                               int8_t colType, int32_t toffset, SMemRowBuilder *pBuilder,
-                                               int32_t rowNum) {
-  tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset);
-  if (pBuilder->compareStat == ROW_COMPARE_NEED) {
-    SMemRowInfo *pRowInfo = pBuilder->rowInfo + rowNum;
-    tdGetColAppendDeltaLen(value, colType, &pRowInfo->dataLen, &pRowInfo->kvLen);
-  }
-}
-
-// Applicable to consume by one row
-static FORCE_INLINE void tscAppendMemRowColValEx(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
-                                                 int8_t colType, int32_t toffset, int32_t *dataLen, int32_t *kvLen,
-                                                 uint8_t compareStat) {
-  tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset);
-  if (compareStat == ROW_COMPARE_NEED) {
-    tdGetColAppendDeltaLen(value, colType, dataLen, kvLen);
-  }
-}
 typedef struct STableDataBlocks {
   SName       tableName;
   int8_t      tsSource;     // where does the UNIX timestamp come from, server or client
@@ -513,16 +470,6 @@ static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) {
   return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen;
 }
 
-static FORCE_INLINE void checkAndConvertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) {
-  if (isDataRow(row)) {
-    if (kvLen < (dataLen * KVRatioConvert)) {
-      memRowSetConvert(row);
-    }
-  } else if (kvLen > dataLen) {
-    memRowSetConvert(row);
-  }
-}
-
 static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) {
   memRowSetType(row, memRowType);
   if (isDataRowT(memRowType)) {
@@ -622,8 +569,7 @@ static uint8_t TRUE_VALUE = (uint8_t)TSDB_TRUE;
 static uint8_t FALSE_VALUE = (uint8_t)TSDB_FALSE;
 
 static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pToken, SMemRow row, char *msg, char **str,
-                                               bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId,
-                                               int32_t *dataLen, int32_t *kvLen, uint8_t compareStat) {
+                                               bool primaryKey, int16_t timePrec, int32_t toffset, int16_t colId) {
   int64_t iv;
   int32_t ret;
   char *  endptr = NULL;
@@ -635,26 +581,22 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
   switch (pSchema->type) {
     case TSDB_DATA_TYPE_BOOL: {  // bool
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         if ((pToken->type == TK_BOOL || pToken->type == TK_STRING) && (pToken->n != 0)) {
           if (strncmp(pToken->z, "true", pToken->n) == 0) {
-            tscAppendMemRowColValEx(row, &TRUE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+            tdAppendMemRowColVal(row, &TRUE_VALUE, true, colId, pSchema->type, toffset);
           } else if (strncmp(pToken->z, "false", pToken->n) == 0) {
-            tscAppendMemRowColValEx(row, &FALSE_VALUE, true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                    compareStat);
+            tdAppendMemRowColVal(row, &FALSE_VALUE, true, colId, pSchema->type, toffset);
           } else {
             return tscSQLSyntaxErrMsg(msg, "invalid bool data", pToken->z);
           }
         } else if (pToken->type == TK_INTEGER) {
           iv = strtoll(pToken->z, NULL, 10);
-          tscAppendMemRowColValEx(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset,
-                                  dataLen, kvLen, compareStat);
+          tdAppendMemRowColVal(row, ((iv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset);
         } else if (pToken->type == TK_FLOAT) {
           double dv = strtod(pToken->z, NULL);
-          tscAppendMemRowColValEx(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset,
-                                  dataLen, kvLen, compareStat);
+          tdAppendMemRowColVal(row, ((dv == 0) ? &FALSE_VALUE : &TRUE_VALUE), true, colId, pSchema->type, toffset);
         } else {
           return tscInvalidOperationMsg(msg, "invalid bool data", pToken->z);
         }
@@ -664,8 +606,7 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
 
     case TSDB_DATA_TYPE_TINYINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -675,15 +616,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         uint8_t tmpVal = (uint8_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_UTINYINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -693,15 +633,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         uint8_t tmpVal = (uint8_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_SMALLINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -711,15 +650,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         int16_t tmpVal = (int16_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_USMALLINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -729,15 +667,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         uint16_t tmpVal = (uint16_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_INT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -747,15 +684,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         int32_t tmpVal = (int32_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_UINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -765,15 +701,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         uint32_t tmpVal = (uint32_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;
 
     case TSDB_DATA_TYPE_BIGINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, true);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -782,14 +717,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
           return tscInvalidOperationMsg(msg, "bigint data overflow", pToken->z);
         }
 
-        tscAppendMemRowColValEx(row, &iv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &iv, true, colId, pSchema->type, toffset);
       }
       break;
 
     case TSDB_DATA_TYPE_UBIGINT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         ret = tStrToInteger(pToken->z, pToken->type, pToken->n, &iv, false);
         if (ret != TSDB_CODE_SUCCESS) {
@@ -799,14 +733,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         uint64_t tmpVal = (uint64_t)iv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
       break;
 
     case TSDB_DATA_TYPE_FLOAT:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         double dv;
         if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) {
@@ -819,14 +752,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         }
 
         float tmpVal = (float)dv;
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
       break;
 
     case TSDB_DATA_TYPE_DOUBLE:
       if (isNullStr(pToken)) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         double dv;
         if (TK_ILLEGAL == tscToDouble(pToken, &dv, &endptr)) {
@@ -837,15 +769,14 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
           return tscInvalidOperationMsg(msg, "illegal double data", pToken->z);
         }
 
-        tscAppendMemRowColValEx(row, &dv, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &dv, true, colId, pSchema->type, toffset);
       }
       break;
 
     case TSDB_DATA_TYPE_BINARY:
       // binary data cannot be null-terminated char string, otherwise the last char of the string is lost
       if (pToken->type == TK_NULL) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {  // too long values will return invalid sql, not be truncated automatically
         if (pToken->n + VARSTR_HEADER_SIZE > pSchema->bytes) {  // todo refactor
           return tscInvalidOperationMsg(msg, "string data overflow", pToken->z);
@@ -853,14 +784,13 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         // STR_WITH_SIZE_TO_VARSTR(payload, pToken->z, pToken->n);
         char *rowEnd = memRowEnd(row);
         STR_WITH_SIZE_TO_VARSTR(rowEnd, pToken->z, pToken->n);
-        tscAppendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset);
       }
       break;
 
     case TSDB_DATA_TYPE_NCHAR:
       if (pToken->type == TK_NULL) {
-        tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                compareStat);
+        tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
       } else {
         // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
         int32_t output = 0;
@@ -872,7 +802,7 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
           return tscInvalidOperationMsg(msg, buf, pToken->z);
         }
         varDataSetLen(rowEnd, output);
-        tscAppendMemRowColValEx(row, rowEnd, false, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, rowEnd, false, colId, pSchema->type, toffset);
       }
       break;
 
@@ -881,17 +811,16 @@ static FORCE_INLINE int32_t tsParseOneColumnKV(SSchema *pSchema, SStrToken *pTok
         if (primaryKey) {
           // When building SKVRow primaryKey, we should not skip even with NULL value.
           int64_t tmpVal = 0;
-          tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+          tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
         } else {
-          tscAppendMemRowColValEx(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset, dataLen, kvLen,
-                                  compareStat);
+          tdAppendMemRowColVal(row, getNullValue(pSchema->type), true, colId, pSchema->type, toffset);
         }
       } else {
         int64_t tmpVal;
         if (tsParseTime(pToken, &tmpVal, str, msg, timePrec) != TSDB_CODE_SUCCESS) {
           return tscInvalidOperationMsg(msg, "invalid timestamp", pToken->z);
         }
-        tscAppendMemRowColValEx(row, &tmpVal, true, colId, pSchema->type, toffset, dataLen, kvLen, compareStat);
+        tdAppendMemRowColVal(row, &tmpVal, true, colId, pSchema->type, toffset);
       }
 
       break;

2.0/src/client/src/tscParseInsert.c

@@ -41,51 +41,15 @@ enum {
 static int32_t tscAllocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t *numOfRows);
 static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDataColInfo *pColInfo, SSchema *pSchema,
                                  char *str, char **end);
-int            initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols,
-                                 int32_t allNullLen) {
-  ASSERT(nRows >= 0 && nCols > 0 && (nBoundCols <= nCols));
-  if (nRows > 0) {
-    // already init(bind multiple rows by single column)
-    if (pBuilder->compareStat == ROW_COMPARE_NEED && (pBuilder->rowInfo != NULL)) {
-      return TSDB_CODE_SUCCESS;
-    }
-  }
+int            initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, SParsedDataColInfo *pColInfo) {
+  ASSERT(nRows >= 0 && pColInfo->numOfCols > 0 && (pColInfo->numOfBound <= pColInfo->numOfCols));
 
-  // default compareStat is  ROW_COMPARE_NO_NEED
-  if (nBoundCols == 0) {  // file input
-    pBuilder->memRowType = SMEM_ROW_DATA;
-    return TSDB_CODE_SUCCESS;
+  uint32_t dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + pColInfo->allNullLen;
+  uint32_t kvLen = TD_MEM_ROW_KV_HEAD_SIZE + pColInfo->numOfBound * sizeof(SColIdx) + pColInfo->boundNullLen;
+  if (isUtilizeKVRow(kvLen, dataLen)) {
+    pBuilder->memRowType = SMEM_ROW_KV;
   } else {
-    float boundRatio = ((float)nBoundCols / (float)nCols);
-
-    if (boundRatio < KVRatioKV) {
-      pBuilder->memRowType = SMEM_ROW_KV;
-      return TSDB_CODE_SUCCESS;
-    } else if (boundRatio > KVRatioData) {
-      pBuilder->memRowType = SMEM_ROW_DATA;
-      return TSDB_CODE_SUCCESS;
-    }
-    pBuilder->compareStat = ROW_COMPARE_NEED;
-
-    if (boundRatio < KVRatioPredict) {
-      pBuilder->memRowType = SMEM_ROW_KV;
-    } else {
-      pBuilder->memRowType = SMEM_ROW_DATA;
-    }
-  }
-
-  pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx);
-
-  if (nRows > 0) {
-    pBuilder->rowInfo = tcalloc(nRows, sizeof(SMemRowInfo));
-    if (pBuilder->rowInfo == NULL) {
-      return TSDB_CODE_TSC_OUT_OF_MEMORY;
-    }
-
-    for (int i = 0; i < nRows; ++i) {
-      (pBuilder->rowInfo + i)->dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen;
-      (pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen;
-    }
+    pBuilder->memRowType = SMEM_ROW_DATA;
   }
 
   return TSDB_CODE_SUCCESS;
@@ -457,8 +421,6 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
   STableMeta *        pTableMeta = pDataBlocks->pTableMeta;
   SSchema *           schema = tscGetTableSchema(pTableMeta);
   SMemRowBuilder *    pBuilder = &pDataBlocks->rowBuilder;
-  int32_t             dataLen = spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE;
-  int32_t             kvLen = pBuilder->kvRowInitLen;
   bool                isParseBindParam = false;
 
   initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound);
@@ -535,8 +497,8 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
     int16_t colId = -1;
     tscGetMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &toffset, &colId);
 
-    int32_t ret = tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset,
-                                     colId, &dataLen, &kvLen, pBuilder->compareStat);
+    int32_t ret =
+        tsParseOneColumnKV(pSchema, &sToken, row, pInsertParam->msg, str, isPrimaryKey, timePrec, toffset, colId);
     if (ret != TSDB_CODE_SUCCESS) {
       return ret;
     }
@@ -551,13 +513,8 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
   }
 
   if (!isParseBindParam) {
-    // 2. check and set convert flag
-    if (pBuilder->compareStat == ROW_COMPARE_NEED) {
-      checkAndConvertMemRow(row, dataLen, kvLen);
-    }
-
-    // 3. set the null value for the columns that do not assign values
-    if ((spd->numOfBound < spd->numOfCols) && isDataRow(row) && !isNeedConvertRow(row)) {
+    // set the null value for the columns that do not assign values
+    if ((spd->numOfBound < spd->numOfCols) && isDataRow(row)) {
       SDataRow dataRow = memRowDataBody(row);
       for (int32_t i = 0; i < spd->numOfCols; ++i) {
         if (spd->cols[i].valStat == VAL_STAT_NONE) {
@@ -567,7 +524,7 @@ int tsParseOneRow(char **str, STableDataBlocks *pDataBlocks, int16_t timePrec, i
     }
   }
 
-  *len = getExtendedRowSize(pDataBlocks);
+  *len = pBuilder->rowSize;
 
   return TSDB_CODE_SUCCESS;
 }
@@ -620,11 +577,10 @@ int32_t tsParseValues(char **str, STableDataBlocks *pDataBlock, int maxRows, SIn
 
   int32_t extendedRowSize = getExtendedRowSize(pDataBlock);
 
-  if (TSDB_CODE_SUCCESS !=
-      (code = initMemRowBuilder(&pDataBlock->rowBuilder, 0, tinfo.numOfColumns, pDataBlock->boundColumnInfo.numOfBound,
-                                pDataBlock->boundColumnInfo.allNullLen))) {
+  if (TSDB_CODE_SUCCESS != (code = initMemRowBuilder(&pDataBlock->rowBuilder, 0, &pDataBlock->boundColumnInfo))) {
     return code;
   }
+  pDataBlock->rowBuilder.rowSize = extendedRowSize;
   while (1) {
     index = 0;
     sToken = tStrGetToken(*str, &index, false);
@@ -703,6 +659,7 @@ void tscSetBoundColumnInfo(SParsedDataColInfo *pColInfo, SSchema *pSchema, int32
     pColInfo->boundedColumns[i] = i;
   }
   pColInfo->allNullLen += pColInfo->flen;
+  pColInfo->boundNullLen = pColInfo->allNullLen;  // default set allNullLen
   pColInfo->extendedVarLen = (uint16_t)(nVar * sizeof(VarDataOffsetT));
 }
 
@@ -1200,6 +1157,7 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
   int32_t nCols = pColInfo->numOfCols;
 
   pColInfo->numOfBound = 0;
+  pColInfo->boundNullLen = 0;
   memset(pColInfo->boundedColumns, 0, sizeof(int32_t) * nCols);
   for (int32_t i = 0; i < nCols; ++i) {
     pColInfo->cols[i].valStat = VAL_STAT_NONE;
@@ -1249,6 +1207,17 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
         pColInfo->cols[t].valStat = VAL_STAT_HAS;
         pColInfo->boundedColumns[pColInfo->numOfBound] = t;
         ++pColInfo->numOfBound;
+        switch (pSchema[t].type) {
+          case TSDB_DATA_TYPE_BINARY:
+            pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES);
+            break;
+          case TSDB_DATA_TYPE_NCHAR:
+            pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
+            break;
+          default:
+            pColInfo->boundNullLen += TYPE_BYTES[pSchema[t].type];
+            break;
+        }
         findColumnIndex = true;
         if (isOrdered && (lastColIdx > t)) {
           isOrdered = false;
@@ -1272,6 +1241,17 @@ static int32_t parseBoundColumns(SInsertStatementParam *pInsertParam, SParsedDat
           pColInfo->cols[t].valStat = VAL_STAT_HAS;
           pColInfo->boundedColumns[pColInfo->numOfBound] = t;
           ++pColInfo->numOfBound;
+          switch (pSchema[t].type) {
+            case TSDB_DATA_TYPE_BINARY:
+              pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + CHAR_BYTES);
+              break;
+            case TSDB_DATA_TYPE_NCHAR:
+              pColInfo->boundNullLen += (VARSTR_HEADER_SIZE + TSDB_NCHAR_SIZE);
+              break;
+            default:
+              pColInfo->boundNullLen += TYPE_BYTES[pSchema[t].type];
+              break;
+          }
           findColumnIndex = true;
           if (isOrdered && (lastColIdx > t)) {
             isOrdered = false;
@@ -1715,12 +1695,17 @@ static void parseFileSendDataBlock(void *param, TAOS_RES *tres, int32_t numOfRow
     goto _error;
   }
 
-  tscAllocateMemIfNeed(pTableDataBlock, getExtendedRowSize(pTableDataBlock), &maxRows);
+  int32_t extendedRowSize = getExtendedRowSize(pTableDataBlock);
+  tscAllocateMemIfNeed(pTableDataBlock, extendedRowSize, &maxRows);
   tokenBuf = calloc(1, TSDB_MAX_BYTES_PER_ROW);
   if (tokenBuf == NULL) {
     code = TSDB_CODE_TSC_OUT_OF_MEMORY;
     goto _error;
   }
+  
+  // insert from .csv means full and ordered columns, thus use SDataRow all the time
+  ASSERT(SMEM_ROW_DATA == pTableDataBlock->rowBuilder.memRowType);
+  pTableDataBlock->rowBuilder.rowSize = extendedRowSize;
 
   while ((readLen = tgetline(&line, &n, fp)) != -1) {
     if (('\r' == line[readLen - 1]) || ('\n' == line[readLen - 1])) {

2.0/src/client/src/tscUtil.c

@@ -1881,18 +1881,11 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SI
     }
   } else {
     for (int32_t i = 0; i < numOfRows; ++i) {
-      char* payload = (blkKeyTuple + i)->payloadAddr;
-      if (isNeedConvertRow(payload)) {
-        convertSMemRow(pDataBlock, payload, pTableDataBlock);
-        TDRowTLenT rowTLen = memRowTLen(pDataBlock);
-        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
-        pBlock->dataLen += rowTLen;
-      } else {
-        TDRowTLenT rowTLen = memRowTLen(payload);
-        memcpy(pDataBlock, payload, rowTLen);
-        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
-        pBlock->dataLen += rowTLen;
-      }
+      char*      payload = (blkKeyTuple + i)->payloadAddr;
+      TDRowLenT rowTLen = memRowTLen(payload);
+      memcpy(pDataBlock, payload, rowTLen);
+      pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
+      pBlock->dataLen += rowTLen;
     }
   }
 

include/common/tdataformat.h

@@ -24,6 +24,13 @@
 extern "C" {
 #endif
 
+// Imported since 3.0 and use bitmap to demonstrate None/Null/Norm, while use Null/Norm below 3.0 without of bitmap.
+#define TD_SUPPORT_BITMAP
+#define TD_SUPPORT_READ2
+#define TD_SUPPORT_BACK2  // suppport back compatibility of 2.0
+
+#define TASSERT(x) ASSERT(x)
+
 #define STR_TO_VARSTR(x, str)                     \
   do {                                            \
     VarDataLenT __len = (VarDataLenT)strlen(str); \
@@ -53,9 +60,9 @@ extern "C" {
 // ----------------- TSDB COLUMN DEFINITION
 typedef struct {
   int8_t   type;    // Column type
-  int16_t  colId;   // column ID
+  col_id_t colId;   // column ID(start from PRIMARYKEY_TIMESTAMP_COL_ID(1))
   int16_t  bytes;   // column bytes (restore to int16_t in case of misuse)
-  uint16_t offset;  // point offset in SDataRow after the header part.
+  uint16_t offset;  // point offset in STpRow after the header part.
 } STColumn;
 
 #define colType(col) ((col)->type)
@@ -72,9 +79,9 @@ typedef struct {
 typedef struct {
   int version;    // version
   int numOfCols;  // Number of columns appended
-  int tlen;  // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) +
-             // (bytes))
-  uint16_t flen;  // First part length in a SDataRow after the header part
+  int tlen;       // maximum length of a STpRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) +
+                  // (bytes))
+  uint16_t flen;  // First part length in a STpRow after the header part
   uint16_t vlen;  // pure value part length, excluded the overhead (bytes only)
   STColumn columns[];
 } STSchema;
@@ -118,6 +125,13 @@ typedef struct {
   STColumn *columns;
 } STSchemaBuilder;
 
+#define TD_VTYPE_BITS 2   // val type
+#define TD_VTYPE_PARTS 4  // 8 bits / TD_VTYPE_BITS = 4
+#define TD_VTYPE_OPTR 3   // TD_VTYPE_PARTS - 1, utilize to get remainder
+
+#define TD_BITMAP_BYTES(cnt) (ceil((double)cnt / TD_VTYPE_PARTS))
+#define TD_BIT_TO_BYTES(cnt) (ceil((double)cnt / 8))
+
 int       tdInitTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
 void      tdDestroyTSchemaBuilder(STSchemaBuilder *pBuilder);
 void      tdResetTSchemaBuilder(STSchemaBuilder *pBuilder, int32_t version);
@@ -125,6 +139,8 @@ int       tdAddColToSchema(STSchemaBuilder *pBuilder, int8_t type, int16_t colId
 STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder);
 
 // ----------------- Semantic timestamp key definition
+#ifdef TD_2_0
+
 typedef uint64_t TKEY;
 
 #define TKEY_INVALID UINT64_MAX
@@ -144,6 +160,29 @@ typedef uint64_t TKEY;
 
 #define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
 
+#else
+
+// typedef uint64_t TKEY;
+#define TKEY TSKEY
+
+#define TKEY_INVALID UINT64_MAX
+#define TKEY_NULL TKEY_INVALID
+#define TKEY_NEGATIVE_FLAG (((TKEY)1) << 63)
+#define TKEY_VALUE_FILTER (~(TKEY_NEGATIVE_FLAG))
+
+#define TKEY_IS_NEGATIVE(tkey) (((tkey)&TKEY_NEGATIVE_FLAG) != 0)
+#define TKEY_IS_DELETED(tkey) (false)
+
+#define tdGetTKEY(key) (key)
+#define tdGetKey(tskey) (tskey)
+
+#define MIN_TS_KEY ((TSKEY)0x8000000000000001)
+#define MAX_TS_KEY ((TSKEY)0x7fffffffffffffff)
+
+#define TD_TO_TKEY(key) tdGetTKEY(((key) < MIN_TS_KEY) ? MIN_TS_KEY : (((key) > MAX_TS_KEY) ? MAX_TS_KEY : key))
+
+#endif
+
 static FORCE_INLINE TKEY keyToTkey(TSKEY key) {
   TSKEY lkey = key;
   if (key > MAX_TS_KEY) {
@@ -168,6 +207,7 @@ static FORCE_INLINE int tkeyComparFn(const void *tkey1, const void *tkey2) {
   }
 }
 
+#if 0
 // ----------------- Data row structure
 
 /* A data row, the format is like below:
@@ -317,10 +357,12 @@ static FORCE_INLINE void tdCopyColOfRowBySchema(SDataRow dst, STSchema *pDstSche
       memcpy(pData, value, pSrcSchema->columns[srcIdx].bytes);
   }
 }
-
+#endif
 // ----------------- Data column structure
 typedef struct SDataCol {
-  int8_t          type;       // column type
+  int8_t          type;        // column type
+  uint8_t         bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
+  uint8_t         reserve : 7;
   int16_t         colId;      // column ID
   int             bytes;      // column data bytes defined
   int             offset;     // data offset in a SDataRow (including the header size)
@@ -328,20 +370,23 @@ typedef struct SDataCol {
   int             len;        // column data length
   VarDataOffsetT *dataOff;    // For binary and nchar data, the offset in the data column
   void *          pData;      // Actual data pointer
+  void *          pBitmap;    // Bitmap pointer
   TSKEY           ts;         // only used in last NULL column
 } SDataCol;
 
 #define isAllRowsNull(pCol) ((pCol)->len == 0)
+#define isAllRowsNone(pCol) ((pCol)->len == 0)
 static FORCE_INLINE void dataColReset(SDataCol *pDataCol) { pDataCol->len = 0; }
 
 int tdAllocMemForCol(SDataCol *pCol, int maxPoints);
 
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints);
 int  dataColAppendVal(SDataCol *pCol, const void *value, int numOfRows, int maxPoints);
-void dataColSetOffset(SDataCol *pCol, int nEle);
+void *dataColSetOffset(SDataCol *pCol, int nEle);
 
 bool isNEleNull(SDataCol *pCol, int nEle);
 
+#if 0
 // Get the data pointer from a column-wised data
 static FORCE_INLINE const void *tdGetColDataOfRow(SDataCol *pCol, int row) {
   if (isAllRowsNull(pCol)) {
@@ -363,13 +408,13 @@ static FORCE_INLINE int32_t dataColGetNEleLen(SDataCol *pDataCol, int rows) {
     return TYPE_BYTES[pDataCol->type] * rows;
   }
 }
-
+#endif
 typedef struct {
-  int       maxCols;    // max number of columns
+  col_id_t  maxCols;    // max number of columns
+  col_id_t  numOfCols;  // Total number of cols
   int       maxPoints;  // max number of points
   int       numOfRows;
-  int       numOfCols;  // Total number of cols
-  int       sversion;   // TODO: set sversion
+  int       sversion;  // TODO: set sversion
   SDataCol *cols;
 } SDataCols;
 
@@ -481,6 +526,7 @@ static FORCE_INLINE void *tdGetKVRowIdxOfCol(SKVRow row, int16_t colId) {
   return taosbsearch(&colId, kvRowColIdx(row), kvRowNCols(row), sizeof(SColIdx), comparTagId, TD_EQ);
 }
 
+#if 0
 // offset here not include kvRow header length
 static FORCE_INLINE int tdAppendKvColVal(SKVRow row, const void *value, bool isCopyValData, int16_t colId, int8_t type,
                                          int32_t offset) {
@@ -527,7 +573,7 @@ static FORCE_INLINE void *tdGetKVRowValOfColEx(SKVRow row, int16_t colId, int32_
   }
   return NULL;
 }
-
+#endif
 // ----------------- K-V data row builder
 typedef struct {
   int16_t  tCols;
@@ -571,8 +617,8 @@ static FORCE_INLINE int tdAddColToKVRow(SKVRowBuilder *pBuilder, int16_t colId,
 
   return 0;
 }
-
-// ----------------- SMemRow appended with sequential data row structure
+#if 0
+// ----------------- SMemRow appended with tuple row structure
 /*
  * |---------|------------------------------------------------- len ---------------------------------->|
  * |<--------     Head      ------>|<---------   flen -------------->|                                 |
@@ -605,22 +651,17 @@ typedef void *SMemRow;
 
 #define SMEM_ROW_DATA 0x0U      // SDataRow
 #define SMEM_ROW_KV 0x01U       // SKVRow
-#define SMEM_ROW_CONVERT 0x80U  // SMemRow convert flag
 
-#define KVRatioKV (0.2f)  // all bool
-#define KVRatioPredict (0.4f)
-#define KVRatioData (0.75f)  // all bigint
 #define KVRatioConvert (0.9f)
 
 #define memRowType(r) ((*(uint8_t *)(r)) & 0x01)
 
 #define memRowSetType(r, t) ((*(uint8_t *)(r)) = (t))  // set the total byte in case of dirty memory
-#define memRowSetConvert(r) ((*(uint8_t *)(r)) = (((*(uint8_t *)(r)) & 0x7F) | SMEM_ROW_CONVERT))  // highest bit
 #define isDataRowT(t) (SMEM_ROW_DATA == (((uint8_t)(t)) & 0x01))
 #define isDataRow(r) (SMEM_ROW_DATA == memRowType(r))
 #define isKvRowT(t) (SMEM_ROW_KV == (((uint8_t)(t)) & 0x01))
 #define isKvRow(r) (SMEM_ROW_KV == memRowType(r))
-#define isNeedConvertRow(r) (((*(uint8_t *)(r)) & 0x80) == SMEM_ROW_CONVERT)
+#define isUtilizeKVRow(k, d) ((k) < ((d)*KVRatioConvert))
 
 #define memRowDataBody(r) POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE)  // section after flag
 #define memRowKvBody(r) \
@@ -630,9 +671,9 @@ typedef void *SMemRow;
 #define memRowKvLen(r) (*(TDRowLenT *)memRowKvBody(r))      //  0~65535
 
 #define memRowDataTLen(r) \
-  ((TDRowTLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE))  // using uint32_t/int32_t to store the TLen
+  ((TDRowLenT)(memRowDataLen(r) + TD_MEM_ROW_TYPE_SIZE))  // using uint32_t/int32_t to store the TLen
 
-#define memRowKvTLen(r) ((TDRowTLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE))
+#define memRowKvTLen(r) ((TDRowLenT)(memRowKvLen(r) + TD_MEM_ROW_KV_TYPE_VER_SIZE))
 
 #define memRowLen(r) (isDataRow(r) ? memRowDataLen(r) : memRowKvLen(r))
 #define memRowTLen(r) (isDataRow(r) ? memRowDataTLen(r) : memRowKvTLen(r))  // using uint32_t/int32_t to store the TLen
@@ -722,31 +763,6 @@ static FORCE_INLINE int32_t tdGetColAppendLen(uint8_t rowType, const void *value
   return len;
 }
 
-/**
- * 1. calculate the delta of AllNullLen for SDataRow.
- * 2. calculate the real len for SKVRow.
- */
-static FORCE_INLINE void tdGetColAppendDeltaLen(const void *value, int8_t colType, int32_t *dataLen, int32_t *kvLen) {
-  switch (colType) {
-    case TSDB_DATA_TYPE_BINARY: {
-      int32_t varLen = varDataLen(value);
-      *dataLen += (varLen - CHAR_BYTES);
-      *kvLen += (varLen + sizeof(SColIdx));
-      break;
-    }
-    case TSDB_DATA_TYPE_NCHAR: {
-      int32_t varLen = varDataLen(value);
-      *dataLen += (varLen - TSDB_NCHAR_SIZE);
-      *kvLen += (varLen + sizeof(SColIdx));
-      break;
-    }
-    default: {
-      *kvLen += (TYPE_BYTES[colType] + sizeof(SColIdx));
-      break;
-    }
-  }
-}
-
 typedef struct {
   int16_t colId;
   uint8_t colType;
@@ -760,53 +776,6 @@ static FORCE_INLINE void setSColInfo(SColInfo *colInfo, int16_t colId, uint8_t c
 }
 
 SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2);
-
-#if 0
-// ----------------- Raw payload structure for row:
-/* |<------------ Head ------------->|<----------- body of column data tuple ------------------->|
- * |                                 |<----------------- flen ------------->|<--- value part --->|
- * |SMemRowType| dataTLen |  nCols   |  colId  | colType | offset   |  ...  | value |...|...|... |
- * +-----------+----------+----------+--------------------------------------|--------------------|
- * | uint8_t   | uint32_t | uint16_t | int16_t | uint8_t | uint16_t |  ...  |.......|...|...|... |
- * +-----------+----------+----------+--------------------------------------+--------------------|
- *  1. offset in column data tuple starts from the value part in case of uint16_t overflow.
- *  2. dataTLen: total length including the header and body.
- */
-
-#define PAYLOAD_NCOLS_LEN sizeof(uint16_t)
-#define PAYLOAD_NCOLS_OFFSET (sizeof(uint8_t) + sizeof(TDRowTLenT))
-#define PAYLOAD_HEADER_LEN (PAYLOAD_NCOLS_OFFSET + PAYLOAD_NCOLS_LEN)
-#define PAYLOAD_ID_LEN sizeof(int16_t)
-#define PAYLOAD_ID_TYPE_LEN (sizeof(int16_t) + sizeof(uint8_t))
-#define PAYLOAD_COL_HEAD_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(uint16_t))
-#define PAYLOAD_PRIMARY_COL_LEN (PAYLOAD_ID_TYPE_LEN + sizeof(TSKEY))
-
-#define payloadBody(r) POINTER_SHIFT(r, PAYLOAD_HEADER_LEN)
-#define payloadType(r) (*(uint8_t *)(r))
-#define payloadSetType(r, t) (payloadType(r) = (t))
-#define payloadTLen(r) (*(TDRowTLenT *)POINTER_SHIFT(r, TD_MEM_ROW_TYPE_SIZE))  // including total header
-#define payloadSetTLen(r, l) (payloadTLen(r) = (l))
-#define payloadNCols(r) (*(TDRowLenT *)POINTER_SHIFT(r, PAYLOAD_NCOLS_OFFSET))
-#define payloadSetNCols(r, n) (payloadNCols(r) = (n))
-#define payloadValuesOffset(r) \
-  (PAYLOAD_HEADER_LEN + payloadNCols(r) * PAYLOAD_COL_HEAD_LEN)    // avoid using the macro in loop
-#define payloadValues(r) POINTER_SHIFT(r, payloadValuesOffset(r))  // avoid using the macro in loop
-#define payloadColId(c) (*(int16_t *)(c))
-#define payloadColType(c) (*(uint8_t *)POINTER_SHIFT(c, PAYLOAD_ID_LEN))
-#define payloadColOffset(c) (*(uint16_t *)POINTER_SHIFT(c, PAYLOAD_ID_TYPE_LEN))
-#define payloadColValue(c) POINTER_SHIFT(c, payloadColOffset(c))
-
-#define payloadColSetId(c, i) (payloadColId(c) = (i))
-#define payloadColSetType(c, t) (payloadColType(c) = (t))
-#define payloadColSetOffset(c, o) (payloadColOffset(c) = (o))
-
-#define payloadTSKey(r) (*(TSKEY *)POINTER_SHIFT(r, payloadValuesOffset(r)))
-#define payloadTKey(r) (*(TKEY *)POINTER_SHIFT(r, payloadValuesOffset(r)))
-#define payloadKey(r) tdGetKey(payloadTKey(r))
-
-
-static FORCE_INLINE char *payloadNextCol(char *pCol) { return (char *)POINTER_SHIFT(pCol, PAYLOAD_COL_HEAD_LEN); }
-
 #endif
 
 #ifdef __cplusplus

include/common/tmsg.h

@@ -25,7 +25,7 @@ extern "C" {
 #include "taoserror.h"
 #include "tarray.h"
 #include "tcoding.h"
-#include "tdataformat.h"
+#include "trow.h"
 #include "thash.h"
 #include "tlist.h"
 
@@ -218,7 +218,7 @@ typedef struct {
 typedef struct {
   int32_t totalLen;
   int32_t len;
-  SMemRow row;
+  STSRow  *row;
 } SSubmitBlkIter;
 
 typedef struct {
@@ -230,7 +230,7 @@ typedef struct {
 int     tInitSubmitMsgIter(SSubmitMsg* pMsg, SSubmitMsgIter* pIter);
 int     tGetSubmitMsgNext(SSubmitMsgIter* pIter, SSubmitBlk** pPBlock);
 int     tInitSubmitBlkIter(SSubmitBlk* pBlock, SSubmitBlkIter* pIter);
-SMemRow tGetSubmitBlkNext(SSubmitBlkIter* pIter);
+STSRow* tGetSubmitBlkNext(SSubmitBlkIter* pIter);
 
 typedef struct {
   int32_t index;  // index of failed block in submit blocks

include/common/tschema.h

@@ -18,13 +18,12 @@
 
 #include "os.h"
 #include "tarray.h"
+#include "ttypes.h"
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-typedef uint16_t col_id_t;
-
 #if 0
 typedef struct STColumn {
   /// column name

include/common/ttypes.h

@@ -10,8 +10,10 @@ extern "C" {
 
 // ----------------- For variable data types such as TSDB_DATA_TYPE_BINARY and TSDB_DATA_TYPE_NCHAR
 typedef int32_t  VarDataOffsetT;
-typedef uint16_t TDRowLenT;    // not including overhead: 0 ~ 65535
-typedef uint32_t TDRowTLenT;   // total length, including overhead
+typedef uint32_t TDRowLenT;
+typedef uint8_t  TDRowValT;
+typedef uint16_t col_id_t;
+typedef int8_t   col_type_t;
 
 typedef struct tstr {
   VarDataLenT len;

include/util/tdef.h

@@ -321,6 +321,7 @@ do { \
 #define TSDB_MAX_BINARY_LEN            (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384
 #define TSDB_MAX_NCHAR_LEN             (TSDB_MAX_FIELD_LEN-TSDB_KEYSIZE) // keep 16384
 #define PRIMARYKEY_TIMESTAMP_COL_ID     1
+#define COL_REACH_END(colId, maxColId)  ((colId) > (maxColId))
 
 #define TSDB_MAX_RPC_THREADS            5
 

include/util/types.h

@@ -72,14 +72,13 @@ static FORCE_INLINE double taos_align_get_double(const char *pBuf) {
   #define SET_DOUBLE_PTR(x, y) { (*(double *)(x)) = (*(double *)(y)); }
 // #endif
 
-typedef int16_t  VarDataLenT;  // maxVarDataLen: 32767
+typedef uint16_t  VarDataLenT;  // maxVarDataLen: 32767
 #define VARSTR_HEADER_SIZE  sizeof(VarDataLenT)
 
 #define varDataLen(v) ((VarDataLenT *)(v))[0]
 #define varDataVal(v) ((void *)((char *)v + VARSTR_HEADER_SIZE))
 
 typedef int32_t VarDataOffsetT;
-typedef int16_t VarDataLenT;  // maxVarDataLen: 32767
 
 #ifdef __cplusplus
 }

source/client/test/clientTests.cpp

@@ -569,7 +569,13 @@ TEST(testCase, projection_query_tables) {
   TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0);
   ASSERT_NE(pConn, nullptr);
 
-  TAOS_RES* pRes = taos_query(pConn, "use abc1");
+  TAOS_RES* pRes = taos_query(pConn, "create database if not exists abc1 vgroups 2");
+  if (taos_errno(pRes) != 0) {
+    printf("error in create db, reason:%s\n", taos_errstr(pRes));
+  }
+  taos_free_result(pRes);
+
+  pRes = taos_query(pConn, "use abc1");
   taos_free_result(pRes);
 
   pRes = taos_query(pConn, "create stable st1 (ts timestamp, k int) tags(a int)");

source/common/src/tdataformat.c

@@ -20,14 +20,23 @@
 #include "tarray.h"
 
 static void dataColSetNEleNull(SDataCol *pCol, int nEle);
+#if 0
 static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
                                int limit2, int tRows, bool forceSetNull);
-
+#endif
 int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
   int spaceNeeded = pCol->bytes * maxPoints;
   if(IS_VAR_DATA_TYPE(pCol->type)) {
     spaceNeeded += sizeof(VarDataOffsetT) * maxPoints;
   }
+#ifdef TD_SUPPORT_BITMAP
+  int32_t nBitmapBytes = (int32_t)TD_BITMAP_BYTES(maxPoints);
+  spaceNeeded += (int)nBitmapBytes;
+  // TODO: Currently, the compression of bitmap parts is affiliated to the column data parts, thus allocate 1 more
+  // TYPE_BYTES as to comprise complete TYPE_BYTES. Otherwise, invalid read/write would be triggered.
+  spaceNeeded += TYPE_BYTES[pCol->type];
+#endif
+
   if(pCol->spaceSize < spaceNeeded) {
     void* ptr = realloc(pCol->pData, spaceNeeded);
     if(ptr == NULL) {
@@ -39,9 +48,18 @@ int tdAllocMemForCol(SDataCol *pCol, int maxPoints) {
       pCol->spaceSize = spaceNeeded;
     }
   }
-  if(IS_VAR_DATA_TYPE(pCol->type)) {
+#ifdef TD_SUPPORT_BITMAP
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
+    pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints);
+    pCol->dataOff = POINTER_SHIFT(pCol->pBitmap, nBitmapBytes);
+  } else {
+    pCol->pBitmap = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints);
+  }
+#else
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
     pCol->dataOff = POINTER_SHIFT(pCol->pData, pCol->bytes * maxPoints);
   }
+#endif
   return 0;
 }
 
@@ -185,11 +203,16 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) {
   schemaFLen(pSchema) = pBuilder->flen;
   schemaVLen(pSchema) = pBuilder->vlen;
 
+#ifdef TD_SUPPORT_BITMAP
+  schemaTLen(pSchema) += (int)TD_BITMAP_BYTES(schemaNCols(pSchema));
+#endif
+
   memcpy(schemaColAt(pSchema, 0), pBuilder->columns, sizeof(STColumn) * pBuilder->nCols);
 
   return pSchema;
 }
 
+#if 0
 /**
  * Initialize a data row
  */
@@ -230,12 +253,13 @@ SMemRow tdMemRowDup(SMemRow row) {
   memRowCpy(trow, row);
   return trow;
 }
+#endif
 
 void dataColInit(SDataCol *pDataCol, STColumn *pCol, int maxPoints) {
   pDataCol->type = colType(pCol);
   pDataCol->colId = colColId(pCol);
   pDataCol->bytes = colBytes(pCol);
-  pDataCol->offset = colOffset(pCol) + TD_DATA_ROW_HEAD_SIZE;
+  pDataCol->offset = colOffset(pCol) + 0; //TD_DATA_ROW_HEAD_SIZE;
 
   pDataCol->len = 0;
 }
@@ -311,7 +335,7 @@ static void dataColSetNEleNull(SDataCol *pCol, int nEle) {
   }
 }
 
-void dataColSetOffset(SDataCol *pCol, int nEle) {
+void *dataColSetOffset(SDataCol *pCol, int nEle) {
   ASSERT(((pCol->type == TSDB_DATA_TYPE_BINARY) || (pCol->type == TSDB_DATA_TYPE_NCHAR)));
 
   void *tptr = pCol->pData;
@@ -323,6 +347,7 @@ void dataColSetOffset(SDataCol *pCol, int nEle) {
     offset += varDataTLen(tptr);
     tptr = POINTER_SHIFT(tptr, varDataTLen(tptr));
   }
+  return POINTER_SHIFT(tptr, varDataTLen(tptr));
 }
 
 SDataCols *tdNewDataCols(int maxCols, int maxRows) {
@@ -440,7 +465,7 @@ void tdResetDataCols(SDataCols *pCols) {
     }
   }
 }
-
+#if 0
 static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
   ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < dataRowKey(row));
 
@@ -475,7 +500,7 @@ static void tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols
   pCols->numOfRows++;
 }
 
-static void tdAppendKvRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
+static void tdAppendKVRowToDataCol(SKVRow row, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
   ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < kvRowKey(row));
 
   int rcol = 0;
@@ -516,7 +541,7 @@ void tdAppendMemRowToDataCol(SMemRow row, STSchema *pSchema, SDataCols *pCols, b
   if (isDataRow(row)) {
     tdAppendDataRowToDataCol(memRowDataBody(row), pSchema, pCols, forceSetNull);
   } else if (isKvRow(row)) {
-    tdAppendKvRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull);
+    tdAppendKVRowToDataCol(memRowKvBody(row), pSchema, pCols, forceSetNull);
   } else {
     ASSERT(0);
   }
@@ -613,6 +638,7 @@ static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, i
     ASSERT(target->numOfRows <= target->maxPoints);
   }
 }
+#endif
 
 SKVRow tdKVRowDup(SKVRow row) {
   SKVRow trow = malloc(kvRowLen(row));
@@ -772,7 +798,7 @@ SKVRow tdGetKVRowFromBuilder(SKVRowBuilder *pBuilder) {
 
   return row;
 }
-
+#if 0
 SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSchema1, STSchema *pSchema2) {
 #if 0
   ASSERT(memRowKey(row1) == memRowKey(row2));
@@ -792,7 +818,7 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch
   dataRowSetVersion(dataRow, schemaVersion(pSchema1));  // use latest schema version
   dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pSchema1->flen));
 
-  TDRowTLenT dataLen = 0, kvLen = TD_MEM_ROW_KV_HEAD_SIZE;
+  TDRowLenT dataLen = 0, kvLen = TD_MEM_ROW_KV_HEAD_SIZE;
 
   int32_t  i = 0;  // row1
   int32_t  j = 0;  // row2
@@ -866,3 +892,4 @@ SMemRow mergeTwoMemRows(void *buffer, SMemRow row1, SMemRow row2, STSchema *pSch
   taosArrayDestroy(stashRow);
   return buffer;
 }
+#endif
\ No newline at end of file

source/common/src/tmsg.c

@@ -67,19 +67,19 @@ int32_t tInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter) {
   if (pBlock->dataLen <= 0) return -1;
   pIter->totalLen = pBlock->dataLen;
   pIter->len = 0;
-  pIter->row = (SMemRow)(pBlock->data + pBlock->schemaLen);
+  pIter->row = (STSRow *)(pBlock->data + pBlock->schemaLen);
   return 0;
 }
 
-SMemRow tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
-  SMemRow row = pIter->row;
+STSRow *tGetSubmitBlkNext(SSubmitBlkIter *pIter) {
+  STSRow *row = pIter->row;
 
   if (pIter->len >= pIter->totalLen) {
     return NULL;
   } else {
-    pIter->len += memRowTLen(row);
+    pIter->len += TD_ROW_LEN(row);
     if (pIter->len < pIter->totalLen) {
-      pIter->row = POINTER_SHIFT(row, memRowTLen(row));
+      pIter->row = POINTER_SHIFT(row, TD_ROW_LEN(row));
     }
     return row;
   }

source/common/src/trow.c

@@ -14,6 +14,89 @@
  */
 
 #include "trow.h"
+#include "tarray.h"
+
+const uint8_t tdVTypeByte[3] = {
+    TD_VTYPE_NORM_BYTE,  // TD_VTYPE_NORM
+    TD_VTYPE_NONE_BYTE,  // TD_VTYPE_NONE
+    TD_VTYPE_NULL_BYTE,  // TD_VTYPE_NULL
+};
+
+// static void dataColSetNEleNull(SDataCol *pCol, int nEle);
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+                               int limit2, int tRows, bool forceSetNull);
+
+static FORCE_INLINE void dataColSetNullAt(SDataCol *pCol, int index, bool setBitmap) {
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
+    pCol->dataOff[index] = pCol->len;
+    char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
+    setVardataNull(ptr, pCol->type);
+    pCol->len += varDataTLen(ptr);
+  } else {
+    setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes);
+    pCol->len += TYPE_BYTES[pCol->type];
+  }
+  if (setBitmap) {
+    tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE);
+  }
+}
+
+// static void dataColSetNEleNull(SDataCol *pCol, int nEle) {
+//   if (IS_VAR_DATA_TYPE(pCol->type)) {
+//     pCol->len = 0;
+//     for (int i = 0; i < nEle; i++) {
+//       dataColSetNullAt(pCol, i);
+//     }
+//   } else {
+//     setNullN(pCol->pData, pCol->type, pCol->bytes, nEle);
+//     pCol->len = TYPE_BYTES[pCol->type] * nEle;
+//   }
+// }
+
+int32_t tdSetBitmapValTypeN(void *pBitmap, int16_t nEle, TDRowValT valType) {
+  TASSERT(valType < TD_VTYPE_MAX);
+  int16_t nBytes = nEle / TD_VTYPE_PARTS;
+  for (int i = 0; i < nBytes; ++i) {
+    *(uint8_t *)pBitmap = tdVTypeByte[valType];
+    pBitmap = POINTER_SHIFT(pBitmap, 1);
+  }
+  int16_t nLeft = nEle - nBytes * TD_VTYPE_BITS;
+
+  for (int j = 0; j < nLeft; ++j) {
+    tdSetBitmapValType(pBitmap, j, valType);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+static FORCE_INLINE void dataColSetNoneAt(SDataCol *pCol, int index, bool setBitmap) {
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
+    pCol->dataOff[index] = pCol->len;
+    char *ptr = POINTER_SHIFT(pCol->pData, pCol->len);
+    setVardataNull(ptr, pCol->type);
+    pCol->len += varDataTLen(ptr);
+  } else {
+    setNull(POINTER_SHIFT(pCol->pData, TYPE_BYTES[pCol->type] * index), pCol->type, pCol->bytes);
+    pCol->len += TYPE_BYTES[pCol->type];
+  }
+  if(setBitmap) {
+    tdSetBitmapValType(pCol->pBitmap, index, TD_VTYPE_NONE);
+  }
+}
+
+static void dataColSetNEleNone(SDataCol *pCol, int nEle) {
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
+    pCol->len = 0;
+    for (int i = 0; i < nEle; ++i) {
+      dataColSetNoneAt(pCol, i, false);
+    }
+  } else {
+    setNullN(pCol->pData, pCol->type, pCol->bytes, nEle);
+    pCol->len = TYPE_BYTES[pCol->type] * nEle;
+  }
+#ifdef TD_SUPPORT_BITMAP
+  tdSetBitmapValTypeN(pCol->pBitmap, nEle, TD_VTYPE_NONE);
+#endif
+}
 
 #if 0
 void trbSetRowInfo(SRowBuilder *pRB, bool del, uint16_t sver) {
@@ -32,4 +115,364 @@ int trbWriteCol(SRowBuilder *pRB, void *pData, col_id_t cid) {
   // TODO
   return 0;
 }
-#endif
+
\ No newline at end of file
+#endif
+
+STSRow* tdRowDup(STSRow *row) {
+  STSRow* trow = malloc(TD_ROW_LEN(row));
+  if (trow == NULL) return NULL;
+
+  tdRowCpy(trow, row);
+  return trow;
+}
+
+int tdAppendValToDataCol(SDataCol *pCol, TDRowValT valType, const void *val, int numOfRows, int maxPoints) {
+  TASSERT(pCol != NULL);
+
+  // Assume that the columns not specified during insert/upsert mean None.
+  if (isAllRowsNone(pCol)) {
+    if (tdValIsNone(valType)) {
+      // all None value yet, just return
+      return 0;
+    }
+
+    if (tdAllocMemForCol(pCol, maxPoints) < 0) return -1;
+    if (numOfRows > 0) {
+      // Find the first not None value, fill all previous values as None
+      dataColSetNEleNone(pCol, numOfRows);
+    }
+  }
+  if (!tdValTypeIsNorm(valType)) {
+    // TODO:
+    // 1. back compatibility and easy to debug with codes of 2.0 to save NULL values.
+    // 2. later on, considering further optimization, don't save Null/None for VarType.
+    val = getNullValue(pCol->type);
+  }
+  if (IS_VAR_DATA_TYPE(pCol->type)) {
+    // set offset
+    pCol->dataOff[numOfRows] = pCol->len;
+    // Copy data
+    memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, varDataTLen(val));
+    // Update the length
+    pCol->len += varDataTLen(val);
+  } else {
+    ASSERT(pCol->len == TYPE_BYTES[pCol->type] * numOfRows);
+    memcpy(POINTER_SHIFT(pCol->pData, pCol->len), val, pCol->bytes);
+    pCol->len += pCol->bytes;
+  }
+#ifdef TD_SUPPORT_BITMAP
+  tdSetBitmapValType(pCol->pBitmap, numOfRows, valType);
+#endif
+  return 0;
+}
+
+// internal
+static int32_t tdAppendTpRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) {
+  ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow));
+
+  int   rcol = 1;
+  int   dcol = 1;
+  void *pBitmap = tdGetBitmapAddrTp(pRow, pSchema->flen);
+
+  SDataCol *pDataCol = &(pCols->cols[0]);
+  if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+    tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts,  pCols->numOfRows, pCols->maxPoints);
+  }
+
+  while (dcol < pCols->numOfCols) {
+    pDataCol = &(pCols->cols[dcol]);
+    if (rcol >= schemaNCols(pSchema)) {
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints);
+      ++dcol;
+      continue;
+    }
+
+    STColumn *pRowCol = schemaColAt(pSchema, rcol);
+    SCellVal  sVal = {0};
+    if (pRowCol->colId == pDataCol->colId) {
+      if (tdGetTpRowValOfCol(&sVal, pRow, pBitmap, pRowCol->type, pRowCol->offset - sizeof(TSKEY), rcol - 1) < 0) {
+        return terrno;
+      }
+      tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints);
+      ++dcol;
+      ++rcol;
+    } else if (pRowCol->colId < pDataCol->colId) {
+      ++rcol;
+    } else {
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints);
+      ++dcol;
+    }
+  }
+  ++pCols->numOfRows;
+
+  return TSDB_CODE_SUCCESS;
+}
+// internal
+static int32_t tdAppendKvRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols) {
+  ASSERT(pCols->numOfRows == 0 || dataColsKeyLast(pCols) < TD_ROW_KEY(pRow));
+
+  int   rcol = 0;
+  int   dcol = 1;
+  int   tRowCols = tdRowGetNCols(pRow) - 1;  // the primary TS key not included in kvRowColIdx part
+  int   tSchemaCols = schemaNCols(pSchema) - 1;
+  void *pBitmap = tdGetBitmapAddrKv(pRow, tdRowGetNCols(pRow));
+
+  SDataCol *pDataCol = &(pCols->cols[0]);
+  if (pDataCol->colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
+    tdAppendValToDataCol(pDataCol, TD_VTYPE_NORM, &pRow->ts, pCols->numOfRows, pCols->maxPoints);
+  }
+
+  while (dcol < pCols->numOfCols) {
+    pDataCol = &(pCols->cols[dcol]);
+    if (rcol >= tRowCols || rcol >= tSchemaCols) {
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints);
+      ++dcol;
+      continue;
+    }
+
+    SKvRowIdx *pIdx = tdKvRowColIdxAt(pRow, rcol);
+    int16_t    colIdx = -1;
+    if (pIdx) {
+      colIdx = POINTER_DISTANCE(pRow->data, pIdx) / sizeof(SKvRowIdx);
+    }
+    SCellVal sVal = {0};
+    if (pIdx->colId == pDataCol->colId) {
+      if (tdGetKvRowValOfCol(&sVal, pRow, pBitmap, pIdx->offset, colIdx) < 0) {
+        return terrno;
+      }
+      tdAppendValToDataCol(pDataCol, sVal.valType, sVal.val, pCols->numOfRows, pCols->maxPoints);
+      ++dcol;
+      ++rcol;
+    } else if (pIdx->colId < pDataCol->colId) {
+      ++rcol;
+    } else {
+      tdAppendValToDataCol(pDataCol, TD_VTYPE_NULL, NULL, pCols->numOfRows, pCols->maxPoints);
+      ++dcol;
+    }
+  }
+  ++pCols->numOfRows;
+
+  return TSDB_CODE_SUCCESS;
+}
+
+/**
+ * @brief exposed
+ *
+ * @param pRow
+ * @param pSchema
+ * @param pCols
+ * @param forceSetNull
+ */
+int32_t tdAppendSTSRowToDataCol(STSRow *pRow, STSchema *pSchema, SDataCols *pCols, bool forceSetNull) {
+  if (TD_IS_TP_ROW(pRow)) {
+    return tdAppendTpRowToDataCol(pRow, pSchema, pCols);
+  } else if (TD_IS_KV_ROW(pRow)) {
+    return tdAppendKvRowToDataCol(pRow, pSchema, pCols);
+  } else {
+    ASSERT(0);
+  }
+  return TSDB_CODE_SUCCESS;
+}
+
+int tdMergeDataCols(SDataCols *target, SDataCols *source, int rowsToMerge, int *pOffset, bool forceSetNull) {
+  ASSERT(rowsToMerge > 0 && rowsToMerge <= source->numOfRows);
+  ASSERT(target->numOfCols == source->numOfCols);
+  int offset = 0;
+
+  if (pOffset == NULL) {
+    pOffset = &offset;
+  }
+
+  SDataCols *pTarget = NULL;
+
+  if ((target->numOfRows == 0) || (dataColsKeyLast(target) < dataColsKeyAtRow(source, *pOffset))) {  // No overlap
+    ASSERT(target->numOfRows + rowsToMerge <= target->maxPoints);
+    for (int i = 0; i < rowsToMerge; i++) {
+      for (int j = 0; j < source->numOfCols; j++) {
+        if (source->cols[j].len > 0 || target->cols[j].len > 0) {
+          SCellVal sVal = {0};
+          if (tdGetColDataOfRow(&sVal, source->cols + j, i + (*pOffset)) < 0) {
+            TASSERT(0);
+          }
+          tdAppendValToDataCol(target->cols + j, sVal.valType, sVal.val, target->numOfRows, target->maxPoints);
+        }
+      }
+      ++target->numOfRows;
+    }
+    (*pOffset) += rowsToMerge;
+  } else {
+    pTarget = tdDupDataCols(target, true);
+    if (pTarget == NULL) goto _err;
+
+    int iter1 = 0;
+    tdMergeTwoDataCols(target, pTarget, &iter1, pTarget->numOfRows, source, pOffset, source->numOfRows,
+                       pTarget->numOfRows + rowsToMerge, forceSetNull);
+  }
+
+  tdFreeDataCols(pTarget);
+  return 0;
+
+_err:
+  tdFreeDataCols(pTarget);
+  return -1;
+}
+
+// src2 data has more priority than src1
+static void tdMergeTwoDataCols(SDataCols *target, SDataCols *src1, int *iter1, int limit1, SDataCols *src2, int *iter2,
+                               int limit2, int tRows, bool forceSetNull) {
+  tdResetDataCols(target);
+  ASSERT(limit1 <= src1->numOfRows && limit2 <= src2->numOfRows);
+
+  while (target->numOfRows < tRows) {
+    if (*iter1 >= limit1 && *iter2 >= limit2) break;
+
+    TSKEY key1 = (*iter1 >= limit1) ? INT64_MAX : dataColsKeyAt(src1, *iter1);
+    TKEY  tkey1 = (*iter1 >= limit1) ? TKEY_NULL : dataColsTKeyAt(src1, *iter1);
+    TSKEY key2 = (*iter2 >= limit2) ? INT64_MAX : dataColsKeyAt(src2, *iter2);
+    // TKEY  tkey2 = (*iter2 >= limit2) ? TKEY_NULL : dataColsTKeyAt(src2, *iter2);
+
+    ASSERT(tkey1 == TKEY_NULL || (!TKEY_IS_DELETED(tkey1)));
+
+    if (key1 < key2) {
+      for (int i = 0; i < src1->numOfCols; i++) {
+        ASSERT(target->cols[i].type == src1->cols[i].type);
+        if (src1->cols[i].len > 0 || target->cols[i].len > 0) {
+          SCellVal sVal = {0};
+          if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) {
+            TASSERT(0);
+          }
+          tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints);
+        }
+      }
+
+      target->numOfRows++;
+      (*iter1)++;
+    } else if (key1 >= key2) {
+      // if ((key1 > key2) || (key1 == key2 && !TKEY_IS_DELETED(tkey2))) {
+      if ((key1 > key2) || (key1 == key2)) {
+        for (int i = 0; i < src2->numOfCols; i++) {
+          SCellVal sVal = {0};
+          ASSERT(target->cols[i].type == src2->cols[i].type);
+          if (src2->cols[i].len > 0 && !isNull(src2->cols[i].pData, src2->cols[i].type)) {
+            if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) {
+              TASSERT(0);
+            }
+            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints);
+          } else if (!forceSetNull && key1 == key2 && src1->cols[i].len > 0) {
+            if (tdGetColDataOfRow(&sVal, src1->cols + i, *iter1) < 0) {
+              TASSERT(0);
+            }
+            tdAppendValToDataCol(&(target->cols[i]), sVal.valType, sVal.val, target->numOfRows, target->maxPoints);
+          } else if (target->cols[i].len > 0) {
+            dataColSetNullAt(&target->cols[i], target->numOfRows, true);
+          }
+        }
+        target->numOfRows++;
+      }
+
+      (*iter2)++;
+      if (key1 == key2) (*iter1)++;
+    }
+
+    ASSERT(target->numOfRows <= target->maxPoints);
+  }
+}
+
+
+
+STSRow* mergeTwoRows(void *buffer, STSRow* row1, STSRow *row2, STSchema *pSchema1, STSchema *pSchema2) {
+#if 0
+  ASSERT(TD_ROW_KEY(row1) == TD_ROW_KEY(row2));
+  ASSERT(schemaVersion(pSchema1) == TD_ROW_SVER(row1));
+  ASSERT(schemaVersion(pSchema2) == TD_ROW_SVER(row2));
+  ASSERT(schemaVersion(pSchema1) >= schemaVersion(pSchema2));
+#endif
+
+#if 0
+  SArray *stashRow = taosArrayInit(pSchema1->numOfCols, sizeof(SColInfo));
+  if (stashRow == NULL) {
+    return NULL;
+  }
+
+  STSRow  pRow = buffer;
+  STpRow dataRow = memRowDataBody(pRow);
+  memRowSetType(pRow, SMEM_ROW_DATA);
+  dataRowSetVersion(dataRow, schemaVersion(pSchema1));  // use latest schema version
+  dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pSchema1->flen));
+
+  TDRowLenT dataLen = 0, kvLen = TD_MEM_ROW_KV_HEAD_SIZE;
+
+  int32_t  i = 0;  // row1
+  int32_t  j = 0;  // row2
+  int32_t  nCols1 = schemaNCols(pSchema1);
+  int32_t  nCols2 = schemaNCols(pSchema2);
+  SColInfo colInfo = {0};
+  int32_t  kvIdx1 = 0, kvIdx2 = 0;
+
+  while (i < nCols1) {
+    STColumn *pCol = schemaColAt(pSchema1, i);
+    void *    val1 = tdGetMemRowDataOfColEx(row1, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx1);
+    // if val1 != NULL, use val1;
+    if (val1 != NULL && !isNull(val1, pCol->type)) {
+      tdAppendColVal(dataRow, val1, pCol->type, pCol->offset);
+      kvLen += tdGetColAppendLen(SMEM_ROW_KV, val1, pCol->type);
+      setSColInfo(&colInfo, pCol->colId, pCol->type, val1);
+      taosArrayPush(stashRow, &colInfo);
+      ++i;  // next col
+      continue;
+    }
+
+    void *val2 = NULL;
+    while (j < nCols2) {
+      STColumn *tCol = schemaColAt(pSchema2, j);
+      if (tCol->colId < pCol->colId) {
+        ++j;
+        continue;
+      }
+      if (tCol->colId == pCol->colId) {
+        val2 = tdGetMemRowDataOfColEx(row2, tCol->colId, tCol->type, TD_DATA_ROW_HEAD_SIZE + tCol->offset, &kvIdx2);
+      } else if (tCol->colId > pCol->colId) {
+        // set NULL
+      }
+      break;
+    }  // end of while(j<nCols2)
+    if (val2 == NULL) {
+      val2 = (void *)getNullValue(pCol->type);
+    }
+    tdAppendColVal(dataRow, val2, pCol->type, pCol->offset);
+    if (!isNull(val2, pCol->type)) {
+      kvLen += tdGetColAppendLen(SMEM_ROW_KV, val2, pCol->type);
+      setSColInfo(&colInfo, pCol->colId, pCol->type, val2);
+      taosArrayPush(stashRow, &colInfo);
+    }
+
+    ++i;  // next col
+  }
+
+  dataLen = TD_ROW_LEN(pRow);
+
+  if (kvLen < dataLen) {
+    // scan stashRow and generate SKVRow
+    memset(buffer, 0, sizeof(dataLen));
+    STSRow tRow = buffer;
+    memRowSetType(tRow, SMEM_ROW_KV);
+    SKVRow kvRow = (SKVRow)memRowKvBody(tRow);
+    int16_t nKvNCols = (int16_t) taosArrayGetSize(stashRow);
+    kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nKvNCols));
+    kvRowSetNCols(kvRow, nKvNCols);
+    memRowSetKvVersion(tRow, pSchema1->version);
+
+    int32_t toffset = 0;
+    int16_t k;
+    for (k = 0; k < nKvNCols; ++k) {
+      SColInfo *pColInfo = taosArrayGet(stashRow, k);
+      tdAppendKvColVal(kvRow, pColInfo->colVal, true, pColInfo->colId, pColInfo->colType, toffset);
+      toffset += sizeof(SColIdx);
+    }
+    ASSERT(kvLen == TD_ROW_LEN(tRow));
+  }
+  taosArrayDestroy(stashRow);
+  return buffer;
+  #endif
+  return NULL;
+}

source/dnode/vnode/src/inc/tsdbMemTable.h

@@ -58,20 +58,20 @@ int            tsdbMemTableInsert(STsdb *pTsdb, STsdbMemTable *pMemTable, SSubmi
 int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
                           TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
 
-static FORCE_INLINE SMemRow tsdbNextIterRow(SSkipListIterator *pIter) {
+static FORCE_INLINE STSRow *tsdbNextIterRow(SSkipListIterator *pIter) {
   if (pIter == NULL) return NULL;
 
   SSkipListNode *node = tSkipListIterGet(pIter);
   if (node == NULL) return NULL;
 
-  return (SMemRow)SL_GET_NODE_DATA(node);
+  return (STSRow *)SL_GET_NODE_DATA(node);
 }
 
 static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator *pIter) {
-  SMemRow row = tsdbNextIterRow(pIter);
+  STSRow *row = tsdbNextIterRow(pIter);
   if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
 
-  return memRowKey(row);
+  return TD_ROW_KEY(row);
 }
 
 #ifdef __cplusplus

source/dnode/vnode/src/inc/tsdbReadImpl.h

@@ -49,6 +49,23 @@ typedef struct {
   TSKEY   keyLast;
 } SBlock;
 
+typedef struct {
+  int64_t  last : 1;
+  int64_t  offset : 63;
+  int32_t  algorithm : 8;
+  int32_t  numOfRows : 24;
+  uint8_t  reserve0;
+  uint8_t  numOfSubBlocks;
+  int16_t  numOfCols;    // not including timestamp column
+  uint32_t len : 32;     // data block length
+  uint32_t keyLen : 24;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+  uint32_t reserve1 : 8;
+  uint64_t blkVer : 8;
+  uint64_t aggrOffset : 56;
+  TSKEY    keyFirst;
+  TSKEY    keyLast;
+} SBlock_3;
+
 typedef struct {
   int32_t  delimiter;  // For recovery usage
   int32_t  tid;
@@ -58,6 +75,8 @@ typedef struct {
 
 typedef struct {
   int16_t  colId;
+  uint16_t bitmap : 1;  // 0: has bitmap if has NULL/NORM rows, 1: no bitmap if all rows are NORM
+  uint16_t reserve : 15;
   int32_t  len;
   uint32_t type : 8;
   uint32_t offset : 24;

source/dnode/vnode/src/tq/tq.c

@@ -448,19 +448,27 @@ SArray* tqRetrieveDataBlock(STqReadHandle* pHandle) {
     taosArrayPush(pArray, &colInfo);
   }
 
-  SMemRow row;
-  int32_t kvIdx = 0;
+  STSRowIter iter = {0};
+  tdSTSRowIterInit(&iter, pTschema);
+  STSRow* row;
+  // int32_t kvIdx = 0;
   int32_t curRow = 0;
   tInitSubmitBlkIter(pHandle->pBlock, &pHandle->blkIter);
   while ((row = tGetSubmitBlkNext(&pHandle->blkIter)) != NULL) {
+    tdSTSRowIterReset(&iter, row);
     // get all wanted col of that block
     for (int32_t i = 0; i < colNumNeed; i++) {
       SColumnInfoData* pColData = taosArrayGet(pArray, i);
       STColumn*        pCol = schemaColAt(pTschema, i);
       // TODO
       ASSERT(pCol->colId == pColData->info.colId);
-      void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx);
-      memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), val, pCol->bytes);
+      // void* val = tdGetMemRowDataOfColEx(row, pCol->colId, pCol->type, TD_DATA_ROW_HEAD_SIZE + pCol->offset, &kvIdx);
+      SCellVal sVal = {0};
+      if (!tdSTSRowIterNext(&iter, pCol->colId, pCol->type, &sVal)) {
+        // TODO: reach end
+        break;
+      }
+      memcpy(POINTER_SHIFT(pColData->pData, curRow * pCol->bytes), sVal.val, pCol->bytes);
     }
     curRow++;
   }

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -89,7 +89,7 @@ int         tsdbWriteBlockIdx(SDFile *pHeadf, SArray *pIdxA, void **ppBuf);
 
 int tsdbApplyRtnOnFSet(STsdb *pRepo, SDFileSet *pSet, SRtn *pRtn) {
   SDiskID   did;
-  SDFileSet nSet;
+  SDFileSet nSet = {0};
   STsdbFS * pfs = REPO_FS(pRepo);
   int       level;
 
@@ -1162,6 +1162,13 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *
       (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max),
                                                &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex),
                                                &(pBlockCol->numOfNull));
+      if (pBlockCol->numOfNull == 0) {
+        TD_SET_COL_ROWS_NORM(pBlockCol);
+      } else {
+        TD_SET_COL_ROWS_MISC(pBlockCol);
+      }
+    } else {
+      TD_SET_COL_ROWS_MISC(pBlockCol);
     }
     nColsNotAllNull++;
   }
@@ -1174,6 +1181,9 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *
   int32_t  tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull);
   int32_t  lsize = tsize;
   int32_t  keyLen = 0;
+  int32_t  nBitmaps = (int32_t)TD_BITMAP_BYTES(rowsToWrite);
+  int32_t  tBitmaps = 0;
+
   for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
     // All not NULL columns finish
     if (ncol != 0 && tcol >= nColsNotAllNull) break;
@@ -1185,6 +1195,23 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *
 
     int32_t flen;  // final length
     int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite);
+
+#ifdef TD_SUPPORT_BITMAP
+    int32_t tBitmaps = 0;
+    if ((ncol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) {
+      if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+        tBitmaps = nBitmaps;
+        tlen += tBitmaps;
+      } else {
+        tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
+        tlen += tBitmaps * TYPE_BYTES[pDataCol->type];
+      }
+      // move bitmap parts ahead
+      // TODO: put bitmap part to the 1st location(pBitmap points to pData) to avoid the memmove
+      memcpy(POINTER_SHIFT(pDataCol->pData, pDataCol->len), pDataCol->pBitmap, nBitmaps);
+    }
+#endif
+
     void *  tptr;
 
     // Make room
@@ -1201,7 +1228,7 @@ int tsdbWriteBlockImpl(STsdb *pRepo, STable *pTable, SDFile *pDFile, SDataCols *
 
     // Compress or just copy
     if (pCfg->compression) {
-      flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr,
+      flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite + tBitmaps, tptr,
                                                       tlen + COMP_OVERFLOW_BYTES, pCfg->compression, *ppCBuf,
                                                       tlen + COMP_OVERFLOW_BYTES);
     } else {
@@ -1492,11 +1519,11 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
 
   while (true) {
     key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
-    SMemRow row = tsdbNextIterRow(pCommitIter->pIter);
-    if (row == NULL || memRowKey(row) > maxKey) {
+    STSRow *row = tsdbNextIterRow(pCommitIter->pIter);
+    if (row == NULL || TD_ROW_KEY(row) > maxKey) {
       key2 = INT64_MAX;
     } else {
-      key2 = memRowKey(row);
+      key2 = TD_ROW_KEY(row);
     }
 
     if (key1 == INT64_MAX && key2 == INT64_MAX) break;
@@ -1504,19 +1531,22 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
     if (key1 < key2) {
       for (int i = 0; i < pDataCols->numOfCols; i++) {
         // TODO: dataColAppendVal may fail
-        dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
-                         pTarget->maxPoints);
+        SCellVal sVal = {0};
+        if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter) < 0) {
+          TASSERT(0);
+        }
+        tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints);
       }
 
       pTarget->numOfRows++;
       (*iter)++;
     } else if (key1 > key2) {
-      if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) {
-        pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row));
+      if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) {
+        pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row));
         ASSERT(pSchema != NULL);
       }
 
-      tdAppendMemRowToDataCol(row, pSchema, pTarget, true);
+      tdAppendSTSRowToDataCol(row, pSchema, pTarget, true);
 
       tSkipListIterNext(pCommitIter->pIter);
     } else {
@@ -1524,20 +1554,23 @@ static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIt
         // copy disk data
         for (int i = 0; i < pDataCols->numOfCols; i++) {
           // TODO: dataColAppendVal may fail
-          dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
-                           pTarget->maxPoints);
+          SCellVal sVal = {0};
+          if (tdGetColDataOfRow(&sVal, pDataCols->cols + i, *iter) < 0) {
+            TASSERT(0);
+          }
+          tdAppendValToDataCol(pTarget->cols + i, sVal.valType, sVal.val, pTarget->numOfRows, pTarget->maxPoints);
         }
 
         if (update == TD_ROW_DISCARD_UPDATE) pTarget->numOfRows++;
       }
       if (update != TD_ROW_DISCARD_UPDATE) {
         // copy mem data
-        if (pSchema == NULL || schemaVersion(pSchema) != memRowVersion(row)) {
-          pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, memRowVersion(row));
+        if (pSchema == NULL || schemaVersion(pSchema) != TD_ROW_SVER(row)) {
+          pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, TD_ROW_SVER(row));
           ASSERT(pSchema != NULL);
         }
 
-        tdAppendMemRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE);
+        tdAppendSTSRowToDataCol(row, pSchema, pTarget, update == TD_ROW_OVERWRITE_UPDATE);
       }
       (*iter)++;
       tSkipListIterNext(pCommitIter->pIter);

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -98,13 +98,13 @@ static int tsdbEncodeDFileSetArray(void **buf, SArray *pArray) {
 }
 
 static void *tsdbDecodeDFileSetArray(STsdb*pRepo, void *buf, SArray *pArray) {
-  uint64_t  nset;
-  SDFileSet dset;
+  uint64_t  nset = 0;
 
   taosArrayClear(pArray);
 
   buf = taosDecodeFixedU64(buf, &nset);
   for (size_t i = 0; i < nset; i++) {
+    SDFileSet dset = {0};
     buf = tsdbDecodeDFileSet(pRepo, buf, &dset);
     taosArrayPush(pArray, (void *)(&dset));
   }

source/dnode/vnode/src/tsdb/tsdbMain.c

@@ -760,7 +760,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea
   int numColumns;
   int32_t blockIdx;
   SDataStatis* pBlockStatis = NULL;
-  SMemRow      row = NULL;
+  STSRow*      row = NULL;
   // restore last column data with last schema
   
   int err = 0;
@@ -866,7 +866,7 @@ static int tsdbRestoreLastColumns(STsdbRepo *pRepo, STable *pTable, SReadH* pRea
         pDataCol = pReadh->pDCols[0]->cols + 0;
         pCol = schemaColAt(pSchema, 0);
         tdAppendColVal(memRowDataBody(row), tdGetColDataOfRow(pDataCol, rowId), pCol->type, pCol->offset);
-        pLastCol->ts = memRowKey(row);
+        pLastCol->ts = TD_ROW_KEY(row);
 
         pTable->restoreColumnNum += 1;
 

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -22,7 +22,7 @@ static void     tsdbFreeTbData(STbData *pTbData);
 static char *   tsdbGetTsTupleKey(const void *data);
 static int      tsdbTbDataComp(const void *arg1, const void *arg2);
 static char *   tsdbTbDataGetUid(const void *arg);
-static int      tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row);
+static int      tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row);
 
 STsdbMemTable *tsdbNewMemTable(STsdb *pTsdb) {
   STsdbMemTable *pMemTable = (STsdbMemTable *)calloc(1, sizeof(*pMemTable));
@@ -124,7 +124,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
   TSKEY      fKey = 0;
   bool       isRowDel = false;
   int        filterIter = 0;
-  SMemRow    row = NULL;
+  STSRow *   row = NULL;
   SMergeInfo mInfo;
 
   if (pMergeInfo == NULL) pMergeInfo = &mInfo;
@@ -135,12 +135,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
   if (pCols) tdResetDataCols(pCols);
 
   row = tsdbNextIterRow(pIter);
-  if (row == NULL || memRowKey(row) > maxKey) {
+  if (row == NULL || TD_ROW_KEY(row) > maxKey) {
     rowKey = INT64_MAX;
     isRowDel = false;
   } else {
-    rowKey = memRowKey(row);
-    isRowDel = memRowDeleted(row);
+    rowKey = TD_ROW_KEY(row);
+    isRowDel = TD_ROW_IS_DELETED(row);
   }
 
   if (filterIter >= nFilterKeys) {
@@ -177,12 +177,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
 
       tSkipListIterNext(pIter);
       row = tsdbNextIterRow(pIter);
-      if (row == NULL || memRowKey(row) > maxKey) {
+      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
         rowKey = INT64_MAX;
         isRowDel = false;
       } else {
-        rowKey = memRowKey(row);
-        isRowDel = memRowDeleted(row);
+        rowKey = TD_ROW_KEY(row);
+        isRowDel = TD_ROW_IS_DELETED(row);
       }
     } else {
       if (isRowDel) {
@@ -207,12 +207,12 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
 
       tSkipListIterNext(pIter);
       row = tsdbNextIterRow(pIter);
-      if (row == NULL || memRowKey(row) > maxKey) {
+      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
         rowKey = INT64_MAX;
         isRowDel = false;
       } else {
-        rowKey = memRowKey(row);
-        isRowDel = memRowDeleted(row);
+        rowKey = TD_ROW_KEY(row);
+        isRowDel = TD_ROW_IS_DELETED(row);
       }
 
       filterIter++;
@@ -233,7 +233,7 @@ static int tsdbScanAndConvertSubmitMsg(STsdb *pTsdb, SSubmitMsg *pMsg) {
   SSubmitMsgIter msgIter = {0};
   SSubmitBlk *   pBlock = NULL;
   SSubmitBlkIter blkIter = {0};
-  SMemRow        row = NULL;
+  STSRow *       row = NULL;
   TSKEY          now = taosGetTimestamp(pTsdb->config.precision);
   TSKEY          minKey = now - tsTickPerDay[pTsdb->config.precision] * pTsdb->config.keep;
   TSKEY          maxKey = now + tsTickPerDay[pTsdb->config.precision] * pTsdb->config.daysPerFile;
@@ -306,7 +306,7 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
   STsdbMemTable *pMemTable = pTsdb->mem;
   void *         tptr;
   STbData *      pTbData;
-  SMemRow        row;
+  STSRow *       row;
   TSKEY          keyMin;
   TSKEY          keyMax;
 
@@ -332,12 +332,12 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
 
   tInitSubmitBlkIter(pBlock, &blkIter);
   if (blkIter.row == NULL) return 0;
-  keyMin = memRowKey(blkIter.row);
+  keyMin = TD_ROW_KEY(blkIter.row);
 
   tSkipListPutBatchByIter(pTbData->pData, &blkIter, (iter_next_fn_t)tGetSubmitBlkNext);
 
   // Set statistics
-  keyMax = memRowKey(blkIter.row);
+  keyMax = TD_ROW_KEY(blkIter.row);
 
   pTbData->nrows += pBlock->numOfRows;
   if (pTbData->keyMin > keyMin) pTbData->keyMin = keyMin;
@@ -347,7 +347,7 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
   if (pMemTable->keyMin > keyMin) pMemTable->keyMin = keyMin;
   if (pMemTable->keyMax < keyMax) pMemTable->keyMax = keyMax;
 
-  // SMemRow lastRow = NULL;
+  // STSRow* lastRow = NULL;
   // int64_t osize = SL_SIZE(pTableData->pData);
   // tsdbSetupSkipListHookFns(pTableData->pData, pRepo, pTable, &points, &lastRow);
   // tSkipListPutBatchByIter(pTableData->pData, &blkIter, (iter_next_fn_t)tsdbGetSubmitBlkNext);
@@ -355,7 +355,7 @@ static int tsdbMemTableInsertTbData(STsdb *pTsdb, SSubmitBlk *pBlock, int32_t *p
   // (*pAffectedRows) += points;
 
   // if(lastRow != NULL) {
-  //   TSKEY lastRowKey = memRowKey(lastRow);
+  //   TSKEY lastRowKey = TD_ROW_KEY(lastRow);
   //   if (pMemTable->keyFirst > firstRowKey) pMemTable->keyFirst = firstRowKey;
   //   pMemTable->numOfRows += dsize;
 
@@ -418,7 +418,7 @@ static void tsdbFreeTbData(STbData *pTbData) {
   }
 }
 
-static char *tsdbGetTsTupleKey(const void *data) { return memRowKeys((SMemRow)data); }
+static char *tsdbGetTsTupleKey(const void *data) { return (char *)TD_ROW_KEY_ADDR((STSRow *)data); }
 
 static int tsdbTbDataComp(const void *arg1, const void *arg2) {
   STbData *pTbData1 = (STbData *)arg1;
@@ -437,17 +437,17 @@ static char *tsdbTbDataGetUid(const void *arg) {
   STbData *pTbData = (STbData *)arg;
   return (char *)(&(pTbData->uid));
 }
-static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row) {
+static int tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row) {
   if (pCols) {
-    if (*ppSchema == NULL || schemaVersion(*ppSchema) != memRowVersion(row)) {
-      *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, memRowVersion(row));
+    if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
+      *ppSchema = tsdbGetTableSchemaImpl(pTable, false, false, TD_ROW_SVER(row));
       if (*ppSchema == NULL) {
         ASSERT(false);
         return -1;
       }
     }
 
-    tdAppendMemRowToDataCol(row, *ppSchema, pCols, true);
+    tdAppendSTSRowToDataCol(row, *ppSchema, pCols, true);
   }
 
   return 0;
@@ -479,7 +479,7 @@ int tsdbInsertDataToMemTable(STsdbMemTable *pMemTable, SSubmitMsg *pMsg) {
 typedef struct {
   int32_t  totalLen;
   int32_t  len;
-  SMemRow  row;
+  STSRow*  row;
 } SSubmitBlkIter;
 
 typedef struct {
@@ -493,17 +493,17 @@ static void         tsdbFreeMemTable(SMemTable *pMemTable);
 static STableData*  tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
 static void         tsdbFreeTableData(STableData *pTableData);
 static int          tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables);
-static int          tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, SMemRow row);
+static int          tsdbAppendTableRowToCols(STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow* row);
 static int          tsdbInitSubmitBlkIter(SSubmitBlk *pBlock, SSubmitBlkIter *pIter);
-static SMemRow      tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
+static STSRow*      tsdbGetSubmitBlkNext(SSubmitBlkIter *pIter);
 static int          tsdbScanAndConvertSubmitMsg(STsdbRepo *pRepo, SSubmitMsg *pMsg);
 static int          tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *affectedrows);
 static int          tsdbInitSubmitMsgIter(SSubmitMsg *pMsg, SSubmitMsgIter *pIter);
 static int          tsdbGetSubmitMsgNext(SSubmitMsgIter *pIter, SSubmitBlk **pPBlock);
 static int          tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pTable);
-static int          tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row);
+static int          tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, STSRow* row);
 
-static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey,
+static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, STSRow* row, TSKEY minKey, TSKEY maxKey,
                                           TSKEY now);
 
 
@@ -685,7 +685,7 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
   TSKEY      fKey = 0;
   bool       isRowDel = false;
   int        filterIter = 0;
-  SMemRow    row = NULL;
+  STSRow*    row = NULL;
   SMergeInfo mInfo;
 
   if (pMergeInfo == NULL) pMergeInfo = &mInfo;
@@ -696,11 +696,11 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
   if (pCols) tdResetDataCols(pCols);
 
   row = tsdbNextIterRow(pIter);
-  if (row == NULL || memRowKey(row) > maxKey) {
+  if (row == NULL || TD_ROW_KEY(row) > maxKey) {
     rowKey = INT64_MAX;
     isRowDel = false;
   } else {
-    rowKey = memRowKey(row);
+    rowKey = TD_ROW_KEY(row);
     isRowDel = memRowDeleted(row);
   }
 
@@ -738,11 +738,11 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
 
       tSkipListIterNext(pIter);
       row = tsdbNextIterRow(pIter);
-      if (row == NULL || memRowKey(row) > maxKey) {
+      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
         rowKey = INT64_MAX;
         isRowDel = false;
       } else {
-        rowKey = memRowKey(row);
+        rowKey = TD_ROW_KEY(row);
         isRowDel = memRowDeleted(row);
       }
     } else {
@@ -768,11 +768,11 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
 
       tSkipListIterNext(pIter);
       row = tsdbNextIterRow(pIter);
-      if (row == NULL || memRowKey(row) > maxKey) {
+      if (row == NULL || TD_ROW_KEY(row) > maxKey) {
         rowKey = INT64_MAX;
         isRowDel = false;
       } else {
-        rowKey = memRowKey(row);
+        rowKey = TD_ROW_KEY(row);
         isRowDel = memRowDeleted(row);
       }
 
@@ -790,9 +790,9 @@ int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey
 
 // ---------------- LOCAL FUNCTIONS ----------------
 
-static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMemRow row, TSKEY minKey, TSKEY maxKey,
+static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, STSRow* row, TSKEY minKey, TSKEY maxKey,
                                           TSKEY now) {
-  TSKEY rowKey = memRowKey(row);
+  TSKEY rowKey = TD_ROW_KEY(row);
   if (rowKey < minKey || rowKey > maxKey) {
     tsdbError("vgId:%d table %s tid %d uid %" PRIu64 " timestamp is out of range! now %" PRId64 " minKey %" PRId64
               " maxKey %" PRId64 " row key %" PRId64,
@@ -807,9 +807,9 @@ static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SMem
 
 
 //row1 has higher priority
-static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRepo,
+static STSRow* tsdbInsertDupKeyMerge(STSRow* row1, STSRow* row2, STsdbRepo* pRepo,
                                      STSchema **ppSchema1, STSchema **ppSchema2,
-                                     STable* pTable, int32_t* pPoints, SMemRow* pLastRow) {
+                                     STable* pTable, int32_t* pPoints, STSRow** pLastRow) {
 
   //for compatiblity, duplicate key inserted when update=0 should be also calculated as affected rows!
   if(row1 == NULL && row2 == NULL && pRepo->config.update == TD_ROW_DISCARD_UPDATE) {
@@ -819,10 +819,10 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep
 
   tsdbTrace("vgId:%d a row is %s table %s tid %d uid %" PRIu64 " key %" PRIu64, REPO_ID(pRepo),
             "updated in", TABLE_CHAR_NAME(pTable), TABLE_TID(pTable), TABLE_UID(pTable),
-            memRowKey(row1));
+            TD_ROW_KEY(row1));
 
   if(row2 == NULL || pRepo->config.update != TD_ROW_PARTIAL_UPDATE) {
-    void* pMem = tsdbAllocBytes(pRepo, memRowTLen(row1));
+    void* pMem = tsdbAllocBytes(pRepo, TD_ROW_LEN(row1));
     if(pMem == NULL) return NULL;
     memRowCpy(pMem, row1);
     (*pPoints)++;
@@ -853,9 +853,9 @@ static SMemRow tsdbInsertDupKeyMerge(SMemRow row1, SMemRow row2, STsdbRepo* pRep
     }
   }
 
-  SMemRow tmp = tsdbMergeTwoRows(pBuf, row1, row2, pSchema1, pSchema2);
+  STSRow* tmp = tsdbMergeTwoRows(pBuf, row1, row2, pSchema1, pSchema2);
 
-  void* pMem = tsdbAllocBytes(pRepo, memRowTLen(tmp));
+  void* pMem = tsdbAllocBytes(pRepo, TD_ROW_LEN(tmp));
   if(pMem == NULL) return NULL;
   memRowCpy(pMem, tmp);
 
@@ -868,7 +868,7 @@ static void* tsdbInsertDupKeyMergePacked(void** args) {
   return tsdbInsertDupKeyMerge(args[0], args[1], args[2], (STSchema**)&args[3], (STSchema**)&args[4], args[5], args[6], args[7]);
 }
 
-static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pPoints, SMemRow* pLastRow) {
+static void tsdbSetupSkipListHookFns(SSkipList* pSkipList, STsdbRepo *pRepo, STable *pTable, int32_t* pPoints, STSRow** pLastRow) {
 
   if(pSkipList->insertHandleFn == NULL) {
     tGenericSavedFunc *dupHandleSavedFunc = genericSavedFuncInit((GenericVaFunc)&tsdbInsertDupKeyMergePacked, 9);
@@ -953,7 +953,7 @@ static int tsdbCheckTableSchema(STsdbRepo *pRepo, SSubmitBlk *pBlock, STable *pT
   return 0;
 }
 
-static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow row) {
+static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, STSRow* row) {
   tsdbDebug("vgId:%d updateTableLatestColumn, %s row version:%d", REPO_ID(pRepo), pTable->name->data,
             memRowVersion(row));
 
@@ -1002,30 +1002,30 @@ static void updateTableLatestColumn(STsdbRepo *pRepo, STable *pTable, SMemRow ro
     assert(pTCol->bytes >= bytes);
     memcpy(pDataCol->pData, value, bytes);
     //tsdbInfo("updateTableLatestColumn vgId:%d cache column %d for %d,%s", REPO_ID(pRepo), j, pDataCol->bytes, (char*)pDataCol->pData);
-    pDataCol->ts = memRowKey(row);
+    pDataCol->ts = TD_ROW_KEY(row);
     // unlock
     TSDB_WUNLOCK_TABLE(pTable); 
   }
 }
 
-static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow row) {
+static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, STSRow* row) {
   STsdbCfg *pCfg = &pRepo->config;
 
   // if cacheLastRow config has been reset, free the lastRow
   if (!pCfg->cacheLastRow && pTable->lastRow != NULL) {
-    SMemRow cachedLastRow = pTable->lastRow;
+    STSRow* cachedLastRow = pTable->lastRow;
     TSDB_WLOCK_TABLE(pTable);
     pTable->lastRow = NULL;
     TSDB_WUNLOCK_TABLE(pTable);
     taosTZfree(cachedLastRow);
   }
 
-  if (tsdbGetTableLastKeyImpl(pTable) <= memRowKey(row)) {
+  if (tsdbGetTableLastKeyImpl(pTable) <= TD_ROW_KEY(row)) {
     if (CACHE_LAST_ROW(pCfg) || pTable->lastRow != NULL) {
-      SMemRow nrow = pTable->lastRow;
-      if (taosTSizeof(nrow) < memRowTLen(row)) {
-        SMemRow orow = nrow;
-        nrow = taosTMalloc(memRowTLen(row));
+      STSRow* nrow = pTable->lastRow;
+      if (taosTSizeof(nrow) < TD_ROW_LEN(row)) {
+        STSRow* orow = nrow;
+        nrow = taosTMalloc(TD_ROW_LEN(row));
         if (nrow == NULL) {
           terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
           return -1;
@@ -1033,18 +1033,18 @@ static int tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable, SMemRow r
 
         memRowCpy(nrow, row);
         TSDB_WLOCK_TABLE(pTable);
-        pTable->lastKey = memRowKey(row);
+        pTable->lastKey = TD_ROW_KEY(row);
         pTable->lastRow = nrow;
         TSDB_WUNLOCK_TABLE(pTable);
         taosTZfree(orow);
       } else {
         TSDB_WLOCK_TABLE(pTable);
-        pTable->lastKey = memRowKey(row);
+        pTable->lastKey = TD_ROW_KEY(row);
         memRowCpy(nrow, row);
         TSDB_WUNLOCK_TABLE(pTable);
       }
     } else {
-      pTable->lastKey = memRowKey(row);
+      pTable->lastKey = TD_ROW_KEY(row);
     }
 
     if (CACHE_LAST_NULL_COLUMN(pCfg)) {

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -157,7 +157,7 @@ typedef struct STableGroupSupporter {
 static STimeWindow updateLastrowForEachGroup(STableGroupInfo *groupList);
 static int32_t checkForCachedLastRow(STsdbReadHandle* pTsdbReadHandle, STableGroupInfo *groupList);
 static int32_t checkForCachedLast(STsdbReadHandle* pTsdbReadHandle);
-//static int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey);
+// static int32_t tsdbGetCachedLastRow(STable* pTable, STSRow** pRes, TSKEY* lastKey);
 
 static void    changeQueryHandleForInterpQuery(tsdbReaderT pHandle);
 static void    doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock);
@@ -657,7 +657,7 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
   STbData** pMem = NULL;
   STbData** pIMem = NULL;
 
-  TKEY tLastKey = 0;  /// keyToTkey(pCheckInfo->lastKey);
+  TSKEY tLastKey = 0;  /// keyToTkey(pCheckInfo->lastKey);
   if (pHandle->pTsdb->mem != NULL) {
     pMem = taosHashGet(pHandle->pTsdb->mem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId));
     if (pMem != NULL) {
@@ -689,8 +689,8 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     assert(node != NULL);
 
-    SMemRow row = (SMemRow)SL_GET_NODE_DATA(node);
-    TSKEY   key = memRowKey(row);  // first timestamp in buffer
+    STSRow* row = (STSRow*)SL_GET_NODE_DATA(node);
+    TSKEY   key = TD_ROW_KEY(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
               "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %s",
               pHandle, pCheckInfo->tableId, key, order, (*pMem)->keyMin, (*pMem)->keyMax, pCheckInfo->lastKey, (*pMem)->nrows, pHandle->idStr);
@@ -709,8 +709,8 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     assert(node != NULL);
 
-    SMemRow row = (SMemRow)SL_GET_NODE_DATA(node);
-    TSKEY   key = memRowKey(row);  // first timestamp in buffer
+    STSRow* row = (STSRow*)SL_GET_NODE_DATA(node);
+    TSKEY   key = TD_ROW_KEY(row);  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
               "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%"PRId64", %s",
               pHandle, pCheckInfo->tableId, key, order, (*pIMem)->keyMin, (*pIMem)->keyMax, pCheckInfo->lastKey, (*pIMem)->nrows, pHandle->idStr);
@@ -733,18 +733,18 @@ static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
 }
 
 static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update) {
-  SMemRow rmem = NULL, rimem = NULL;
+  STSRow *rmem = NULL, *rimem = NULL;
   if (pCheckInfo->iter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     if (node != NULL) {
-      rmem = (SMemRow)SL_GET_NODE_DATA(node);
+      rmem = (STSRow*)SL_GET_NODE_DATA(node);
     }
   }
 
   if (pCheckInfo->iiter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     if (node != NULL) {
-      rimem = (SMemRow)SL_GET_NODE_DATA(node);
+      rimem = (STSRow*)SL_GET_NODE_DATA(node);
     }
   }
 
@@ -754,16 +754,16 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
 
   if (rmem != NULL && rimem == NULL) {
     pCheckInfo->chosen = CHECKINFO_CHOSEN_MEM;
-    return memRowKey(rmem);
+    return TD_ROW_KEY(rmem);
   }
 
   if (rmem == NULL && rimem != NULL) {
     pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
-    return memRowKey(rimem);
+    return TD_ROW_KEY(rimem);
   }
 
-  TSKEY r1 = memRowKey(rmem);
-  TSKEY r2 = memRowKey(rimem);
+  TSKEY r1 = TD_ROW_KEY(rmem);
+  TSKEY r2 = TD_ROW_KEY(rimem);
 
   if (r1 == r2) {
     if(update == TD_ROW_DISCARD_UPDATE){
@@ -787,19 +787,19 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
   }
 }
 
-static SMemRow getSMemRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, SMemRow* extraRow) {
-  SMemRow rmem = NULL, rimem = NULL;
+static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow) {
+  STSRow *rmem = NULL, *rimem = NULL;
   if (pCheckInfo->iter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
     if (node != NULL) {
-      rmem = (SMemRow)SL_GET_NODE_DATA(node);
+      rmem = (STSRow*)SL_GET_NODE_DATA(node);
     }
   }
 
   if (pCheckInfo->iiter) {
     SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
     if (node != NULL) {
-      rimem = (SMemRow)SL_GET_NODE_DATA(node);
+      rimem = (STSRow*)SL_GET_NODE_DATA(node);
     }
   }
 
@@ -817,8 +817,8 @@ static SMemRow getSMemRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order,
     return rimem;
   }
 
-  TSKEY r1 = memRowKey(rmem);
-  TSKEY r2 = memRowKey(rimem);
+  TSKEY r1 = TD_ROW_KEY(rmem);
+  TSKEY r2 = TD_ROW_KEY(rimem);
 
   if (r1 == r2) {
     if (update == TD_ROW_DISCARD_UPDATE) {
@@ -831,7 +831,7 @@ static SMemRow getSMemRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order,
       return rmem;
     } else {
       pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
-      extraRow = rimem;
+      *extraRow = rimem;
       return rmem;
     }
   } else {
@@ -904,12 +904,12 @@ static bool hasMoreDataInCache(STsdbReadHandle* pHandle) {
     initTableMemIterator(pHandle, pCheckInfo);
   }
 
-  SMemRow row = getSMemRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL);
+  STSRow* row = getSRowInTableMem(pCheckInfo, pHandle->order, pCfg->update, NULL);
   if (row == NULL) {
     return false;
   }
 
-  pCheckInfo->lastKey = memRowKey(row);  // first timestamp in buffer
+  pCheckInfo->lastKey = TD_ROW_KEY(row);  // first timestamp in buffer
   tsdbDebug("%p uid:%" PRId64", check data in buffer from skey:%" PRId64 ", order:%d, %s", pHandle,
       pCheckInfo->tableId,  pCheckInfo->lastKey, pHandle->order, pHandle->idStr);
 
@@ -1418,8 +1418,11 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
 
         // todo refactor, only copy one-by-one
         for (int32_t k = start; k < num + start; ++k) {
-          const char* p = tdGetColDataOfRow(src, k);
-          memcpy(dst, p, varDataTLen(p));
+          SCellVal    sVal = {0};
+          if(tdGetColDataOfRow(&sVal, src, k) < 0){
+            TASSERT(0);
+          }
+          memcpy(dst, sVal.val, varDataTLen(sVal.val));
           dst += bytes;
         }
       }
@@ -1470,44 +1473,45 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t
 }
 
 // Note: row1 always has high priority
-static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows,
-                               SMemRow row1, SMemRow row2, int32_t numOfCols, uint64_t uid,
-                               STSchema* pSchema1, STSchema* pSchema2, bool forceSetNull) {
-  char* pData = NULL;
-  STSchema* pSchema;
-  SMemRow row;
-  int16_t colId;
-  int16_t offset;
-
-  bool isRow1DataRow = isDataRow(row1);
+static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacity, int32_t numOfRows, STSRow* row1,
+                               STSRow* row2, int32_t numOfCols, uint64_t uid, STSchema* pSchema1, STSchema* pSchema2,
+                               bool forceSetNull) {
+#if 1
+  char*       pData = NULL;
+  STSchema*   pSchema;
+  STSRow*     row;
+  int16_t     colId;
+  int16_t     offset;
+
+  bool isRow1DataRow = TD_IS_TP_ROW(row1);
   bool isRow2DataRow;
   bool isChosenRowDataRow;
   int32_t chosen_itr;
-  void *value;
+  SCellVal sVal = {0};
 
-  // the schema version info is embeded in SDataRow
+  // the schema version info is embeded in STSRow
   int32_t numOfColsOfRow1 = 0;
 
   if (pSchema1 == NULL) {
-    pSchema1 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0);
+    pSchema1 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, TD_ROW_SVER(row1));
   }
 
   if(isRow1DataRow) {
     numOfColsOfRow1 = schemaNCols(pSchema1);
   } else {
-    numOfColsOfRow1 = kvRowNCols(memRowKvBody(row1));
+    numOfColsOfRow1 = tdRowGetNCols(row1);
   }
 
   int32_t numOfColsOfRow2 = 0;
   if(row2) {
-    isRow2DataRow = isDataRow(row2);
+    isRow2DataRow = TD_IS_TP_ROW(row2);
     if (pSchema2 == NULL) {
-      pSchema2 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, 0);
+      pSchema2 = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, uid, TD_ROW_SVER(row2));
     }
     if(isRow2DataRow) {
       numOfColsOfRow2 = schemaNCols(pSchema2);
     } else {
-      numOfColsOfRow2 = kvRowNCols(memRowKvBody(row2));
+      numOfColsOfRow2 = tdRowGetNCols(row2);
     }
   }
 
@@ -1528,8 +1532,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
     } else if(isRow1DataRow) {
       colIdOfRow1 = pSchema1->columns[j].colId;
     } else {
-      void *rowBody = memRowKvBody(row1);
-      SColIdx *pColIdx = kvRowColIdxAt(rowBody, j);
+      SKvRowIdx *pColIdx = tdKvRowColIdxAt(row1, j);
       colIdOfRow1 = pColIdx->colId;
     }
 
@@ -1539,8 +1542,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
     } else if(isRow2DataRow) {
       colIdOfRow2 = pSchema2->columns[k].colId;
     } else {
-      void *rowBody = memRowKvBody(row2);
-      SColIdx *pColIdx = kvRowColIdxAt(rowBody, k);
+      SKvRowIdx *pColIdx = tdKvRowColIdxAt(row2, k);
       colIdOfRow2 = pColIdx->colId;
     }
 
@@ -1576,57 +1578,57 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
     if(isChosenRowDataRow) {
       colId = pSchema->columns[chosen_itr].colId;
       offset = pSchema->columns[chosen_itr].offset;
-      void *rowBody = memRowDataBody(row);
-      value = tdGetRowDataOfCol(rowBody, (int8_t)pColInfo->info.type, TD_DATA_ROW_HEAD_SIZE + offset);
+      tdSTpRowGetVal(row, colId, pSchema->columns[chosen_itr].type, pSchema->flen, offset, chosen_itr, &sVal);
     } else {
-      void *rowBody = memRowKvBody(row);
-      SColIdx *pColIdx = kvRowColIdxAt(rowBody, chosen_itr);
+      SKvRowIdx *pColIdx = tdKvRowColIdxAt(row, chosen_itr);
       colId = pColIdx->colId;
       offset = pColIdx->offset;
-      value = tdGetKvRowDataOfCol(rowBody, pColIdx->offset);
+      tdSKvRowGetVal(row, colId, offset, chosen_itr, &sVal);
     }
 
 
     if (colId == pColInfo->info.colId) {
-      if(forceSetNull || (!isNull(value, (int8_t)pColInfo->info.type))) {
+      if (tdValTypeIsNorm(sVal.valType)) {
         switch (pColInfo->info.type) {
           case TSDB_DATA_TYPE_BINARY:
           case TSDB_DATA_TYPE_NCHAR:
-            memcpy(pData, value, varDataTLen(value));
+            memcpy(pData, sVal.val, varDataTLen(sVal.val));
             break;
           case TSDB_DATA_TYPE_NULL:
           case TSDB_DATA_TYPE_BOOL:
           case TSDB_DATA_TYPE_TINYINT:
           case TSDB_DATA_TYPE_UTINYINT:
-            *(uint8_t *)pData = *(uint8_t *)value;
+            *(uint8_t *)pData = *(uint8_t *)sVal.val;
             break;
           case TSDB_DATA_TYPE_SMALLINT:
           case TSDB_DATA_TYPE_USMALLINT:
-            *(uint16_t *)pData = *(uint16_t *)value;
+            *(uint16_t *)pData = *(uint16_t *)sVal.val;
             break;
           case TSDB_DATA_TYPE_INT:
           case TSDB_DATA_TYPE_UINT:
-            *(uint32_t *)pData = *(uint32_t *)value;
+            *(uint32_t *)pData = *(uint32_t *)sVal.val;
             break;
           case TSDB_DATA_TYPE_BIGINT:
           case TSDB_DATA_TYPE_UBIGINT:
-            *(uint64_t *)pData = *(uint64_t *)value;
+            *(uint64_t *)pData = *(uint64_t *)sVal.val;
             break;
           case TSDB_DATA_TYPE_FLOAT:
-            SET_FLOAT_PTR(pData, value);
+            SET_FLOAT_PTR(pData, sVal.val);
             break;
           case TSDB_DATA_TYPE_DOUBLE:
-            SET_DOUBLE_PTR(pData, value);
+            SET_DOUBLE_PTR(pData, sVal.val);
             break;
           case TSDB_DATA_TYPE_TIMESTAMP:
-            if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) {
-              *(TSKEY *)pData = tdGetKey(*(TKEY *)value);
-            } else {
-              *(TSKEY *)pData = *(TSKEY *)value;
-            }
+            *(TSKEY*)pData = *(TSKEY*)sVal.val;
             break;
           default:
-            memcpy(pData, value, pColInfo->info.bytes);
+            memcpy(pData, sVal.val, pColInfo->info.bytes);
+        }
+      } else if (forceSetNull) {
+        if (pColInfo->info.type == TSDB_DATA_TYPE_BINARY || pColInfo->info.type == TSDB_DATA_TYPE_NCHAR) {
+          setVardataNull(pData, pColInfo->info.type);
+        } else {
+          setNull(pData, pColInfo->info.type, pColInfo->info.bytes);
         }
       }
       i++;
@@ -1666,6 +1668,7 @@ static void mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capacit
       i++;
     }
   }
+#endif
 }
 
 static void moveDataToFront(STsdbReadHandle* pTsdbReadHandle, int32_t numOfRows, int32_t numOfCols) {
@@ -1848,13 +1851,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
   } else if (pCheckInfo->iter != NULL || pCheckInfo->iiter != NULL) {
     SSkipListNode* node = NULL;
     do {
-      SMemRow row2 = NULL;
-      SMemRow row1 = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2);
+      STSRow* row2 = NULL;
+      STSRow* row1 = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, &row2);
       if (row1 == NULL) {
         break;
       }
 
-      TSKEY key = memRowKey(row1);
+      TSKEY key = TD_ROW_KEY(row1);
       if ((key > pTsdbReadHandle->window.ekey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
           (key < pTsdbReadHandle->window.ekey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         break;
@@ -1867,13 +1870,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
 
       if ((key < tsArray[pos] && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
           (key > tsArray[pos] && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
-        if (rv1 != memRowVersion(row1)) {
+        if (rv1 != TD_ROW_SVER(row1)) {
 //          pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
-          rv1 = memRowVersion(row1);
+          rv1 = TD_ROW_SVER(row1);
         }
-        if(row2 && rv2 != memRowVersion(row2)) {
+        if(row2 && rv2 != TD_ROW_SVER(row2)) {
 //          pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2));
-          rv2 = memRowVersion(row2);
+          rv2 = TD_ROW_SVER(row2);
         }
         
         mergeTwoRowFromMem(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, row1, row2, numOfCols, pCheckInfo->tableId, pSchema1, pSchema2, true);
@@ -1892,13 +1895,13 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
           if(pCfg->update == TD_ROW_PARTIAL_UPDATE) {
             doCopyRowsFromFileBlock(pTsdbReadHandle, pTsdbReadHandle->outputCapacity, numOfRows, pos, pos);
           }
-          if (rv1 != memRowVersion(row1)) {
+          if (rv1 != TD_ROW_SVER(row1)) {
 //            pSchema1 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row1));
-            rv1 = memRowVersion(row1);
+            rv1 = TD_ROW_SVER(row1);
           }
-          if(row2 && rv2 != memRowVersion(row2)) {
+          if(row2 && rv2 != TD_ROW_SVER(row2)) {
 //            pSchema2 = tsdbGetTableSchemaByVersion(pTable, memRowVersion(row2));
-            rv2 = memRowVersion(row2);
+            rv2 = TD_ROW_SVER(row2);
           }
           
           bool forceSetNull = pCfg->update != TD_ROW_PARTIAL_UPDATE;
@@ -1951,9 +1954,9 @@ static void doMergeTwoLevelData(STsdbReadHandle* pTsdbReadHandle, STableCheckInf
        * copy them all to result buffer, since it may be overlapped with file data block.
        */
       if (node == NULL ||
-          ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) &&
+          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) > pTsdbReadHandle->window.ekey) &&
            ASCENDING_TRAVERSE(pTsdbReadHandle->order)) ||
-          ((memRowKey((SMemRow)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) &&
+          ((TD_ROW_KEY((STSRow*)SL_GET_NODE_DATA(node)) < pTsdbReadHandle->window.ekey) &&
            !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
         // no data in cache or data in cache is greater than the ekey of time window, load data from file block
         if (cur->win.skey == TSKEY_INITIAL_VAL) {
@@ -2538,12 +2541,12 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
   STSchema* pSchema = NULL;
 
   do {
-    SMemRow row = getSMemRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL);
+    STSRow* row = getSRowInTableMem(pCheckInfo, pTsdbReadHandle->order, pCfg->update, NULL);
     if (row == NULL) {
       break;
     }
 
-    TSKEY key = memRowKey(row);
+    TSKEY key = TD_ROW_KEY(row);
     if ((key > maxKey && ASCENDING_TRAVERSE(pTsdbReadHandle->order)) || (key < maxKey && !ASCENDING_TRAVERSE(pTsdbReadHandle->order))) {
       tsdbDebug("%p key:%"PRIu64" beyond qrange:%"PRId64" - %"PRId64", no more data in buffer", pTsdbReadHandle, key, pTsdbReadHandle->window.skey,
                 pTsdbReadHandle->window.ekey);
@@ -2556,9 +2559,9 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int
     }
 
     win->ekey = key;
-    if (rv != memRowVersion(row)) {
+    if (rv != TD_ROW_SVER(row)) {
       pSchema = metaGetTbTSchema(pTsdbReadHandle->pTsdb->pMeta, pCheckInfo->tableId, 0);
-      rv = memRowVersion(row);
+      rv = TD_ROW_SVER(row);
     }
     mergeTwoRowFromMem(pTsdbReadHandle, maxRowsToRead, numOfRows, row, NULL, numOfCols, pCheckInfo->tableId, pSchema, NULL, true);
 
@@ -2681,7 +2684,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) {
 
   SQueryFilePos* cur = &pTsdbReadHandle->cur;
 
-  SMemRow  pRow = NULL;
+  STSRow*  pRow = NULL;
   TSKEY    key  = TSKEY_INITIAL_VAL;
   int32_t  step = ASCENDING_TRAVERSE(pTsdbReadHandle->order)? 1:-1;
 
@@ -3090,7 +3093,7 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) {
  * if lastRow == NULL, return TSDB_CODE_TDB_NO_CACHE_LAST_ROW
  * else set pRes and return TSDB_CODE_SUCCESS and save lastKey
  */
-//int32_t tsdbGetCachedLastRow(STable* pTable, SMemRow* pRes, TSKEY* lastKey) {
+// int32_t tsdbGetCachedLastRow(STable* pTable, STSRow** pRes, TSKEY* lastKey) {
 //  int32_t code = TSDB_CODE_SUCCESS;
 //
 //  TSDB_RLOCK_TABLE(pTable);
@@ -3107,7 +3110,7 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) {
 //    }
 //  }
 //
-//out:
+// out:
 //  TSDB_RUNLOCK_TABLE(pTable);
 //  return code;
 //}

source/dnode/vnode/src/tsdb/tsdbReadImpl.c

@@ -21,7 +21,8 @@ static void tsdbResetReadTable(SReadH *pReadh);
 static void tsdbResetReadFile(SReadH *pReadh);
 static int  tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols);
 static int  tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
-                                         int maxPoints, char *buffer, int bufferSize);
+                                         int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer,
+                                         int bufferSize);
 static int  tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, const int16_t *colIds,
                                       int numOfColIds);
 static int  tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol);
@@ -463,6 +464,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
   // Recover the data
   int ccol = 0;  // loop iter for SBlockCol object
   int dcol = 0;  // loop iter for SDataCols object
+  int nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows);
+  SBlockCol *pBlockCol = NULL;
   while (dcol < pDataCols->numOfCols) {
     SDataCol *pDataCol = &(pDataCols->cols[dcol]);
     if (dcol != 0 && ccol >= pBlockData->numOfCols) {
@@ -477,12 +480,26 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
     int32_t  tlen = pBlock->keyLen;
 
     if (dcol != 0) {
-      SBlockCol *pBlockCol = &(pBlockData->cols[ccol]);
+      pBlockCol = &(pBlockData->cols[ccol]);
       tcolId = pBlockCol->colId;
       toffset = tsdbGetBlockColOffset(pBlockCol);
       tlen = pBlockCol->len;
+      pDataCol->bitmap = pBlockCol->bitmap;
     } else {
       ASSERT(pDataCol->colId == tcolId);
+      TD_SET_COL_ROWS_NORM(pDataCol);
+    }
+
+    int32_t tBitmaps = 0;
+    int32_t tLenBitmap = 0;
+    if ((dcol != 0) && !TD_COL_ROWS_NORM(pBlockCol)) {
+      if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+        tBitmaps = nBitmaps;
+        tLenBitmap = tBitmaps;
+      } else {
+        tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
+        tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type];
+      }
     }
 
     if (tcolId == pDataCol->colId) {
@@ -492,7 +509,7 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
       }
 
       if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm,
-                                       pBlock->numOfRows, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
+                                       pBlock->numOfRows, tBitmaps, tLenBitmap, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
                                        (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
         tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %u",
                   TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
@@ -516,7 +533,8 @@ static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDat
 }
 
 static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
-                                        int maxPoints, char *buffer, int bufferSize) {
+                                        int numOfBitmaps, int lenOfBitmaps, int maxPoints, char *buffer,
+                                        int bufferSize) {
   if (!taosCheckChecksumWhole((uint8_t *)content, len)) {
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
     return -1;
@@ -527,8 +545,9 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
   // Decode the data
   if (comp) {
     // Need to decompress
-    int tlen = (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData,
-                                                          pDataCol->spaceSize, comp, buffer, bufferSize);
+    int tlen =
+        (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows + numOfBitmaps,
+                                                   pDataCol->pData, pDataCol->spaceSize, comp, buffer, bufferSize);
     if (tlen <= 0) {
       tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d",
                 len, comp, numOfRows, maxPoints, bufferSize);
@@ -542,9 +561,22 @@ static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32
     memcpy(pDataCol->pData, content, pDataCol->len);
   }
 
-  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+  if (lenOfBitmaps > 0) {
+    pDataCol->len -= lenOfBitmaps;
+
+    void *pSrcBitmap = NULL;
+    if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+      pSrcBitmap = dataColSetOffset(pDataCol, numOfRows);
+    } else {
+      pSrcBitmap = POINTER_SHIFT(pDataCol->pData, numOfRows * TYPE_BYTES[pDataCol->type]);
+    }
+    void *pDestBitmap = POINTER_SHIFT(pDataCol->pData, pDataCol->bytes * maxPoints);
+    // restore the bitmap parts
+    memcpy(pDestBitmap, pSrcBitmap, lenOfBitmaps);
+  } else if (IS_VAR_DATA_TYPE(pDataCol->type)) {
     dataColSetOffset(pDataCol, numOfRows);
   }
+
   return 0;
 }
 
@@ -590,6 +622,7 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
 
     if (colId == PRIMARYKEY_TIMESTAMP_COL_ID) {  // load the key row
       blockCol.colId = colId;
+      TD_SET_COL_ROWS_NORM(&blockCol);  // default is NORM for the primary key column
       blockCol.len = pBlock->keyLen;
       blockCol.type = pDataCol->type;
       blockCol.offset = TSDB_KEY_COL_OFFSET;
@@ -617,6 +650,8 @@ static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *
       }
 
       ASSERT(pBlockCol->colId == pDataCol->colId);
+      // set the bitmap
+      pDataCol->bitmap = pBlockCol->bitmap;
     }
 
     if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1;
@@ -630,7 +665,22 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
 
   STsdb *   pRepo = TSDB_READ_REPO(pReadh);
   STsdbCfg *pCfg = REPO_CFG(pRepo);
-  int       tsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES;
+
+  int     nBitmaps = (int)TD_BITMAP_BYTES(pBlock->numOfRows);
+  int32_t tBitmaps = 0;
+  int32_t tLenBitmap = 0;
+
+  if (!TD_COL_ROWS_NORM(pBlockCol)) {
+    if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+      tBitmaps = nBitmaps;
+      tLenBitmap = tBitmaps;
+    } else {
+      tBitmaps = (int32_t)ceil((double)nBitmaps / TYPE_BYTES[pDataCol->type]);
+      tLenBitmap = tBitmaps * TYPE_BYTES[pDataCol->type];
+    }
+  }
+
+  int       tsize = pDataCol->bytes * pBlock->numOfRows + tLenBitmap + COMP_OVERFLOW_BYTES;
 
   if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
   if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
@@ -658,7 +708,8 @@ static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBloc
   }
 
   if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows,
-                                   pCfg->maxRowsPerFileBlock, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) {
+                                   tBitmaps, tLenBitmap, pCfg->maxRowsPerFileBlock, pReadh->pCBuf,
+                                   (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) {
     tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
               pBlockCol->colId, offset);
     return -1;

source/libs/parser/inc/dataBlockMgt.h

@@ -34,11 +34,6 @@ typedef enum EValStat {
   VAL_STAT_NONE = 0x01,  // 1 means no val
 } EValStat;
 
-typedef enum ERowCompareStat {
-  ROW_COMPARE_NO_NEED = 0,
-  ROW_COMPARE_NEED = 1,
-} ERowCompareStat;
-
 typedef struct SBoundColumn {
   int32_t offset;   // all column offset value
   int32_t toffset;  // first part offset for SDataRow TODO: get offset from STSchema on future
@@ -55,24 +50,18 @@ typedef struct SParsedDataColInfo {
   int16_t        numOfCols;
   int16_t        numOfBound;
   uint16_t       flen;        // TODO: get from STSchema
-  uint16_t       allNullLen;  // TODO: get from STSchema
+  uint16_t       allNullLen;  // TODO: get from STSchema(base on SDataRow)
   uint16_t       extendedVarLen;
-  int32_t        *boundedColumns;  // bound column idx according to schema
-  SBoundColumn   *cols;
-  SBoundIdxInfo  *colIdxInfo;
+  uint16_t       boundNullLen;    // bound column len with all NULL value(without VarDataOffsetT/SColIdx part)
+  int32_t *      boundedColumns;  // bound column idx according to schema
+  SBoundColumn * cols;
+  SBoundIdxInfo *colIdxInfo;
   int8_t         orderStatus;  // bound columns
 } SParsedDataColInfo;
 
-typedef struct SMemRowInfo {
-  int32_t dataLen;  // len of SDataRow
-  int32_t kvLen;    // len of SKVRow
-} SMemRowInfo;
-
 typedef struct {
-  uint8_t      memRowType;   // default is 0, that is SDataRow 
-  uint8_t      compareStat;  // 0 no need, 1 need compare
-  TDRowTLenT   kvRowInitLen;
-  SMemRowInfo *rowInfo;
+  uint8_t memRowType;  // default is 0, that is SDataRow
+  int32_t rowSize;
 } SMemRowBuilder;
 
 typedef struct STableDataBlocks {
@@ -91,68 +80,41 @@ typedef struct STableDataBlocks {
   STagData    tagData; 
   
   SParsedDataColInfo boundColumnInfo;
-  SMemRowBuilder rowBuilder;
+  SRowBuilder        rowBuilder;
 } STableDataBlocks;
 
-static FORCE_INLINE void initSMemRow(SMemRow row, uint8_t memRowType, STableDataBlocks *pBlock, int16_t nBoundCols) {
-  memRowSetType(row, memRowType);
-  if (isDataRowT(memRowType)) {
-    dataRowSetVersion(memRowDataBody(row), pBlock->pTableMeta->sversion);
-    dataRowSetLen(memRowDataBody(row), (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + pBlock->boundColumnInfo.flen));
-  } else {
-    ASSERT(nBoundCols > 0);
-    memRowSetKvVersion(row, pBlock->pTableMeta->sversion);
-    kvRowSetNCols(memRowKvBody(row), nBoundCols);
-    kvRowSetLen(memRowKvBody(row), (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols));
-  }
-}
-
 static FORCE_INLINE int32_t getExtendedRowSize(STableDataBlocks *pBlock) {
-  ASSERT(pBlock->rowSize == pBlock->pTableMeta->tableInfo.rowSize);
-  return pBlock->rowSize + TD_MEM_ROW_DATA_HEAD_SIZE + pBlock->boundColumnInfo.extendedVarLen;
-}
-
-// Applicable to consume by one row
-static FORCE_INLINE void appendMemRowColValEx(SMemRow row, const void *value, bool isCopyVarData, int16_t colId,
-                                                 int8_t colType, int32_t toffset, int32_t *dataLen, int32_t *kvLen,
-                                                 uint8_t compareStat) {
-  tdAppendMemRowColVal(row, value, isCopyVarData, colId, colType, toffset);
-  if (compareStat == ROW_COMPARE_NEED) {
-    tdGetColAppendDeltaLen(value, colType, dataLen, kvLen);
-  }
+  STableComInfo *pTableInfo = &pBlock->pTableMeta->tableInfo;
+  ASSERT(pBlock->rowSize == pTableInfo->rowSize);
+  return pBlock->rowSize + TD_ROW_HEAD_LEN - sizeof(TSKEY) + pBlock->boundColumnInfo.extendedVarLen +
+         (int32_t)TD_BITMAP_BYTES(pTableInfo->numOfColumns - 1);
 }
 
-static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t memRowType, SParsedDataColInfo *spd,
-                                                int32_t idx, int32_t *toffset) {
+static FORCE_INLINE void getMemRowAppendInfo(SSchema *pSchema, uint8_t rowType, SParsedDataColInfo *spd,
+                                                int32_t idx, int32_t *toffset, int32_t *colIdx) {
   int32_t schemaIdx = 0;
   if (IS_DATA_COL_ORDERED(spd)) {
     schemaIdx = spd->boundedColumns[idx] - 1;
-    if (isDataRowT(memRowType)) {
+    if (TD_IS_TP_ROW_T(rowType)) {
       *toffset = (spd->cols + schemaIdx)->toffset;  // the offset of firstPart
+      *colIdx = schemaIdx;
     } else {
       *toffset = idx * sizeof(SColIdx);  // the offset of SColIdx
+      *colIdx = idx;
     }
   } else {
     ASSERT(idx == (spd->colIdxInfo + idx)->boundIdx);
     schemaIdx = (spd->colIdxInfo + idx)->schemaColIdx;
-    if (isDataRowT(memRowType)) {
+    if (TD_IS_TP_ROW_T(rowType)) {
       *toffset = (spd->cols + schemaIdx)->toffset;
+      *colIdx = schemaIdx;
     } else {
       *toffset = ((spd->colIdxInfo + idx)->finalIdx) * sizeof(SColIdx);
+      *colIdx = (spd->colIdxInfo + idx)->finalIdx;
     }
   }
 }
 
-static FORCE_INLINE void convertMemRow(SMemRow row, int32_t dataLen, int32_t kvLen) {
-  if (isDataRow(row)) {
-    if (kvLen < (dataLen * KVRatioConvert)) {
-      memRowSetConvert(row);
-    }
-  } else if (kvLen > dataLen) {
-    memRowSetConvert(row);
-  }
-}
-
 static FORCE_INLINE int32_t setBlockInfo(SSubmitBlk *pBlocks, const STableMeta *pTableMeta, int32_t numOfRows) {
   pBlocks->tid = pTableMeta->suid;
   pBlocks->uid = pTableMeta->uid;
@@ -172,8 +134,7 @@ void setBoundColumnInfo(SParsedDataColInfo* pColList, SSchema* pSchema, int32_t
 void destroyBoundColumnInfo(SParsedDataColInfo* pColList);
 void destroyBlockArrayList(SArray* pDataBlockList);
 void destroyBlockHashmap(SHashObj* pDataBlockHash);
-
-int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen);
+int  initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo);
 int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t * numOfRows);
 int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize,
     const STableMeta* pTableMeta, STableDataBlocks** dataBlocks, SArray* pBlockList);

source/libs/parser/src/dataBlockMgt.c

@@ -174,81 +174,18 @@ int32_t getDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int3
 }
 
 static int32_t getRowExpandSize(STableMeta* pTableMeta) {
-  int32_t  result = TD_MEM_ROW_DATA_HEAD_SIZE;
+  int32_t  result = TD_ROW_HEAD_LEN - sizeof(TSKEY);
   int32_t  columns = getNumOfColumns(pTableMeta);
   SSchema* pSchema = getTableColumnSchema(pTableMeta);
-  for (int32_t i = 0; i < columns; i++) {
+  for (int32_t i = 0; i < columns; ++i) {
     if (IS_VAR_DATA_TYPE((pSchema + i)->type)) {
       result += TYPE_BYTES[TSDB_DATA_TYPE_BINARY];
     }
   }
+  result += (int32_t)TD_BITMAP_BYTES(columns - 1);
   return result;
 }
 
-/**
- * TODO: Move to tdataformat.h and refactor when STSchema available.
- *    - fetch flen and toffset from STSChema and remove param spd
- */
-static FORCE_INLINE void convertToSDataRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, SParsedDataColInfo *spd) {
-  ASSERT(isKvRow(src));
-  SKVRow   kvRow = memRowKvBody(src);
-  SDataRow dataRow = memRowDataBody(dest);
-
-  memRowSetType(dest, SMEM_ROW_DATA);
-  dataRowSetVersion(dataRow, memRowKvVersion(src));
-  dataRowSetLen(dataRow, (TDRowLenT)(TD_DATA_ROW_HEAD_SIZE + spd->flen));
-
-  int32_t kvIdx = 0;
-  for (int i = 0; i < nCols; ++i) {
-    SSchema *schema = pSchema + i;
-    void *   val = tdGetKVRowValOfColEx(kvRow, schema->colId, &kvIdx);
-    tdAppendDataColVal(dataRow, val != NULL ? val : getNullValue(schema->type), true, schema->type,
-                       (spd->cols + i)->toffset);
-  }
-}
-
-// TODO: Move to tdataformat.h and refactor when STSchema available.
-static FORCE_INLINE void convertToSKVRow(SMemRow dest, SMemRow src, SSchema *pSchema, int nCols, int nBoundCols, SParsedDataColInfo *spd) {
-  ASSERT(isDataRow(src));
-
-  SDataRow dataRow = memRowDataBody(src);
-  SKVRow   kvRow = memRowKvBody(dest);
-
-  memRowSetType(dest, SMEM_ROW_KV);
-  memRowSetKvVersion(kvRow, dataRowVersion(dataRow));
-  kvRowSetNCols(kvRow, nBoundCols);
-  kvRowSetLen(kvRow, (TDRowLenT)(TD_KV_ROW_HEAD_SIZE + sizeof(SColIdx) * nBoundCols));
-
-  int32_t toffset = 0, kvOffset = 0;
-  for (int i = 0; i < nCols; ++i) {
-    if ((spd->cols + i)->valStat == VAL_STAT_HAS) {
-      SSchema *schema = pSchema + i;
-      toffset = (spd->cols + i)->toffset;
-      void *val = tdGetRowDataOfCol(dataRow, schema->type, toffset + TD_DATA_ROW_HEAD_SIZE);
-      tdAppendKvColVal(kvRow, val, true, schema->colId, schema->type, kvOffset);
-      kvOffset += sizeof(SColIdx);
-    }
-  }
-}
-
-// TODO: Move to tdataformat.h and refactor when STSchema available.
-static FORCE_INLINE void convertSMemRow(SMemRow dest, SMemRow src, STableDataBlocks *pBlock) {
-  STableMeta *        pTableMeta = pBlock->pTableMeta;
-  STableComInfo       tinfo = getTableInfo(pTableMeta);
-  SSchema *           pSchema = getTableColumnSchema(pTableMeta);
-  SParsedDataColInfo *spd = &pBlock->boundColumnInfo;
-
-  ASSERT(dest != src);
-
-  if (isDataRow(src)) {
-    // TODO: Can we use pBlock -> numOfParam directly?
-    ASSERT(spd->numOfBound > 0);
-    convertToSKVRow(dest, src, pSchema, tinfo.numOfColumns, spd->numOfBound, spd);
-  } else {
-    convertToSDataRow(dest, src, pSchema, tinfo.numOfColumns, spd);
-  }
-}
-
 static void destroyDataBlock(STableDataBlocks* pDataBlock) {
   if (pDataBlock == NULL) {
     return;
@@ -361,7 +298,7 @@ int sortRemoveDataBlockDupRows(STableDataBlocks *dataBuf, SBlockKeyInfo *pBlkKey
   char *          pBlockData = pBlocks->data;
   int             n = 0;
   while (n < nRows) {
-    pBlkKeyTuple->skey = memRowKey(pBlockData);
+    pBlkKeyTuple->skey = TD_ROW_KEY((STSRow *)pBlockData);
     pBlkKeyTuple->payloadAddr = pBlockData;
 
     // next loop
@@ -446,37 +383,32 @@ static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SB
   int32_t numOfRows = pBlock->numOfRows;
 
   if (isRawPayload) {
-    for (int32_t i = 0; i < numOfRows; ++i) {
-      SMemRow memRow = (SMemRow)pDataBlock;
-      memRowSetType(memRow, SMEM_ROW_DATA);
-      SDataRow trow = memRowDataBody(memRow);
-      dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen));
-      dataRowSetVersion(trow, pTableMeta->sversion);
+    SRowBuilder builder = {0};
+    
+    tdSRowInit(&builder, pTableMeta->sversion);
+    tdSRowSetInfo(&builder, getNumOfColumns(pTableMeta), -1, flen);
 
+    for (int32_t i = 0; i < numOfRows; ++i) {
+      tdSRowResetBuf(&builder, pDataBlock);
       int toffset = 0;
-      for (int32_t j = 0; j < tinfo.numOfColumns; j++) {
-        tdAppendColVal(trow, p, pSchema[j].type, toffset);
-        toffset += TYPE_BYTES[pSchema[j].type];
+      for (int32_t j = 0; j < tinfo.numOfColumns; ++j) {
+        int8_t  colType = pSchema[j].type;
+        uint8_t valType = isNull(p, colType) ? TD_VTYPE_NULL : TD_VTYPE_NORM;
+        tdAppendColValToRow(&builder, pSchema[j].colId, colType, valType, p, true, toffset, j);
+        toffset += TYPE_BYTES[colType];
         p += pSchema[j].bytes;
       }
-
-      pDataBlock = (char*)pDataBlock + memRowTLen(memRow);
-      pBlock->dataLen += memRowTLen(memRow);
+      int32_t rowLen = TD_ROW_LEN((STSRow*)pDataBlock);
+      pDataBlock = (char*)pDataBlock + rowLen;
+      pBlock->dataLen += rowLen;
     }
   } else {
     for (int32_t i = 0; i < numOfRows; ++i) {
-      char* payload = (blkKeyTuple + i)->payloadAddr;
-      if (isNeedConvertRow(payload)) {
-        convertSMemRow(pDataBlock, payload, pTableDataBlock);
-        TDRowTLenT rowTLen = memRowTLen(pDataBlock);
-        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
-        pBlock->dataLen += rowTLen;
-      } else {
-        TDRowTLenT rowTLen = memRowTLen(payload);
-        memcpy(pDataBlock, payload, rowTLen);
-        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
-        pBlock->dataLen += rowTLen;
-      }
+      char*      payload = (blkKeyTuple + i)->payloadAddr;
+      TDRowLenT  rowTLen = TD_ROW_LEN((STSRow*)payload);
+      memcpy(pDataBlock, payload, rowTLen);
+      pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
+      pBlock->dataLen += rowTLen;
     }
   }
 
@@ -594,51 +526,10 @@ int32_t allocateMemIfNeed(STableDataBlocks *pDataBlock, int32_t rowSize, int32_t
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t initMemRowBuilder(SMemRowBuilder *pBuilder, uint32_t nRows, uint32_t nCols, uint32_t nBoundCols, int32_t allNullLen) {
-  ASSERT(nRows >= 0 && nCols > 0 && (nBoundCols <= nCols));
-  if (nRows > 0) {
-    // already init(bind multiple rows by single column)
-    if (pBuilder->compareStat == ROW_COMPARE_NEED && (pBuilder->rowInfo != NULL)) {
-      return TSDB_CODE_SUCCESS;
-    }
-  }
-
-  // default compareStat is  ROW_COMPARE_NO_NEED
-  if (nBoundCols == 0) {  // file input
-    pBuilder->memRowType = SMEM_ROW_DATA;
-    return TSDB_CODE_SUCCESS;
-  } else {
-    float boundRatio = ((float)nBoundCols / (float)nCols);
-
-    if (boundRatio < KVRatioKV) {
-      pBuilder->memRowType = SMEM_ROW_KV;
-      return TSDB_CODE_SUCCESS;
-    } else if (boundRatio > KVRatioData) {
-      pBuilder->memRowType = SMEM_ROW_DATA;
-      return TSDB_CODE_SUCCESS;
-    }
-    pBuilder->compareStat = ROW_COMPARE_NEED;
-
-    if (boundRatio < KVRatioPredict) {
-      pBuilder->memRowType = SMEM_ROW_KV;
-    } else {
-      pBuilder->memRowType = SMEM_ROW_DATA;
-    }
-  }
-
-  pBuilder->kvRowInitLen = TD_MEM_ROW_KV_HEAD_SIZE + nBoundCols * sizeof(SColIdx);
-
-  if (nRows > 0) {
-    pBuilder->rowInfo = calloc(nRows, sizeof(SMemRowInfo));
-    if (pBuilder->rowInfo == NULL) {
-      return TSDB_CODE_TSC_OUT_OF_MEMORY;
-    }
-
-    for (int i = 0; i < nRows; ++i) {
-      (pBuilder->rowInfo + i)->dataLen = TD_MEM_ROW_DATA_HEAD_SIZE + allNullLen;
-      (pBuilder->rowInfo + i)->kvLen = pBuilder->kvRowInitLen;
-    }
-  }
-
+int  initRowBuilder(SRowBuilder *pBuilder, int16_t schemaVer, SParsedDataColInfo *pColInfo) {
+  ASSERT(pColInfo->numOfCols > 0 && (pColInfo->numOfBound <= pColInfo->numOfCols));
+  tdSRowInit(pBuilder, schemaVer);
+  tdSRowSetExtendedInfo(pBuilder, pColInfo->numOfCols, pColInfo->numOfBound, pColInfo->flen, pColInfo->allNullLen,
+                        pColInfo->boundNullLen);
   return TSDB_CODE_SUCCESS;
 }

source/libs/parser/src/insertParser.c

@@ -259,31 +259,30 @@ static int parseTime(char **end, SToken *pToken, int16_t timePrec, int64_t *time
 }
 
 typedef struct SMemParam {
-  SMemRow row;
+  SRowBuilder* rb;
   SSchema* schema;
   int32_t toffset;
-  uint8_t compareStat;
-  int32_t dataLen;
-  int32_t kvLen;
+  int32_t      colIdx;
 } SMemParam;
 
-static FORCE_INLINE int32_t MemRowAppend(const void *value, int32_t len, void *param) {
-  SMemParam* pa = (SMemParam*)param;
+static FORCE_INLINE int32_t MemRowAppend(const void* value, int32_t len, void* param) {
+  SMemParam*   pa = (SMemParam*)param;
+  SRowBuilder* rb = pa->rb;
   if (TSDB_DATA_TYPE_BINARY == pa->schema->type) {
-    char *rowEnd = memRowEnd(pa->row);
+    const char* rowEnd = tdRowEnd(rb->pBuf);
     STR_WITH_SIZE_TO_VARSTR(rowEnd, value, len);
-    appendMemRowColValEx(pa->row, rowEnd, true, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat);
+    tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx);
   } else if (TSDB_DATA_TYPE_NCHAR == pa->schema->type) {
     // if the converted output len is over than pColumnModel->bytes, return error: 'Argument list too long'
-    int32_t output = 0;
-    char *  rowEnd = memRowEnd(pa->row);
-    if (!taosMbsToUcs4(value, len, (char *)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
+    int32_t     output = 0;
+    const char* rowEnd = tdRowEnd(rb->pBuf);
+    if (!taosMbsToUcs4(value, len, (char*)varDataVal(rowEnd), pa->schema->bytes - VARSTR_HEADER_SIZE, &output)) {
       return TSDB_CODE_TSC_SQL_SYNTAX_ERROR;
     }
     varDataSetLen(rowEnd, output);
-    appendMemRowColValEx(pa->row, rowEnd, false, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat);
+    tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, rowEnd, false, pa->toffset, pa->colIdx);
   } else {
-    appendMemRowColValEx(pa->row, value, true, pa->schema->colId, pa->schema->type, pa->toffset, &pa->dataLen, &pa->kvLen, pa->compareStat);
+    tdAppendColValToRow(rb, pa->schema->colId, pa->schema->type, TD_VTYPE_NORM, value, true, pa->toffset, pa->colIdx);
   }
   return TSDB_CODE_SUCCESS;
 }
@@ -411,27 +410,27 @@ static int32_t parseUsingClause(SInsertParseContext* pCxt, SToken* pTbnameToken)
   return TSDB_CODE_SUCCESS;
 }
 
-static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, char* tmpTokenBuf) {  
+static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks, int16_t timePrec, int32_t* len, char* tmpTokenBuf) {
   SParsedDataColInfo* spd = &pDataBlocks->boundColumnInfo;
-  SMemRowBuilder* pBuilder = &pDataBlocks->rowBuilder;
-  char *row = pDataBlocks->pData + pDataBlocks->size;  // skip the SSubmitBlk header
-  initSMemRow(row, pBuilder->memRowType, pDataBlocks, spd->numOfBound);
+  SRowBuilder*        pBuilder = &pDataBlocks->rowBuilder;
+  STSRow*             row = (STSRow*)(pDataBlocks->pData + pDataBlocks->size);  // skip the SSubmitBlk header
+
+  tdSRowResetBuf(pBuilder, row);
 
   bool isParseBindParam = false;
   SSchema* schema = getTableColumnSchema(pDataBlocks->pTableMeta);
-  SMemParam param = {.row = row};
+  SMemParam param = {.rb = pBuilder};
   SToken sToken = {0};
   // 1. set the parsed value from sql string
   for (int i = 0; i < spd->numOfBound; ++i) {
     NEXT_TOKEN(pCxt->pSql, sToken);
     SSchema *pSchema = &schema[spd->boundedColumns[i] - 1];
     param.schema = pSchema;
-    param.compareStat = pBuilder->compareStat;
-    getMemRowAppendInfo(schema, pBuilder->memRowType, spd, i, &param.toffset);
+    getMemRowAppendInfo(schema, pBuilder->rowType, spd, i, &param.toffset, &param.colIdx);
     CHECK_CODE(parseValueToken(&pCxt->pSql, &sToken, pSchema, timePrec, tmpTokenBuf, MemRowAppend, &param, &pCxt->msg));
 
     if (PRIMARYKEY_TIMESTAMP_COL_ID == pSchema->colId) {
-      TSKEY tsKey = memRowKey(row);
+      TSKEY tsKey = TD_ROW_KEY(row);
       if (checkTimestamp(pDataBlocks, (const char *)&tsKey) != TSDB_CODE_SUCCESS) {
         buildSyntaxErrMsg(&pCxt->msg, "client time/server time can not be mixed up", sToken.z);
         return TSDB_CODE_TSC_INVALID_TIME_STAMP;
@@ -440,23 +439,18 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks,
   }
 
   if (!isParseBindParam) {
-    // 2. check and set convert flag
-    if (pBuilder->compareStat == ROW_COMPARE_NEED) {
-      convertMemRow(row, spd->allNullLen + TD_MEM_ROW_DATA_HEAD_SIZE, pBuilder->kvRowInitLen);
-    }
-
-    // 3. set the null value for the columns that do not assign values
-    if ((spd->numOfBound < spd->numOfCols) && isDataRow(row) && !isNeedConvertRow(row)) {
-      SDataRow dataRow = memRowDataBody(row);
+    // set the null value for the columns that do not assign values
+    if ((spd->numOfBound < spd->numOfCols) && TD_IS_TP_ROW(row)) {
       for (int32_t i = 0; i < spd->numOfCols; ++i) {
-        if (spd->cols[i].valStat == VAL_STAT_NONE) {
-          tdAppendDataColVal(dataRow, getNullValue(schema[i].type), true, schema[i].type, spd->cols[i].toffset);
+        if (spd->cols[i].valStat == VAL_STAT_NONE) {  // the primary TS key is not VAL_STAT_NONE
+          tdAppendColValToTpRow(pBuilder, TD_VTYPE_NONE, getNullValue(schema[i].type), true, schema[i].type, i,
+                                spd->cols[i].toffset);
         }
       }
     }
   }
 
-  *len = getExtendedRowSize(pDataBlocks);
+  *len = pBuilder->extendedRowSize;
   return TSDB_CODE_SUCCESS;
 }
 
@@ -464,7 +458,7 @@ static int parseOneRow(SInsertParseContext* pCxt, STableDataBlocks* pDataBlocks,
 static int32_t parseValues(SInsertParseContext* pCxt, STableDataBlocks* pDataBlock, int maxRows, int32_t* numOfRows) {
   STableComInfo tinfo = getTableInfo(pDataBlock->pTableMeta);
   int32_t extendedRowSize = getExtendedRowSize(pDataBlock);
-  CHECK_CODE(initMemRowBuilder(&pDataBlock->rowBuilder, 0, tinfo.numOfColumns, pDataBlock->boundColumnInfo.numOfBound, pDataBlock->boundColumnInfo.allNullLen));
+  CHECK_CODE(initRowBuilder(&pDataBlock->rowBuilder, pDataBlock->pTableMeta->sversion, &pDataBlock->boundColumnInfo));
 
   (*numOfRows) = 0;
   char tmpTokenBuf[TSDB_MAX_BYTES_PER_ROW] = {0};  // used for deleting Escape character: \\, \', \"

source/libs/parser/src/parserUtil.c

@@ -402,7 +402,7 @@ static void createInputDataFilterInfo(SQueryStmtInfo* px, int32_t numOfCol1, int
  * @param dataBlocks
  * @return
  */
-//int32_t tscCreateDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, SName* name,
+//  int32_t tscCreateDataBlock(size_t defaultSize, int32_t rowSize, int32_t startOffset, SName* name,
 //                           STableMeta* pTableMeta, STableDataBlocks** dataBlocks) {
 //  STableDataBlocks* dataBuf = (STableDataBlocks*)calloc(1, sizeof(STableDataBlocks));
 //  if (dataBuf == NULL) {
@@ -449,7 +449,7 @@ static void createInputDataFilterInfo(SQueryStmtInfo* px, int32_t numOfCol1, int
 //  return TSDB_CODE_SUCCESS;
 //}
 //
-//int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize,
+//  int32_t tscGetDataBlockFromList(SHashObj* pHashList, int64_t id, int32_t size, int32_t startOffset, int32_t rowSize,
 //                                SName* name, STableMeta* pTableMeta, STableDataBlocks** dataBlocks,
 //                                SArray* pBlockList) {
 //  *dataBlocks = NULL;
@@ -474,7 +474,7 @@ static void createInputDataFilterInfo(SQueryStmtInfo* px, int32_t numOfCol1, int
 //}
 //
 //// Erase the empty space reserved for binary data
-//static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SInsertStatementParam* insertParam,
+//  static int trimDataBlock(void* pDataBlock, STableDataBlocks* pTableDataBlock, SInsertStatementParam* insertParam,
 //                         SBlockKeyTuple* blkKeyTuple) {
 //  // TODO: optimize this function, handle the case while binary is not presented
 //  STableMeta*     pTableMeta = pTableDataBlock->pTableMeta;
@@ -518,7 +518,7 @@ static void createInputDataFilterInfo(SQueryStmtInfo* px, int32_t numOfCol1, int
 //
 //  if (IS_RAW_PAYLOAD(insertParam->payloadType)) {
 //    for (int32_t i = 0; i < numOfRows; ++i) {
-//      SMemRow memRow = (SMemRow)pDataBlock;
+//      STSRow* memRow = (STSRow*)pDataBlock;
 //      memRowSetType(memRow, SMEM_ROW_DATA);
 //      SDataRow trow = memRowDataBody(memRow);
 //      dataRowSetLen(trow, (uint16_t)(TD_DATA_ROW_HEAD_SIZE + flen));
@@ -531,32 +531,25 @@ static void createInputDataFilterInfo(SQueryStmtInfo* px, int32_t numOfCol1, int
 //        p += pSchema[j].bytes;
 //      }
 //
-//      pDataBlock = (char*)pDataBlock + memRowTLen(memRow);
-//      pBlock->dataLen += memRowTLen(memRow);
+//      pDataBlock = (char*)pDataBlock + TD_ROW_LEN(memRow);
+//      pBlock->dataLen += TD_ROW_LEN(memRow);
 //    }
 //  } else {
 //    for (int32_t i = 0; i < numOfRows; ++i) {
-//      char* payload = (blkKeyTuple + i)->payloadAddr;
-//      if (isNeedConvertRow(payload)) {
-//        convertSMemRow(pDataBlock, payload, pTableDataBlock);
-//        TDRowTLenT rowTLen = memRowTLen(pDataBlock);
-//        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
-//        pBlock->dataLen += rowTLen;
-//      } else {
-//        TDRowTLenT rowTLen = memRowTLen(payload);
-//        memcpy(pDataBlock, payload, rowTLen);
-//        pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
-//        pBlock->dataLen += rowTLen;
-//      }
+//       char*      payload = (blkKeyTuple + i)->payloadAddr;
+//       TDRowLenT rowTLen = TD_ROW_LEN(payload);
+//       memcpy(pDataBlock, payload, rowTLen);
+//       pDataBlock = POINTER_SHIFT(pDataBlock, rowTLen);
+//       pBlock->dataLen += rowTLen;
 //    }
 //  }
-//
+
 //  int32_t len = pBlock->dataLen + pBlock->schemaLen;
 //  pBlock->dataLen = htonl(pBlock->dataLen);
 //  pBlock->schemaLen = htonl(pBlock->schemaLen);
-//
+
 //  return len;
-//}
+// }
 
 TAOS_FIELD createField(const SSchema* pSchema) {
   TAOS_FIELD f = { .type = pSchema->type, .bytes = pSchema->bytes, };