refact data

include/common/tdataformat.h

@@ -38,6 +38,8 @@ typedef struct SKVIdx        SKVIdx;
 // STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow);
 int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow);
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
+                  int8_t *flags);
 
 // STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema);
@@ -86,6 +88,7 @@ struct STSRowBuilder {
   uint8_t  *pKVBuf;
   int32_t   szTPBuf;
   uint8_t  *pTPBuf;
+  uint8_t  *pBitBuf;
   int32_t   nCols;
   int32_t   kvVLen;
   int32_t   tpVLen;

include/common/tmsg.h

@@ -271,12 +271,13 @@ typedef struct {
 
 int32_t tEncodeSSubmitRsp(SEncoder* pEncoder, const SSubmitRsp* pRsp);
 int32_t tDecodeSSubmitRsp(SDecoder* pDecoder, SSubmitRsp* pRsp);
-void tFreeSSubmitRsp(SSubmitRsp *pRsp);
-
-#define COL_SMA_ON  ((int8_t)0x1)
-#define COL_IDX_ON  ((int8_t)0x2)
-#define COL_VAL_SET ((int8_t)0x4)
+void    tFreeSSubmitRsp(SSubmitRsp* pRsp);
 
+#define COL_SMA_ON   ((int8_t)0x1)
+#define COL_IDX_ON   ((int8_t)0x2)
+#define COL_SET_VAL  ((int8_t)0x10)
+#define COL_SET_NONE ((int8_t)0x20)
+#define COL_SET_NULL ((int8_t)0x40)
 typedef struct SSchema {
   int8_t   type;
   int8_t   flags;
@@ -285,6 +286,8 @@ typedef struct SSchema {
   char     name[TSDB_COL_NAME_LEN];
 } SSchema;
 
+#define COL_IS_SET(s) ((s)->flags & (COL_SET_VAL | COL_SET_NONE | COL_SET_NULL) != 0)
+
 #define IS_BSMA_ON(s) (((s)->flags & 0x01) == COL_SMA_ON)
 
 #define SSCHMEA_TYPE(s)  ((s)->type)

source/common/src/tdataformat.c

@@ -19,13 +19,17 @@
 #include "tdatablock.h"
 #include "tlog.h"
 
-#define TD_KV_ROW 0x1U
+#define TD_HAS_NONE 0x1U
+#define TD_HAS_NULL 0x2U
+#define TD_HAS_VAL  0x4U
+#define TD_KV_ROW   0x10U
 
 struct SKVIdx {
   int32_t cid;
   int32_t offset;
 };
 
+// STSRow2
 int32_t tEncodeTSRow(SEncoder *pEncoder, const STSRow2 *pRow) {
   if (tEncodeI64(pEncoder, pRow->ts) < 0) return -1;
   if (tEncodeU32v(pEncoder, pRow->flags) < 0) return -1;
@@ -50,6 +54,59 @@ int32_t tDecodeTSRow(SDecoder *pDecoder, STSRow2 *pRow) {
   return 0;
 }
 
+int32_t tTSRowGet(const STSRow2 *pRow, STSchema *pTSchema, int32_t cid, const uint8_t **ppData, uint32_t *nData,
+                  int8_t *flags) {
+  if (cid == 0) {
+    *ppData = (uint8_t *)&pRow->ts;
+    *nData = sizeof(TSKEY);
+    *flags = 0;
+  } else {
+    uint32_t tflags = pRow->flags & 0xf;
+    *ppData = NULL;
+    *nData = 0;
+
+    switch (tflags) {
+      case TD_HAS_NONE:
+        *flags = -1;
+        break;
+      case TD_HAS_NULL:
+        *flags = 1;
+        break;
+      case TD_HAS_VAL:
+        *flags = 0;
+        // find the row
+        break;
+      case TD_HAS_NULL | TD_HAS_NONE:
+        // read bit map (todo)
+        if (0) {
+          *flags = 1;
+        } else {
+          *flags = -1;
+        }
+        break;
+      case TD_HAS_VAL | TD_HAS_NONE:
+      case TD_HAS_VAL | TD_HAS_NULL:
+        // read bitmap (todo)
+        if (0) {
+          if (tflags & TD_HAS_NONE) {
+            *flags = -1;
+          } else {
+            *flags = 1;
+          }
+        } else {
+          // get value (todo)
+        }
+        break;
+      case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
+        break;
+      default:
+        return -1;
+    }
+  }
+  return 0;
+}
+
+// STSchema
 int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) {
   *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols);
   if (*ppTSchema == NULL) {
@@ -87,6 +144,7 @@ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema *
 
 void tTSchemaDestroy(STSchema *pTSchema) { taosMemoryFree(pTSchema); }
 
+// STSRowBuilder
 int32_t tTSRowBuilderInit(STSRowBuilder *pBuilder, int32_t sver, SSchema *pSchema, int32_t nCols) {
   int32_t  kvBufLen;
   int32_t  tpBufLen;
@@ -131,7 +189,7 @@ void tTSRowBuilderReset(STSRowBuilder *pBuilder) {
   for (int32_t iCol = pBuilder->pTSchema->numOfCols - 1; iCol >= 0; iCol--) {
     pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
 
-    pBuilder->pTColumn->flags &= (~COL_VAL_SET);
+    pBuilder->pTColumn->flags &= 0xf;
   }
 
   pBuilder->nCols = 0;
@@ -144,7 +202,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   int32_t  iCol;
   uint8_t *p;
 
-  // search column
+  // search column (todo: make here faster)
   if (pBuilder->pTColumn->colId < cid) {
     iCol = (pBuilder->pTColumn - pBuilder->pTSchema->columns) / sizeof(STColumn) + 1;
     for (; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
@@ -160,7 +218,7 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   }
 
   // check
-  if (pBuilder->pTColumn->colId != cid || pBuilder->pTColumn->flags & COL_VAL_SET) {
+  if (pBuilder->pTColumn->colId != cid || COL_IS_SET(pBuilder->pTColumn)) {
     return -1;
   }
 
@@ -168,9 +226,12 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
   if (cid == 0) {
     ASSERT(pData && nData == sizeof(TSKEY));
     pBuilder->row.ts = *(TSKEY *)pData;
+
+    pBuilder->pTColumn->flags |= COL_SET_VAL;
   } else {
     if (pData) {
-      // ASSERT(!IS_NULL(pData));
+      pBuilder->row.flags |= TD_HAS_VAL;
+      pBuilder->pTColumn->flags |= COL_SET_VAL;
 
       // set tuple data
       p = pBuilder->pTPBuf + pBuilder->pTColumn->offset;
@@ -197,51 +258,79 @@ int32_t tTSRowBuilderPut(STSRowBuilder *pBuilder, int32_t cid, const uint8_t *pD
         pBuilder->kvVLen += nData;
       }
     } else {
-      // set NULL val
+      pBuilder->row.flags |= TD_HAS_NULL;
+      pBuilder->pTColumn->flags |= COL_SET_NULL;
+
+      p = pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols;
+      ((SKVIdx *)p)->cid = cid;
+      ((SKVIdx *)p)->offset = -1;
     }
+
+    pBuilder->nCols++;
   }
 
-  pBuilder->pTColumn->flags |= COL_VAL_SET;
-  pBuilder->nCols++;
   return 0;
 }
 
 int32_t tTSRowBuilderGetRow(STSRowBuilder *pBuilder, const STSRow2 **ppRow) {
-  if ((pBuilder->pTSchema->columns[0].flags & COL_VAL_SET) == 0) {
+  int32_t  tpDataLen, kvDataLen;
+  uint32_t flags;
+
+  // error not set ts
+  if (!COL_IS_SET(pBuilder->pTSchema->columns)) {
     return -1;
   }
 
-  if (pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen < pBuilder->pTSchema->flen + pBuilder->tpVLen) {
-    // encode as TD_KV_ROW
-    pBuilder->row.flags |= TD_KV_ROW;
-    pBuilder->row.ncols = pBuilder->nCols;
-    pBuilder->row.nData = pBuilder->nCols * sizeof(SKVIdx) + pBuilder->kvVLen;
-    pBuilder->row.pData = pBuilder->pKVBuf;
-
-    if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) {
-      memmove(pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->nCols,
-              pBuilder->pKVBuf + sizeof(SKVIdx) * pBuilder->pTSchema->numOfCols, pBuilder->kvVLen);
-    }
-  } else {
-    // encode as TD_TUPLE_ROW
-    pBuilder->row.flags &= (~TD_KV_ROW);
-    pBuilder->row.sver = pBuilder->pTSchema->version;
-    pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen;
-    pBuilder->row.pData = pBuilder->pTPBuf;
-
-    if (pBuilder->nCols < pBuilder->pTSchema->numOfCols) {
-      // set non-set cols as None
-      for (int32_t iCol = 1; iCol < pBuilder->pTSchema->numOfCols; iCol++) {
-        pBuilder->pTColumn = &pBuilder->pTSchema->columns[iCol];
-        if (pBuilder->pTColumn->flags & COL_VAL_SET) continue;
-
-        {
-          // set None (todo)
-        }
-
-        pBuilder->pTColumn->flags |= COL_VAL_SET;
+  if (pBuilder->nCols < pBuilder->pTSchema->numOfCols - 1) {
+    pBuilder->row.flags |= TD_HAS_NONE;
+  }
+
+  flags = pBuilder->row.flags & 0xf;
+  switch (flags) {
+    case TD_HAS_NONE:
+    case TD_HAS_NULL:
+      pBuilder->row.sver = pBuilder->pTSchema->version;
+      pBuilder->row.nData = 0;
+      pBuilder->row.pData = NULL;
+      break;
+    case TD_HAS_VAL:
+      pBuilder->row.sver = pBuilder->pTSchema->version;
+      pBuilder->row.nData = pBuilder->pTSchema->flen + pBuilder->tpVLen;
+      pBuilder->row.pData = pBuilder->pTPBuf;
+      break;
+    case TD_HAS_NULL | TD_HAS_NONE:
+      pBuilder->row.sver = pBuilder->pTSchema->version;
+      // set bitmap (todo)
+      pBuilder->row.nData = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1;
+      pBuilder->row.pData = pBuilder->pBitBuf;
+      break;
+    case TD_HAS_VAL | TD_HAS_NONE:
+    case TD_HAS_VAL | TD_HAS_NULL:
+    case TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE:
+      if (flags == TD_HAS_VAL | TD_HAS_NULL | TD_HAS_NONE) {
+        tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 4) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
+      } else {
+        tpDataLen = ((pBuilder->pTSchema->numOfCols - 1) / 8) + 1 + pBuilder->pTSchema->flen + pBuilder->tpVLen;
       }
-    }
+      kvDataLen = sizeof(SKVIdx) * pBuilder->nCols + pBuilder->kvVLen;
+
+      if (kvDataLen < tpDataLen) {
+        pBuilder->row.flags |= TD_KV_ROW;
+        pBuilder->row.ncols = pBuilder->nCols;
+        pBuilder->row.nData = kvDataLen;
+        pBuilder->row.pData = pBuilder->pKVBuf;
+        // memmove(); todo
+        // qsort
+      } else {
+        pBuilder->row.sver = pBuilder->pTSchema->numOfCols;
+        // set bitmap etc (todo)
+        pBuilder->row.nData = tpDataLen;
+        pBuilder->row.pData = pBuilder->pTPBuf;
+      }
+      break;
+    default:
+      ASSERT(0);
+      return -1;
   }
 
   *ppRow = &pBuilder->row;