feat: tsdb multi-version 1

source/dnode/vnode/src/inc/tsdb.h

@@ -39,35 +39,46 @@ typedef struct SDelOp  SDelOp;
 
 static int tsdbKeyCmprFn(const void *p1, const void *p2);
 
-// tsdbMemTable ================
-typedef struct STbData    STbData;
-typedef struct SMemTable  SMemTable;
-typedef struct SMergeInfo SMergeInfo;
-typedef struct STable     STable;
-
-int  tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
-void tsdbMemTableDestroy(SMemTable *pMemTable);
-int  tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead,
-                           SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
+// tsdbMemTable ==============================================================================================
+typedef struct STbData     STbData;
+typedef struct SMemTable   SMemTable;
+typedef struct STbDataIter STbDataIter;
+typedef struct SMergeInfo  SMergeInfo;
+typedef struct STable      STable;
+
+// SMemTable
+int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
+void    tsdbMemTableDestroy(SMemTable *pMemTable);
+void    tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);
+
+// STbDataIter
+int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter);
+void   *tsdbTbDataIterDestroy(STbDataIter *pIter);
+void    tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter);
+bool    tsdbTbDataIterNext(STbDataIter *pIter);
+bool    tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow);
+
+int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead,
+                          SDataCols *pCols, TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo);
 
 // tsdbMemTable2.c ==============================================================================================
-typedef struct SMemTable2   SMemTable2;
-typedef struct SMemData     SMemData;
-typedef struct SMemDataIter SMemDataIter;
+// typedef struct SMemTable2   SMemTable2;
+// typedef struct SMemData     SMemData;
+// typedef struct SMemDataIter SMemDataIter;
 
-int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable2 **ppMemTable);
-void    tsdbMemTableDestroy2(SMemTable2 *pMemTable);
-int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk);
-int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
+// int32_t tsdbMemTableCreate2(STsdb *pTsdb, SMemTable2 **ppMemTable);
+// void    tsdbMemTableDestroy2(SMemTable2 *pMemTable);
+// int32_t tsdbInsertTableData2(STsdb *pTsdb, int64_t version, SVSubmitBlk *pSubmitBlk);
+// int32_t tsdbDeleteTableData2(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 
-/* SMemDataIter */
-void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter);
-bool tsdbMemDataIterNext(SMemDataIter *pIter);
-void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow);
+// /* SMemDataIter */
+// void tsdbMemDataIterOpen(SMemData *pMemData, TSDBKEY *pKey, int8_t backward, SMemDataIter *pIter);
+// bool tsdbMemDataIterNext(SMemDataIter *pIter);
+// void tsdbMemDataIterGet(SMemDataIter *pIter, TSDBROW **ppRow);
 
-// tsdbCommit2.c ==============================================================================================
-int32_t tsdbBegin2(STsdb *pTsdb);
-int32_t tsdbCommit2(STsdb *pTsdb);
+// // tsdbCommit2.c ==============================================================================================
+// int32_t tsdbBegin2(STsdb *pTsdb);
+// int32_t tsdbCommit2(STsdb *pTsdb);
 
 // tsdbFile.c ==============================================================================================
 typedef int32_t          TSDB_FILE_T;
@@ -211,29 +222,39 @@ struct TSDBKEY {
   TSKEY   ts;
 };
 
+typedef struct SMemSkipListNode SMemSkipListNode;
+struct SMemSkipListNode {
+  int8_t            level;
+  SMemSkipListNode *forwards[0];
+};
+typedef struct SMemSkipList {
+  uint32_t          seed;
+  int64_t           size;
+  int8_t            maxLevel;
+  int8_t            level;
+  SMemSkipListNode *pHead;
+  SMemSkipListNode *pTail;
+} SMemSkipList;
+
 struct STbData {
-  tb_uid_t   suid;
-  tb_uid_t   uid;
-  TSDBKEY    minKey;
-  TSDBKEY    maxKey;
-  SDelOp    *pHead;
-  SDelOp    *pTail;
-  int64_t    nrows;
-  SSkipList *pData;
+  tb_uid_t     suid;
+  tb_uid_t     uid;
+  TSDBKEY      minKey;
+  TSDBKEY      maxKey;
+  SDelOp      *pHead;
+  SDelOp      *pTail;
+  SMemSkipList sl;
 };
 
 struct SMemTable {
-  STsdb     *pTsdb;
-  int32_t    nRef;
-  SRWLatch   latch;
-  TSDBKEY    minKey;
-  TSDBKEY    maxKey;
-  int64_t    nRow;
-  int64_t    nDelOp;
-  SDelOp    *pHead;
-  SDelOp    *pTail;
-  SSkipList *pSlIdx;  // SSkiplist<STbData>
-  SHashObj  *pHashIdx;
+  SRWLatch latch;
+  STsdb   *pTsdb;
+  int32_t  nRef;
+  TSDBKEY  minKey;
+  TSDBKEY  maxKey;
+  int64_t  nRow;
+  int64_t  nDelOp;
+  SArray  *aTbData;  // SArray<STbData>
 };
 
 struct STsdbFSMeta {
@@ -656,7 +677,7 @@ struct SFSIter {
 
 struct TSDBROW {
   int64_t version;
-  STSRow2 tsRow;
+  STSRow *pTSRow;
 };
 
 struct TABLEID {
@@ -709,16 +730,6 @@ static FORCE_INLINE int tsdbKeyCmprFn(const void *p1, const void *p2) {
   return 0;
 }
 
-typedef struct SMemSkipListNode SMemSkipListNode;
-typedef struct SMemSkipList {
-  uint32_t          seed;
-  int32_t           size;
-  int8_t            maxLevel;
-  int8_t            level;
-  SMemSkipListNode *pHead;
-  SMemSkipListNode *pTail;
-} SMemSkipList;
-
 struct SMemData {
   tb_uid_t     suid;
   tb_uid_t     uid;
@@ -730,13 +741,19 @@ struct SMemData {
 };
 
 struct SMemDataIter {
-  SMemData         *pMemData;
+  STbData          *pMemData;
   int8_t            backward;
   TSDBROW          *pRow;
   SMemSkipListNode *pNode;  // current node
   TSDBROW           row;
 };
 
+struct STbDataIter {
+  STbData          *pTbData;
+  int8_t            backward;
+  SMemSkipListNode *pNode;
+};
+
 #endif
 
 #ifdef __cplusplus

source/dnode/vnode/src/inc/vnodeInt.h

@@ -116,7 +116,9 @@ int          tsdbBegin(STsdb* pTsdb);
 int32_t      tsdbCommit(STsdb* pTsdb);
 int          tsdbScanAndConvertSubmitMsg(STsdb* pTsdb, SSubmitReq* pMsg);
 int          tsdbInsertData(STsdb* pTsdb, int64_t version, SSubmitReq* pMsg, SSubmitRsp* pRsp);
-int          tsdbInsertTableData(STsdb* pTsdb, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock, SSubmitBlkRsp* pRsp);
+int32_t      tsdbInsertTableData(STsdb* pTsdb, int64_t version, SSubmitMsgIter* pMsgIter, SSubmitBlk* pBlock,
+                                 SSubmitBlkRsp* pRsp);
+int32_t      tsdbDeleteTableData(STsdb* pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey);
 tsdbReaderT* tsdbQueryTables(SVnode* pVnode, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,
                              uint64_t taskId);
 tsdbReaderT  tsdbQueryCacheLastT(STsdb* tsdb, SQueryTableDataCond* pCond, STableListInfo* tableList, uint64_t qId,

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -58,26 +58,26 @@ typedef struct {
 #define TSDB_COMMIT_DEFAULT_ROWS(ch) TSDB_DEFAULT_BLOCK_ROWS(TSDB_COMMIT_REPO(ch)->pVnode->config.tsdbCfg.maxRows)
 #define TSDB_COMMIT_TXN_VERSION(ch)  FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
 
-static void tsdbStartCommit(STsdb *pRepo);
-static void tsdbEndCommit(STsdb *pTsdb, int eno);
-static int  tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo);
-static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key);
-static int  tsdbNextCommitFid(SCommitH *pCommith);
-static void tsdbDestroyCommitH(SCommitH *pCommith);
-static int  tsdbCreateCommitIters(SCommitH *pCommith);
-static void tsdbDestroyCommitIters(SCommitH *pCommith);
-static int  tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
-static void tsdbResetCommitFile(SCommitH *pCommith);
-static int  tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
-static int  tsdbCommitToTable(SCommitH *pCommith, int tid);
-static bool tsdbCommitIsSameFile(SCommitH *pCommith, int bidx);
-static int  tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx);
-static int  tsdbSetCommitTable(SCommitH *pCommith, STable *pTable);
-static int  tsdbComparKeyBlock(const void *arg1, const void *arg2);
-static int  tsdbWriteBlockInfo(SCommitH *pCommih);
-static int  tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData);
-static int  tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx);
-static int  tsdbMoveBlock(SCommitH *pCommith, int bidx);
+static void    tsdbStartCommit(STsdb *pRepo);
+static void    tsdbEndCommit(STsdb *pTsdb, int eno);
+static int     tsdbInitCommitH(SCommitH *pCommith, STsdb *pRepo);
+static void    tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key);
+static int     tsdbNextCommitFid(SCommitH *pCommith);
+static void    tsdbDestroyCommitH(SCommitH *pCommith);
+static int32_t tsdbCreateCommitIters(SCommitH *pCommith);
+static void    tsdbDestroyCommitIters(SCommitH *pCommith);
+static int     tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
+static void    tsdbResetCommitFile(SCommitH *pCommith);
+static int     tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
+static int     tsdbCommitToTable(SCommitH *pCommith, int tid);
+static bool    tsdbCommitIsSameFile(SCommitH *pCommith, int bidx);
+static int     tsdbMoveBlkIdx(SCommitH *pCommith, SBlockIdx *pIdx);
+static int     tsdbSetCommitTable(SCommitH *pCommith, STable *pTable);
+static int     tsdbComparKeyBlock(const void *arg1, const void *arg2);
+static int     tsdbWriteBlockInfo(SCommitH *pCommih);
+static int     tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData);
+static int     tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx);
+static int     tsdbMoveBlock(SCommitH *pCommith, int bidx);
 static int  tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks);
 static int  tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit,
                                bool isLastOneBlock);
@@ -453,11 +453,32 @@ static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
   return 0;
 }
 
-static int tsdbCreateCommitIters(SCommitH *pCommith) {
-  STsdb             *pRepo = TSDB_COMMIT_REPO(pCommith);
-  SMemTable         *pMem = pRepo->imem;
+static int32_t tsdbCreateCommitIters(SCommitH *pCommith) {
+  int32_t      code = 0;
+  STsdb       *pRepo = TSDB_COMMIT_REPO(pCommith);
+  SMemTable   *pMem = pRepo->imem;
+  SCommitIter *pCommitIter;
+
+  pCommith->niters = taosArrayGetSize(pMem->aTbData);
+  pCommith->iters = (SCommitIter *)taosMemoryCalloc(pCommith->niters, sizeof(SCommitIter));
+  if (pCommith->iters == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+  for (int32_t iIter = 0; iIter < pCommith->niters; iIter++) {
+    pCommitIter = (SCommitIter *)taosArrayGetP(pMem->aTbData, iIter);
+    // TODO
+
+    // pCommitIter->pIter =
+  }
+
+  return code;
+
+_err:
+  return code;
+#if 0
   SSkipListIterator *pSlIter;
-  SCommitIter       *pCommitIter;
   SSkipListNode     *pNode;
   STbData           *pTbData;
   STSchema          *pTSchema = NULL;
@@ -495,8 +516,7 @@ static int tsdbCreateCommitIters(SCommitH *pCommith) {
     }
   }
   tSkipListDestroyIter(pSlIter);
-
-  return 0;
+#endif
 }
 
 static void tsdbDestroyCommitIters(SCommitH *pCommith) {

source/dnode/vnode/src/tsdb/tsdbMemTable.c

@@ -15,67 +15,299 @@
 
 #include "tsdb.h"
 
-static STbData *tsdbNewTbData(tb_uid_t uid);
-static void     tsdbFreeTbData(STbData *pTbData);
-static char    *tsdbGetTsTupleKey(const void *data);
-static int      tsdbTbDataComp(const void *arg1, const void *arg2);
-static char    *tsdbTbDataGetUid(const void *arg);
-static int tsdbAppendTableRowToCols(STsdb *pTsdb, STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row,
-                                    bool merge);
+#define SL_MAX_LEVEL 5
 
-int tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) {
-  SMemTable *pMemTable;
-  SVnode    *pVnode;
+#define SL_NODE_SIZE(l)        (sizeof(SMemSkipListNode) + sizeof(SMemSkipListNode *) * (l)*2)
+#define SL_NODE_FORWARD(n, l)  ((n)->forwards[l])
+#define SL_NODE_BACKWARD(n, l) ((n)->forwards[(n)->level + (l)])
+#define SL_NODE_DATA(n)        (&SL_NODE_BACKWARD(n, (n)->level))
 
-  *ppMemTable = NULL;
-  pVnode = pTsdb->pVnode;
+#define SL_MOVE_BACKWARD 0x1
+#define SL_MOVE_FROM_POS 0x2
+
+static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow);
+static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow);
+static void    tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags);
+static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData);
+static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version,
+                                       SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp);
+
+int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable) {
+  int32_t    code = 0;
+  SMemTable *pMemTable = NULL;
 
-  // alloc handle
   pMemTable = (SMemTable *)taosMemoryCalloc(1, sizeof(*pMemTable));
   if (pMemTable == NULL) {
-    return -1;
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
   }
-
+  taosInitRWLatch(&pMemTable->latch);
   pMemTable->pTsdb = pTsdb;
   pMemTable->nRef = 1;
-  taosInitRWLatch(&pMemTable->latch);
   pMemTable->minKey = (TSDBKEY){.ts = TSKEY_MAX, .version = INT64_MAX};
   pMemTable->maxKey = (TSDBKEY){.ts = TSKEY_MIN, .version = -1};
   pMemTable->nRow = 0;
-  pMemTable->pSlIdx = tSkipListCreate(pVnode->config.tsdbCfg.slLevel, TSDB_DATA_TYPE_BIGINT, sizeof(tb_uid_t),
-                                      tsdbTbDataComp, SL_DISCARD_DUP_KEY, tsdbTbDataGetUid);
-  if (pMemTable->pSlIdx == NULL) {
+  pMemTable->nDelOp = 0;
+  pMemTable->aTbData = taosArrayInit(128, sizeof(STbData *));
+  if (pMemTable->aTbData == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
     taosMemoryFree(pMemTable);
-    return -1;
+    goto _err;
   }
 
-  pMemTable->pHashIdx = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK);
-  if (pMemTable->pHashIdx == NULL) {
-    tSkipListDestroy(pMemTable->pSlIdx);
+  *ppMemTable = pMemTable;
+  return code;
+
+_err:
+  *ppMemTable = NULL;
+  return code;
+}
+
+void tsdbMemTableDestroy(SMemTable *pMemTable) {
+  if (pMemTable) {
+    taosArrayDestroy(pMemTable->aTbData);
     taosMemoryFree(pMemTable);
+  }
+}
+
+static int32_t tbDataPCmprFn(const void *p1, const void *p2) {
+  STbData *pTbData1 = *(STbData **)p1;
+  STbData *pTbData2 = *(STbData **)p2;
+
+  if (pTbData1->suid < pTbData2->suid) {
     return -1;
+  } else if (pTbData1->suid > pTbData2->suid) {
+    return 1;
+  }
+
+  if (pTbData1->uid < pTbData2->uid) {
+    return -1;
+  } else if (pTbData1->uid > pTbData2->uid) {
+    return 1;
   }
 
-  *ppMemTable = pMemTable;
   return 0;
 }
+void tsdbGetTbDataFromMemTable(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) {
+  STbData *pTbData = &(STbData){.suid = suid, .uid = uid};
+  void    *p = taosArraySearch(pMemTable->aTbData, &pTbData, tbDataPCmprFn, TD_EQ);
+  *ppTbData = p ? *(STbData **)p : NULL;
+}
 
-void tsdbMemTableDestroy(SMemTable *pMemTable) {
-  if (pMemTable) {
-    taosHashCleanup(pMemTable->pHashIdx);
-    SSkipListIterator *pIter = tSkipListCreateIter(pMemTable->pSlIdx);
-    SSkipListNode     *pNode = NULL;
-    STbData           *pTbData = NULL;
-    for (;;) {
-      if (!tSkipListIterNext(pIter)) break;
-      pNode = tSkipListIterGet(pIter);
-      pTbData = (STbData *)pNode->pData;
-      tsdbFreeTbData(pTbData);
+int32_t tsdbInsertTableData(STsdb *pTsdb, int64_t version, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock,
+                            SSubmitBlkRsp *pRsp) {
+  int32_t    code = 0;
+  SMemTable *pMemTable = pTsdb->mem;
+  STbData   *pTbData = NULL;
+  tb_uid_t   suid = pMsgIter->suid;
+  tb_uid_t   uid = pMsgIter->uid;
+  int32_t    sverNew;
+
+  // check if table exists (todo: refact)
+  SMetaReader mr = {0};
+  SMetaEntry  me = {0};
+  metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0);
+  if (metaGetTableEntryByUid(&mr, pMsgIter->uid) < 0) {
+    metaReaderClear(&mr);
+    code = TSDB_CODE_PAR_TABLE_NOT_EXIST;
+    goto _err;
+  }
+  if (pRsp->tblFName) strcat(pRsp->tblFName, mr.me.name);
+
+  if (mr.me.type == TSDB_NORMAL_TABLE) {
+    sverNew = mr.me.ntbEntry.schemaRow.version;
+  } else {
+    metaGetTableEntryByUid(&mr, mr.me.ctbEntry.suid);
+    sverNew = mr.me.stbEntry.schemaRow.version;
+  }
+  metaReaderClear(&mr);
+  pRsp->sver = sverNew;
+
+  // create/get STbData to op
+  code = tsdbGetOrCreateTbData(pMemTable, suid, uid, &pTbData);
+  if (code) {
+    goto _err;
+  }
+
+  // do insert impl
+  code = tsdbInsertTableDataImpl(pMemTable, pTbData, version, pMsgIter, pBlock, pRsp);
+  if (code) {
+    goto _err;
+  }
+
+  return code;
+
+_err:
+  return code;
+}
+
+int32_t tsdbDeleteTableData(STsdb *pTsdb, int64_t version, tb_uid_t suid, tb_uid_t uid, TSKEY sKey, TSKEY eKey) {
+  int32_t    code = 0;
+  SMemTable *pMemTable = pTsdb->mem;
+  STbData   *pTbData = NULL;
+  SVBufPool *pPool = pTsdb->pVnode->inUse;
+
+  // check if table exists (todo)
+
+  code = tsdbGetOrCreateTbData(pMemTable, suid, uid, &pTbData);
+  if (code) {
+    goto _err;
+  }
+
+  // do delete
+  SDelOp *pDelOp = (SDelOp *)vnodeBufPoolMalloc(pPool, sizeof(*pDelOp));
+  if (pDelOp == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pDelOp->version = version;
+  pDelOp->sKey = sKey;
+  pDelOp->eKey = eKey;
+  pDelOp->pNext = NULL;
+  if (pTbData->pHead == NULL) {
+    ASSERT(pTbData->pTail == NULL);
+    pTbData->pHead = pTbData->pTail = pDelOp;
+  } else {
+    pTbData->pTail->pNext = pDelOp;
+    pTbData->pTail = pDelOp;
+  }
+
+  // update the state of pMemTable and other (todo)
+
+  pMemTable->nDelOp++;
+
+  tsdbError("vgId:%d delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64
+            " since %s",
+            TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
+  return code;
+
+_err:
+  tsdbError("vgId:%d failed to delete data from table suid:%" PRId64 " uid:%" PRId64 " skey:%" PRId64 " eKey:%" PRId64
+            " since %s",
+            TD_VID(pTsdb->pVnode), suid, uid, sKey, eKey, tstrerror(code));
+  return code;
+}
+
+static int tsdbAppendTableRowToCols(STsdb *pTsdb, STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row,
+                                    bool merge) {
+  if (pCols) {
+    if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
+      *ppSchema = tsdbGetTableSchemaImpl(pTsdb, pTable, false, false, TD_ROW_SVER(row));
+      if (*ppSchema == NULL) {
+        ASSERT(false);
+        return -1;
+      }
+    }
+
+    tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge);
+  }
+
+  return 0;
+}
+
+int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter **ppIter) {
+  int32_t code = 0;
+
+  (*ppIter) = (STbDataIter *)taosMemoryCalloc(1, sizeof(STbDataIter));
+  if ((*ppIter) == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+
+  tsdbTbDataIterOpen(pTbData, pFrom, backward, *ppIter);
+
+_exit:
+  return code;
+}
+
+void *tsdbTbDataIterDestroy(STbDataIter *pIter) {
+  if (pIter) {
+    taosMemoryFree(pIter);
+  }
+
+  return NULL;
+}
+
+void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter) {
+  SMemSkipListNode *pos[SL_MAX_LEVEL];
+
+  pIter->pTbData = pTbData;
+  pIter->backward = backward;
+  if (pFrom == NULL) {
+    // create from head or tail
+    if (backward) {
+      pIter->pNode = SL_NODE_BACKWARD(pTbData->sl.pTail, 0);
+    } else {
+      pIter->pNode = SL_NODE_FORWARD(pTbData->sl.pHead, 0);
+    }
+  } else {
+    // create from a key
+    if (backward) {
+      tbDataMovePosTo(pTbData, pos, pFrom, SL_MOVE_BACKWARD);
+      pIter->pNode = SL_NODE_BACKWARD(pos[0], 0);
+    } else {
+      tbDataMovePosTo(pTbData, pos, pFrom, 0);
+      pIter->pNode = SL_NODE_FORWARD(pos[0], 0);
+    }
+  }
+}
+
+bool tsdbTbDataIterNext(STbDataIter *pIter) {
+  SMemSkipListNode *pHead = pIter->pTbData->sl.pHead;
+  SMemSkipListNode *pTail = pIter->pTbData->sl.pTail;
+
+  if (pIter->backward) {
+    ASSERT(pIter->pNode != pTail);
+
+    if (pIter->pNode == pHead) {
+      return false;
+    }
+
+    pIter->pNode = SL_NODE_BACKWARD(pIter->pNode, 0);
+    if (pIter->pNode == pHead) {
+      return false;
+    }
+  } else {
+    ASSERT(pIter->pNode != pHead);
+
+    if (pIter->pNode == pTail) {
+      return false;
+    }
+
+    pIter->pNode = SL_NODE_FORWARD(pIter->pNode, 0);
+    if (pIter->pNode == pTail) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+bool tsdbTbDataIterGet(STbDataIter *pIter, TSDBROW *pRow) {
+  SMemSkipListNode *pHead = pIter->pTbData->sl.pHead;
+  SMemSkipListNode *pTail = pIter->pTbData->sl.pTail;
+  TSDBROW           row = {0};
+
+  if (pRow == NULL) {
+    pRow = &row;
+  }
+
+  if (pIter->backward) {
+    ASSERT(pIter->pNode != pTail);
+
+    if (pIter->pNode == pHead) {
+      return false;
+    }
+  } else {
+    ASSERT(pIter->pNode != pHead);
+
+    if (pIter->pNode == pTail) {
+      return false;
     }
-    tSkipListDestroyIter(pIter);
-    tSkipListDestroy(pMemTable->pSlIdx);
-    taosMemoryFree(pMemTable);
   }
+
+  tGetTSDBRow((uint8_t *)SL_NODE_DATA(pIter->pNode), pRow);
+  return true;
 }
 
 /**
@@ -238,151 +470,276 @@ int tsdbLoadDataFromCache(STsdb *pTsdb, STable *pTable, SSkipListIterator *pIter
   return 0;
 }
 
-int tsdbInsertTableData(STsdb *pTsdb, SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp) {
-  SSubmitBlkIter blkIter = {0};
-  SMemTable     *pMemTable = pTsdb->mem;
-  void          *tptr;
-  STbData       *pTbData;
-  STSRow        *row;
-  TSKEY          keyMin;
-  TSKEY          keyMax;
-  SSubmitBlk    *pBlkCopy;
-  int64_t        sverNew;
-
-  // check if table exists
-  SMetaReader mr = {0};
-  SMetaEntry  me = {0};
-  metaReaderInit(&mr, pTsdb->pVnode->pMeta, 0);
-  if (metaGetTableEntryByUid(&mr, pMsgIter->uid) < 0) {
-    metaReaderClear(&mr);
-    terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST;
-    return -1;
+static int32_t tsdbGetOrCreateTbData(SMemTable *pMemTable, tb_uid_t suid, tb_uid_t uid, STbData **ppTbData) {
+  int32_t  code = 0;
+  int32_t  idx = 0;
+  STbData *pTbData = NULL;
+  STbData *pTbDataT = &(STbData){.suid = suid, .uid = uid};
+
+  // get
+  idx = taosArraySearchIdx(pMemTable->aTbData, &pTbDataT, tbDataPCmprFn, TD_GE);
+  if (idx >= 0) {
+    pTbData = (STbData *)taosArrayGetP(pMemTable->aTbData, idx);
+    if (tbDataPCmprFn(&pTbDataT, &pTbData) == 0) goto _exit;
   }
-  if (pRsp->tblFName) strcat(pRsp->tblFName, mr.me.name);
 
-  if (mr.me.type == TSDB_NORMAL_TABLE) {
-    sverNew = mr.me.ntbEntry.schemaRow.version;
-  } else {
-    metaGetTableEntryByUid(&mr, mr.me.ctbEntry.suid);
-    sverNew = mr.me.stbEntry.schemaRow.version;
+  // create
+  SVBufPool *pPool = pMemTable->pTsdb->pVnode->inUse;
+  int8_t     maxLevel = pMemTable->pTsdb->pVnode->config.tsdbCfg.slLevel;
+
+  pTbData = vnodeBufPoolMalloc(pPool, sizeof(*pTbData) + SL_NODE_SIZE(maxLevel) * 2);
+  if (pTbData == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
   }
-  metaReaderClear(&mr);
+  pTbData->suid = suid;
+  pTbData->uid = uid;
+  pTbData->minKey = (TSDBKEY){.ts = TSKEY_MAX, .version = INT64_MAX};
+  pTbData->maxKey = (TSDBKEY){.ts = TSKEY_MIN, .version = -1};
+  pTbData->pHead = NULL;
+  pTbData->pTail = NULL;
+  pTbData->sl.seed = taosRand();
+  pTbData->sl.size = 0;
+  pTbData->sl.maxLevel = maxLevel;
+  pTbData->sl.level = 0;
+  pTbData->sl.pHead = (SMemSkipListNode *)&pTbData[1];
+  pTbData->sl.pTail = (SMemSkipListNode *)POINTER_SHIFT(pTbData->sl.pHead, SL_NODE_SIZE(maxLevel));
+  pTbData->sl.pHead->level = maxLevel;
+  pTbData->sl.pTail->level = maxLevel;
+  for (int8_t iLevel = 0; iLevel < maxLevel; iLevel++) {
+    SL_NODE_FORWARD(pTbData->sl.pHead, iLevel) = pTbData->sl.pTail;
+    SL_NODE_BACKWARD(pTbData->sl.pTail, iLevel) = pTbData->sl.pHead;
+
+    SL_NODE_BACKWARD(pTbData->sl.pHead, iLevel) = NULL;
+    SL_NODE_FORWARD(pTbData->sl.pTail, iLevel) = NULL;
+  }
+
+  if (taosArrayInsert(pMemTable->aTbData, idx < 0 ? 0 : idx, &pTbData) == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+
+_exit:
+  *ppTbData = pTbData;
+  return code;
 
-  // create container is nedd
-  tptr = taosHashGet(pMemTable->pHashIdx, &(pMsgIter->uid), sizeof(pMsgIter->uid));
-  if (tptr == NULL) {
-    pTbData = tsdbNewTbData(pMsgIter->uid);
-    if (pTbData == NULL) {
-      return -1;
+_err:
+  *ppTbData = NULL;
+  return code;
+}
+
+static void tbDataMovePosTo(STbData *pTbData, SMemSkipListNode **pos, TSDBKEY *pKey, int32_t flags) {
+  SMemSkipListNode *px;
+  SMemSkipListNode *pn;
+  TSDBKEY          *pTKey;
+  int               c;
+  int               backward = flags & SL_MOVE_BACKWARD;
+  int               fromPos = flags & SL_MOVE_FROM_POS;
+
+  if (backward) {
+    px = pTbData->sl.pTail;
+
+    for (int8_t iLevel = pTbData->sl.maxLevel - 1; iLevel >= pTbData->sl.level; iLevel--) {
+      pos[iLevel] = px;
     }
 
-    // Put into hash
-    taosHashPut(pMemTable->pHashIdx, &(pMsgIter->uid), sizeof(pMsgIter->uid), &(pTbData), sizeof(pTbData));
+    if (pTbData->sl.level) {
+      if (fromPos) px = pos[pTbData->sl.level - 1];
 
-    // Put into skiplist
-    tSkipListPut(pMemTable->pSlIdx, pTbData);
+      for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) {
+        pn = SL_NODE_BACKWARD(px, iLevel);
+        while (pn != pTbData->sl.pHead) {
+          pTKey = (TSDBKEY *)SL_NODE_DATA(pn);
+
+          c = tsdbKeyCmprFn(pTKey, pKey);
+          if (c <= 0) {
+            break;
+          } else {
+            px = pn;
+            pn = SL_NODE_BACKWARD(px, iLevel);
+          }
+        }
+
+        pos[iLevel] = px;
+      }
+    }
   } else {
-    pTbData = *(STbData **)tptr;
+    px = pTbData->sl.pHead;
+
+    for (int8_t iLevel = pTbData->sl.maxLevel - 1; iLevel >= pTbData->sl.level; iLevel--) {
+      pos[iLevel] = px;
+    }
+
+    if (pTbData->sl.level) {
+      if (fromPos) px = pos[pTbData->sl.level - 1];
+
+      for (int8_t iLevel = pTbData->sl.level - 1; iLevel >= 0; iLevel--) {
+        pn = SL_NODE_FORWARD(px, iLevel);
+        while (pn != pTbData->sl.pHead) {
+          pTKey = (TSDBKEY *)SL_NODE_DATA(pn);
+
+          c = tsdbKeyCmprFn(pTKey, pKey);
+          if (c >= 0) {
+            break;
+          } else {
+            px = pn;
+            pn = SL_NODE_FORWARD(px, iLevel);
+          }
+        }
+
+        pos[iLevel] = px;
+      }
+    }
   }
+}
 
-  // copy data to buffer pool
-  int32_t tlen = pMsgIter->dataLen + pMsgIter->schemaLen + sizeof(*pBlock);
-  pBlkCopy = (SSubmitBlk *)vnodeBufPoolMalloc(pTsdb->pVnode->inUse, tlen);
-  memcpy(pBlkCopy, pBlock, tlen);
+static FORCE_INLINE int8_t tsdbMemSkipListRandLevel(SMemSkipList *pSl) {
+  int8_t         level = 1;
+  int8_t         tlevel = TMIN(pSl->maxLevel, pSl->level + 1);
+  const uint32_t factor = 4;
 
-  tInitSubmitBlkIter(pMsgIter, pBlkCopy, &blkIter);
-  if (blkIter.row == NULL) return 0;
-  keyMin = TD_ROW_KEY(blkIter.row);
+  while ((taosRandR(&pSl->seed) % factor) == 0 && level < tlevel) {
+    level++;
+  }
 
-  tSkipListPutBatchByIter(pTbData->pData, &blkIter, (iter_next_fn_t)tGetSubmitBlkNext);
+  return level;
+}
+static int32_t tbDataDoPut(SMemTable *pMemTable, STbData *pTbData, SMemSkipListNode **pos, TSDBROW *pRow,
+                           int8_t forward) {
+  int32_t           code = 0;
+  int8_t            level;
+  SMemSkipListNode *pNode;
+  SVBufPool        *pPool = pMemTable->pTsdb->pVnode->inUse;
+
+  // node
+  level = tsdbMemSkipListRandLevel(&pTbData->sl);
+  pNode = (SMemSkipListNode *)vnodeBufPoolMalloc(pPool, SL_NODE_SIZE(level) + tPutTSDBRow(NULL, pRow));
+  if (pNode == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _exit;
+  }
+  pNode->level = level;
+  for (int8_t iLevel = 0; iLevel < level; iLevel++) {
+    SL_NODE_FORWARD(pNode, iLevel) = NULL;
+    SL_NODE_BACKWARD(pNode, iLevel) = NULL;
+  }
 
-#ifdef TD_DEBUG_PRINT_ROW
-  printf("!!! %s:%d vgId:%d dir:%s table:%" PRIi64 " has %d rows in skiplist\n\n", __func__, __LINE__,
-         TD_VID(pTsdb->pVnode), pTsdb->dir, pTbData->uid, SL_SIZE(pTbData->pData));
-#endif
+  tPutTSDBRow((uint8_t *)SL_NODE_DATA(pNode), pRow);
 
-  // Set statistics
-  keyMax = TD_ROW_KEY(blkIter.row);
+  // put
+  for (int8_t iLevel = 0; iLevel < pNode->level; iLevel++) {
+    SMemSkipListNode *px = pos[iLevel];
 
-  pTbData->nrows += pMsgIter->numOfRows;
-  if (pTbData->minKey.ts > keyMin) pTbData->minKey.ts = keyMin;
-  if (pTbData->maxKey.ts < keyMax) pTbData->maxKey.ts = keyMax;
+    if (forward) {
+      SMemSkipListNode *pNext = SL_NODE_FORWARD(px, iLevel);
 
-  pMemTable->nRow += pMsgIter->numOfRows;
-  if (pMemTable->minKey.ts > keyMin) pMemTable->minKey.ts = keyMin;
-  if (pMemTable->maxKey.ts < keyMax) pMemTable->maxKey.ts = keyMax;
+      SL_NODE_FORWARD(pNode, iLevel) = pNext;
+      SL_NODE_BACKWARD(pNode, iLevel) = px;
 
-  pRsp->numOfRows = pMsgIter->numOfRows;
-  pRsp->affectedRows = pMsgIter->numOfRows;
-  pRsp->sver = sverNew;
+      SL_NODE_BACKWARD(pNext, iLevel) = pNode;
+      SL_NODE_FORWARD(px, iLevel) = pNode;
+    } else {
+      SMemSkipListNode *pPrev = SL_NODE_BACKWARD(px, iLevel);
 
-  return 0;
-}
+      SL_NODE_FORWARD(pNode, iLevel) = px;
+      SL_NODE_BACKWARD(pNode, iLevel) = pPrev;
 
-static STbData *tsdbNewTbData(tb_uid_t uid) {
-  STbData *pTbData = (STbData *)taosMemoryCalloc(1, sizeof(*pTbData));
-  if (pTbData == NULL) {
-    return NULL;
+      SL_NODE_FORWARD(pPrev, iLevel) = pNode;
+      SL_NODE_BACKWARD(px, iLevel) = pNode;
+    }
   }
 
-  pTbData->uid = uid;
-  pTbData->minKey.ts = TSKEY_MAX;
-  pTbData->maxKey.ts = TSKEY_MIN;
-  pTbData->nrows = 0;
-#if 0
-  pTbData->pData = tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_DISCARD_DUP_KEY,
-                                   tsdbGetTsTupleKey);
-#endif
-  pTbData->pData =
-      tSkipListCreate(5, TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), tkeyComparFn, SL_ALLOW_DUP_KEY, tsdbGetTsTupleKey);
-  if (pTbData->pData == NULL) {
-    taosMemoryFree(pTbData);
-    return NULL;
+  pTbData->sl.size++;
+  if (pTbData->sl.level < pNode->level) {
+    pTbData->sl.level = pNode->level;
   }
 
-  return pTbData;
+_exit:
+  return code;
 }
 
-static void tsdbFreeTbData(STbData *pTbData) {
-  if (pTbData) {
-    tSkipListDestroy(pTbData->pData);
-    taosMemoryFree(pTbData);
+static int32_t tsdbInsertTableDataImpl(SMemTable *pMemTable, STbData *pTbData, int64_t version,
+                                       SSubmitMsgIter *pMsgIter, SSubmitBlk *pBlock, SSubmitBlkRsp *pRsp) {
+  int32_t           code = 0;
+  SSubmitBlkIter    blkIter = {0};
+  TSDBKEY           key = {.version = version};
+  SMemSkipListNode *pos[SL_MAX_LEVEL];
+  TSDBROW           row = {.version = version, .pTSRow = NULL};
+  int32_t           nRow = 0;
+
+  tInitSubmitBlkIter(pMsgIter, pBlock, &blkIter);
+
+  // backward put first data
+  row.pTSRow = tGetSubmitBlkNext(&blkIter);
+  key.ts = row.pTSRow->ts;
+  nRow++;
+  tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_BACKWARD);
+  code = tbDataDoPut(pMemTable, pTbData, pos, &row, 0);
+  if (code) {
+    goto _err;
   }
-}
 
-static char *tsdbGetTsTupleKey(const void *data) { return (char *)TD_ROW_KEY_ADDR((STSRow *)data); }
+  if (tsdbKeyCmprFn(&key, &pTbData->minKey) < 0) {
+    pTbData->minKey = key;
+  }
 
-static int tsdbTbDataComp(const void *arg1, const void *arg2) {
-  STbData *pTbData1 = (STbData *)arg1;
-  STbData *pTbData2 = (STbData *)arg2;
+  if (tsdbKeyCmprFn(&key, &pMemTable->minKey) < 0) {
+    pMemTable->minKey = key;
+  }
 
-  if (pTbData1->uid > pTbData2->uid) {
-    return 1;
-  } else if (pTbData1->uid == pTbData2->uid) {
-    return 0;
-  } else {
-    return -1;
+  // forward put rest data
+  row.pTSRow = tGetSubmitBlkNext(&blkIter);
+  if (row.pTSRow) {
+    key.ts = row.pTSRow->ts;
+    for (int8_t iLevel = 0; iLevel < pTbData->sl.maxLevel; iLevel++) {
+      pos[iLevel] = SL_NODE_BACKWARD(pos[iLevel], iLevel);
+    }
+    do {
+      nRow++;
+      tbDataMovePosTo(pTbData, pos, &key, SL_MOVE_FROM_POS);
+      code = tbDataDoPut(pMemTable, pTbData, pos, &row, 1);
+      if (code) {
+        goto _err;
+      }
+
+      row.pTSRow = tGetSubmitBlkNext(&blkIter);
+      key.ts = row.pTSRow->ts;
+    } while (row.pTSRow);
+  }
+
+  if (tsdbKeyCmprFn(&key, &pTbData->maxKey) > 0) {
+    pTbData->maxKey = key;
   }
+
+  if (tsdbKeyCmprFn(&key, &pMemTable->maxKey) > 0) {
+    pMemTable->maxKey = key;
+  }
+  pMemTable->nRef++;
+  pRsp->numOfRows = nRow;
+  pRsp->affectedRows = nRow;
+
+  return code;
+
+_err:
+  return code;
 }
 
-static char *tsdbTbDataGetUid(const void *arg) {
-  STbData *pTbData = (STbData *)arg;
-  return (char *)(&(pTbData->uid));
+static int32_t tPutTSDBRow(uint8_t *p, TSDBROW *pRow) {
+  int32_t n = 0;
+
+  n += tPutI64(p, pRow->version);
+  if (p) memcpy(p + n, pRow->pTSRow, pRow->pTSRow->len);
+  n += pRow->pTSRow->len;
+
+  return n;
 }
 
-static int tsdbAppendTableRowToCols(STsdb *pTsdb, STable *pTable, SDataCols *pCols, STSchema **ppSchema, STSRow *row,
-                                    bool merge) {
-  if (pCols) {
-    if (*ppSchema == NULL || schemaVersion(*ppSchema) != TD_ROW_SVER(row)) {
-      *ppSchema = tsdbGetTableSchemaImpl(pTsdb, pTable, false, false, TD_ROW_SVER(row));
-      if (*ppSchema == NULL) {
-        ASSERT(false);
-        return -1;
-      }
-    }
+static int32_t tGetTSDBRow(uint8_t *p, TSDBROW *pRow) {
+  int32_t n = 0;
 
-    tdAppendSTSRowToDataCol(row, *ppSchema, pCols, merge);
-  }
+  n += tGetI64(p, &pRow->version);
+  pRow->pTSRow = (STSRow *)(p + n);
+  n += pRow->pTSRow->len;
 
-  return 0;
+  return n;
 }
\ No newline at end of file

source/dnode/vnode/src/tsdb/tsdbMemTable2.c

@@ -15,11 +15,6 @@
 
 #include "tsdb.h"
 
-struct SMemSkipListNode {
-  int8_t            level;
-  SMemSkipListNode *forwards[0];
-};
-
 typedef struct {
   tb_uid_t  uid;
   STSchema *pTSchema;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -67,15 +67,16 @@ enum {
 };
 
 typedef struct STableCheckInfo {
-  uint64_t           tableId;
-  TSKEY              lastKey;
-  SBlockInfo*        pCompInfo;
-  int32_t            compSize;
-  int32_t            numOfBlocks : 29;  // number of qualified data blocks not the original blocks
-  uint8_t            chosen : 2;        // indicate which iterator should move forward
-  bool               initBuf : 1;       // whether to initialize the in-memory skip list iterator or not
-  SSkipListIterator* iter;              // mem buffer skip list iterator
-  SSkipListIterator* iiter;             // imem buffer skip list iterator
+  uint64_t     suid;
+  uint64_t     tableId;
+  TSKEY        lastKey;
+  SBlockInfo*  pCompInfo;
+  int32_t      compSize;
+  int32_t      numOfBlocks : 29;  // number of qualified data blocks not the original blocks
+  uint8_t      chosen : 2;        // indicate which iterator should move forward
+  bool         initBuf : 1;       // whether to initialize the in-memory skip list iterator or not
+  STbDataIter* iter;              // mem buffer skip list iterator
+  STbDataIter* iiter;             // imem buffer skip list iterator
 } STableCheckInfo;
 
 typedef struct STableBlockInfo {
@@ -265,8 +266,8 @@ static void resetCheckInfo(STsdbReadHandle* pTsdbReadHandle) {
   for (int32_t i = 0; i < numOfTables; ++i) {
     STableCheckInfo* pCheckInfo = (STableCheckInfo*)taosArrayGet(pTsdbReadHandle->pTableCheckInfo, i);
     pCheckInfo->lastKey = pTsdbReadHandle->window.skey;
-    pCheckInfo->iter = tSkipListDestroyIter(pCheckInfo->iter);
-    pCheckInfo->iiter = tSkipListDestroyIter(pCheckInfo->iiter);
+    pCheckInfo->iter = tsdbTbDataIterDestroy(pCheckInfo->iter);
+    pCheckInfo->iiter = tsdbTbDataIterDestroy(pCheckInfo->iiter);
     pCheckInfo->initBuf = false;
 
     if (ASCENDING_TRAVERSE(pTsdbReadHandle->order)) {
@@ -752,23 +753,21 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
   pCheckInfo->initBuf = true;
   int32_t order = pHandle->order;
 
-  STbData** pMem = NULL;
-  STbData** pIMem = NULL;
+  STbData* pMem = NULL;
+  STbData* pIMem = NULL;
 
   TSKEY tLastKey = keyToTkey(pCheckInfo->lastKey);
   if (pHandle->pTsdb->mem != NULL) {
-    pMem = taosHashGet(pHandle->pTsdb->mem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId));
+    tsdbGetTbDataFromMemTable(pHandle->pTsdb->mem, pCheckInfo->suid, pCheckInfo->tableId, &pMem);
     if (pMem != NULL) {
-      pCheckInfo->iter =
-          tSkipListCreateIterFromVal((*pMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
+      tsdbTbDataIterCreate(pMem, &(TSDBKEY){.version = 0, .ts = tLastKey}, 0, &pCheckInfo->iter);
     }
   }
 
   if (pHandle->pTsdb->imem != NULL) {
-    pIMem = taosHashGet(pHandle->pTsdb->imem->pHashIdx, &pCheckInfo->tableId, sizeof(pCheckInfo->tableId));
+    tsdbGetTbDataFromMemTable(pHandle->pTsdb->mem, pCheckInfo->suid, pCheckInfo->tableId, &pIMem);
     if (pIMem != NULL) {
-      pCheckInfo->iiter =
-          tSkipListCreateIterFromVal((*pIMem)->pData, (const char*)&tLastKey, TSDB_DATA_TYPE_TIMESTAMP, order);
+      tsdbTbDataIterCreate(pIMem, &(TSDBKEY){.version = 0, .ts = tLastKey}, 0, &pCheckInfo->iiter);
     }
   }
 
@@ -777,22 +776,21 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
     return false;
   }
 
-  bool memEmpty = (pCheckInfo->iter == NULL) || (pCheckInfo->iter != NULL && !tSkipListIterNext(pCheckInfo->iter));
-  bool imemEmpty = (pCheckInfo->iiter == NULL) || (pCheckInfo->iiter != NULL && !tSkipListIterNext(pCheckInfo->iiter));
+  bool memEmpty = (pCheckInfo->iter == NULL) || (pCheckInfo->iter != NULL && !tsdbTbDataIterNext(pCheckInfo->iter));
+  bool imemEmpty = (pCheckInfo->iiter == NULL) || (pCheckInfo->iiter != NULL && !tsdbTbDataIterNext(pCheckInfo->iiter));
   if (memEmpty && imemEmpty) {  // buffer is empty
     return false;
   }
 
   if (!memEmpty) {
-    SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
-    assert(node != NULL);
+    TSDBROW row;
 
-    STSRow* row = (STSRow*)SL_GET_NODE_DATA(node);
-    TSKEY   key = TD_ROW_KEY(row);  // first timestamp in buffer
+    tsdbTbDataIterGet(pCheckInfo->iter, &row);
+    TSKEY key = row.pTSRow->ts;  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
               "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%" PRId64 ", %s",
-              pHandle, pCheckInfo->tableId, key, order, (*pMem)->minKey.ts, (*pMem)->maxKey.ts, pCheckInfo->lastKey,
-              (*pMem)->nrows, pHandle->idStr);
+              pHandle, pCheckInfo->tableId, key, order, pMem->minKey.ts, pMem->maxKey.ts, pCheckInfo->lastKey,
+              pMem->sl.size, pHandle->idStr);
 
     if (ASCENDING_TRAVERSE(order)) {
       assert(pCheckInfo->lastKey <= key);
@@ -805,15 +803,14 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
   }
 
   if (!imemEmpty) {
-    SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
-    assert(node != NULL);
+    TSDBROW row;
 
-    STSRow* row = (STSRow*)SL_GET_NODE_DATA(node);
-    TSKEY   key = TD_ROW_KEY(row);  // first timestamp in buffer
+    tsdbTbDataIterGet(pCheckInfo->iter, &row);
+    TSKEY key = row.pTSRow->ts;  // first timestamp in buffer
     tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64
               "-%" PRId64 ", lastKey:%" PRId64 ", numOfRows:%" PRId64 ", %s",
-              pHandle, pCheckInfo->tableId, key, order, (*pIMem)->minKey.ts, (*pIMem)->maxKey.ts, pCheckInfo->lastKey,
-              (*pIMem)->nrows, pHandle->idStr);
+              pHandle, pCheckInfo->tableId, key, order, pIMem->minKey.ts, pIMem->maxKey.ts, pCheckInfo->lastKey,
+              pIMem->sl.size, pHandle->idStr);
 
     if (ASCENDING_TRAVERSE(order)) {
       assert(pCheckInfo->lastKey <= key);
@@ -828,31 +825,23 @@ static bool initTableMemIterator(STsdbReadHandle* pHandle, STableCheckInfo* pChe
 }
 
 static void destroyTableMemIterator(STableCheckInfo* pCheckInfo) {
-  tSkipListDestroyIter(pCheckInfo->iter);
-  tSkipListDestroyIter(pCheckInfo->iiter);
+  tsdbTbDataIterDestroy(pCheckInfo->iter);
+  tsdbTbDataIterDestroy(pCheckInfo->iiter);
 }
 
 static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, TDRowVerT maxVer) {
+  TSDBROW row = {0};
   STSRow *rmem = NULL, *rimem = NULL;
+
   if (pCheckInfo->iter) {
-    SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
-    if (node != NULL) {
-      rmem = (STSRow*)SL_GET_NODE_DATA(node);
-      // TODO: filter max version
-      // if (TD_ROW_VER(rmem) > maxVer) {
-      //   rmem = NULL;
-      // }
+    if (tsdbTbDataIterGet(pCheckInfo->iter, &row)) {
+      rmem = row.pTSRow;
     }
   }
 
   if (pCheckInfo->iiter) {
-    SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
-    if (node != NULL) {
-      rimem = (STSRow*)SL_GET_NODE_DATA(node);
-      // TODO: filter max version
-      // if (TD_ROW_VER(rimem) > maxVer) {
-      //   rimem = NULL;
-      // }
+    if (tsdbTbDataIterGet(pCheckInfo->iiter, &row)) {
+      rimem = row.pTSRow;
     }
   }
 
@@ -889,7 +878,7 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
       pCheckInfo->chosen = CHECKINFO_CHOSEN_BOTH;
     } else {
       pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
-      tSkipListIterNext(pCheckInfo->iter);
+      tsdbTbDataIterNext(pCheckInfo->iter);
     }
     return r1;
   } else if (r1 < r2 && ASCENDING_TRAVERSE(order)) {
@@ -903,28 +892,17 @@ static TSKEY extractFirstTraverseKey(STableCheckInfo* pCheckInfo, int32_t order,
 
 static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int32_t update, STSRow** extraRow,
                                  TDRowVerT maxVer) {
+  TSDBROW row;
   STSRow *rmem = NULL, *rimem = NULL;
   if (pCheckInfo->iter) {
-    SSkipListNode* node = tSkipListIterGet(pCheckInfo->iter);
-    if (node != NULL) {
-      rmem = (STSRow*)SL_GET_NODE_DATA(node);
-#if 0  // TODO: skiplist refactor
-      if (TD_ROW_VER(rmem) > maxVer) {
-        rmem = NULL;
-      }
-#endif
+    if (tsdbTbDataIterGet(pCheckInfo->iter, &row)) {
+      rmem = row.pTSRow;
     }
   }
 
   if (pCheckInfo->iiter) {
-    SSkipListNode* node = tSkipListIterGet(pCheckInfo->iiter);
-    if (node != NULL) {
-      rimem = (STSRow*)SL_GET_NODE_DATA(node);
-#if 0  // TODO: skiplist refactor
-      if (TD_ROW_VER(rimem) > maxVer) {
-        rimem = NULL;
-      }
-#endif
+    if (tsdbTbDataIterGet(pCheckInfo->iiter, &row)) {
+      rimem = row.pTSRow;
     }
   }
 
@@ -966,7 +944,7 @@ static STSRow* getSRowInTableMem(STableCheckInfo* pCheckInfo, int32_t order, int
       *extraRow = rimem;
       return rmem;
     } else {
-      tSkipListIterNext(pCheckInfo->iter);
+      tsdbTbDataIterNext(pCheckInfo->iter);
       pCheckInfo->chosen = CHECKINFO_CHOSEN_IMEM;
       return rimem;
     }
@@ -995,7 +973,7 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
   bool hasNext = false;
   if (pCheckInfo->chosen == CHECKINFO_CHOSEN_MEM) {
     if (pCheckInfo->iter != NULL) {
-      hasNext = tSkipListIterNext(pCheckInfo->iter);
+      hasNext = tsdbTbDataIterNext(pCheckInfo->iter);
     }
 
     if (hasNext) {
@@ -1003,11 +981,11 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
     }
 
     if (pCheckInfo->iiter != NULL) {
-      return tSkipListIterGet(pCheckInfo->iiter) != NULL;
+      return tsdbTbDataIterGet(pCheckInfo->iiter, NULL);
     }
   } else if (pCheckInfo->chosen == CHECKINFO_CHOSEN_IMEM) {
     if (pCheckInfo->iiter != NULL) {
-      hasNext = tSkipListIterNext(pCheckInfo->iiter);
+      hasNext = tsdbTbDataIterNext(pCheckInfo->iiter);
     }
 
     if (hasNext) {
@@ -1015,14 +993,14 @@ static bool moveToNextRowInMem(STableCheckInfo* pCheckInfo) {
     }
 
     if (pCheckInfo->iter != NULL) {
-      return tSkipListIterGet(pCheckInfo->iter) != NULL;
+      return tsdbTbDataIterGet(pCheckInfo->iter, NULL);
     }
   } else {
     if (pCheckInfo->iter != NULL) {
-      hasNext = tSkipListIterNext(pCheckInfo->iter);
+      hasNext = tsdbTbDataIterNext(pCheckInfo->iter);
     }
     if (pCheckInfo->iiter != NULL) {
-      hasNext = tSkipListIterNext(pCheckInfo->iiter) || hasNext;
+      hasNext = tsdbTbDataIterNext(pCheckInfo->iiter) || hasNext;
     }
   }
 

source/dnode/vnode/src/tsdb/tsdbWrite.c

@@ -39,7 +39,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp *
     SSubmitBlkRsp r = {0};
     tGetSubmitMsgNext(&msgIter, &pBlock);
     if (pBlock == NULL) break;
-    if (tsdbInsertTableData(pTsdb, &msgIter, pBlock, &r) < 0) {
+    if (tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r) < 0) {
       return -1;
     }
 

source/dnode/vnode/src/vnd/vnodeSvr.c

@@ -779,7 +779,7 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq
       sprintf(submitBlkRsp.tblFName, "%s.", pVnode->config.dbname);
     }
 
-    if (tsdbInsertTableData(pVnode->pTsdb, &msgIter, pBlock, &submitBlkRsp) < 0) {
+    if (tsdbInsertTableData(pVnode->pTsdb, version, &msgIter, pBlock, &submitBlkRsp) < 0) {
       submitBlkRsp.code = terrno;
     }