other:merge 3.0

examples/c/CMakeLists.txt

@@ -53,7 +53,7 @@ IF (TD_LINUX)
       )
 
   target_include_directories(demoapi
-      PUBLIC "${TD_SOURCE_DIR}/include/client"
+      PUBLIC "${TD_SOURCE_DIR}/include/client"other:merge 3.0
       PUBLIC "${TD_SOURCE_DIR}/include/os"
       PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/inc"
       )

source/dnode/vnode/src/inc/sma.h

@@ -148,6 +148,7 @@ struct SRSmaInfoItem {
 };
 
 struct SRSmaInfo {
+  SSma     *pSma;
   STSchema *pTSchema;
   int64_t   suid;
   int64_t   lastRecv;  // ms

source/dnode/vnode/src/inc/tsdb.h

@@ -43,14 +43,14 @@ typedef struct STbDataIter   STbDataIter;
 typedef struct SMapData      SMapData;
 typedef struct SBlockIdx     SBlockIdx;
 typedef struct SDataBlk      SDataBlk;
-typedef struct SSstBlk       SSstBlk;
+typedef struct SSttBlk       SSttBlk;
 typedef struct SColData      SColData;
 typedef struct SDiskDataHdr  SDiskDataHdr;
 typedef struct SBlockData    SBlockData;
 typedef struct SDelFile      SDelFile;
 typedef struct SHeadFile     SHeadFile;
 typedef struct SDataFile     SDataFile;
-typedef struct SSstFile      SSstFile;
+typedef struct SSttFile      SSttFile;
 typedef struct SSmaFile      SSmaFile;
 typedef struct SDFileSet     SDFileSet;
 typedef struct SDataFWriter  SDataFWriter;
@@ -65,12 +65,14 @@ typedef struct SBlockInfo    SBlockInfo;
 typedef struct SSmaInfo      SSmaInfo;
 typedef struct SBlockCol     SBlockCol;
 typedef struct SVersionRange SVersionRange;
+typedef struct SLDataIter    SLDataIter;
 
 #define TSDB_FILE_DLMT         ((uint32_t)0xF00AFA0F)
 #define TSDB_MAX_SUBBLOCKS     8
-#define TSDB_MAX_LAST_FILE     16
-#define TSDB_DEFAULT_LAST_FILE 8
+#define TSDB_MAX_STT_FILE      16
+#define TSDB_DEFAULT_STT_FILE  8
 #define TSDB_FHDR_SIZE         512
+#define TSDB_DEFAULT_PAGE_SIZE 4096
 
 #define HAS_NONE  ((int8_t)0x1)
 #define HAS_NULL  ((int8_t)0x2)
@@ -82,6 +84,14 @@ typedef struct SVersionRange SVersionRange;
 #define TSDBKEY_MIN ((TSDBKEY){.ts = TSKEY_MIN, .version = VERSION_MIN})
 #define TSDBKEY_MAX ((TSDBKEY){.ts = TSKEY_MAX, .version = VERSION_MAX})
 
+#define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM))
+#define LOGIC_TO_FILE_OFFSET(LOFFSET, PAGE) \
+  ((LOFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (LOFFSET) % PAGE_CONTENT_SIZE(PAGE))
+#define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE))
+#define PAGE_OFFSET(PGNO, PAGE)            (((PGNO)-1) * (PAGE))
+#define OFFSET_PGNO(OFFSET, PAGE)          ((OFFSET) / (PAGE) + 1)
+#define LOGIC_TO_FILE_SIZE(LSIZE, PAGE)    OFFSET_PGNO(LOGIC_TO_FILE_OFFSET(LSIZE, PAGE), PAGE) * (PAGE)
+
 // tsdbUtil.c ==============================================================================================
 // TSDBROW
 #define TSDBROW_TS(ROW)                       (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow])
@@ -120,9 +130,9 @@ int32_t tPutDataBlk(uint8_t *p, void *ph);
 int32_t tGetDataBlk(uint8_t *p, void *ph);
 int32_t tDataBlkCmprFn(const void *p1, const void *p2);
 bool    tDataBlkHasSma(SDataBlk *pDataBlk);
-// SSstBlk
-int32_t tPutSstBlk(uint8_t *p, void *ph);
-int32_t tGetSstBlk(uint8_t *p, void *ph);
+// SSttBlk
+int32_t tPutSttBlk(uint8_t *p, void *ph);
+int32_t tGetSttBlk(uint8_t *p, void *ph);
 // SBlockIdx
 int32_t tPutBlockIdx(uint8_t *p, void *ph);
 int32_t tGetBlockIdx(uint8_t *p, void *ph);
@@ -195,7 +205,6 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
                         uint8_t **ppBuf);
 int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData,
                           uint8_t **ppBuf);
-int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck);
 // tsdbMemTable ==============================================================================================
 // SMemTable
 int32_t  tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable);
@@ -219,7 +228,7 @@ bool    tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2);
 int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype);
 int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile);
 int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile);
-int32_t tPutSstFile(uint8_t *p, SSstFile *pSstFile);
+int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile);
 int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile);
 int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile);
 int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile);
@@ -228,7 +237,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet);
 
 void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]);
 void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]);
-void tsdbSstFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSstFile *pSstF, char fname[]);
+void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]);
 void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]);
 // SDelFile
 void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]);
@@ -254,7 +263,7 @@ int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync);
 int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter);
 int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx);
 int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx);
-int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk);
+int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk);
 int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
                            int8_t cmprAlg, int8_t toLast);
 
@@ -264,10 +273,10 @@ int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pS
 int32_t tsdbDataFReaderClose(SDataFReader **ppReader);
 int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx);
 int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData);
-int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk);
+int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk);
 int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg);
 int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData);
-int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData);
+int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData);
 // SDelFWriter
 int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb);
 int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync);
@@ -439,7 +448,7 @@ struct SDataBlk {
   SSmaInfo   smaInfo;
 };
 
-struct SSstBlk {
+struct SSttBlk {
   int64_t    suid;
   int64_t    minUid;
   int64_t    maxUid;
@@ -541,7 +550,7 @@ struct SDataFile {
   int64_t size;
 };
 
-struct SSstFile {
+struct SSttFile {
   volatile int32_t nRef;
 
   int64_t commitID;
@@ -562,8 +571,8 @@ struct SDFileSet {
   SHeadFile *pHeadF;
   SDataFile *pDataF;
   SSmaFile  *pSmaF;
-  uint8_t    nSstF;
-  SSstFile  *aSstF[TSDB_MAX_LAST_FILE];
+  uint8_t    nSttF;
+  SSttFile  *aSttF[TSDB_MAX_STT_FILE];
 };
 
 struct SRowIter {
@@ -578,45 +587,53 @@ struct SRowMerger {
   SArray   *pArray;  // SArray<SColVal>
 };
 
+typedef struct {
+  char     *path;
+  int32_t   szPage;
+  int32_t   flag;
+  TdFilePtr pFD;
+  int64_t   pgno;
+  uint8_t  *pBuf;
+  int64_t   szFile;
+} STsdbFD;
+
 struct SDelFWriter {
   STsdb   *pTsdb;
   SDelFile fDel;
-  TdFilePtr pWriteH;
-
+  STsdbFD *pWriteH;
   uint8_t *aBuf[1];
 };
 
+struct STsdbReadSnap {
+  SMemTable *pMem;
+  SMemTable *pIMem;
+  STsdbFS    fs;
+};
+
 struct SDataFWriter {
   STsdb    *pTsdb;
   SDFileSet wSet;
 
-  TdFilePtr pHeadFD;
-  TdFilePtr pDataFD;
-  TdFilePtr pSmaFD;
-  TdFilePtr pLastFD;
+  STsdbFD *pHeadFD;
+  STsdbFD *pDataFD;
+  STsdbFD *pSmaFD;
+  STsdbFD *pSttFD;
 
   SHeadFile fHead;
   SDataFile fData;
   SSmaFile  fSma;
-  SSstFile  fSst[TSDB_MAX_LAST_FILE];
+  SSttFile  fStt[TSDB_MAX_STT_FILE];
 
   uint8_t *aBuf[4];
 };
 
-struct STsdbReadSnap {
-  SMemTable *pMem;
-  SMemTable *pIMem;
-  STsdbFS    fs;
-};
-
 struct SDataFReader {
   STsdb     *pTsdb;
   SDFileSet *pSet;
-  TdFilePtr  pHeadFD;
-  TdFilePtr  pDataFD;
-  TdFilePtr  pSmaFD;
-  TdFilePtr  aLastFD[TSDB_MAX_LAST_FILE];
-
+  STsdbFD   *pHeadFD;
+  STsdbFD   *pDataFD;
+  STsdbFD   *pSmaFD;
+  STsdbFD   *aSttFD[TSDB_MAX_STT_FILE];
   uint8_t   *aBuf[3];
 };
 
@@ -630,11 +647,12 @@ typedef struct SMergeTree {
   int8_t      backward;
   SRBTree     rbt;
   SArray     *pIterList;
-  struct SLDataIter *pIter;
+  SLDataIter *pIter;
 } SMergeTree;
 
-int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader* pFReader, uint64_t suid, uint64_t uid, STimeWindow* pTimeWindow, SVersionRange* pVerRange);
-void tMergeTreeAddIter(SMergeTree *pMTree, struct SLDataIter *pIter);
+int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid,
+                       STimeWindow *pTimeWindow, SVersionRange *pVerRange);
+void    tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter);
 bool    tMergeTreeNext(SMergeTree *pMTree);
 TSDBROW tMergeTreeGetRow(SMergeTree *pMTree);
 void    tMergeTreeClose(SMergeTree *pMTree);

source/dnode/vnode/src/sma/smaCommit.c

@@ -188,13 +188,15 @@ static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) {
 
   for (int32_t i = 0; i < taosArrayGetSize(pFS->aQTaskInf);) {
     SQTaskFile *pTaskF = taosArrayGet(pFS->aQTaskInf, i);
-    if (atomic_sub_fetch_32(&pTaskF->nRef, 1) <= 0) {
+    int32_t     oldVal = atomic_fetch_sub_32(&pTaskF->nRef, 1);
+    if ((oldVal <= 1) && (pTaskF->version < committed)) {
       tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName);
       if (taosRemoveFile(qTaskInfoFullName) < 0) {
-        smaWarn("vgId:%d, cleanup qinf, failed to remove %s since %s", TD_VID(pVnode), qTaskInfoFullName,
-                tstrerror(TAOS_SYSTEM_ERROR(errno)));
+        smaWarn("vgId:%d, cleanup qinf, committed %" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed,
+                qTaskInfoFullName, tstrerror(TAOS_SYSTEM_ERROR(errno)));
       } else {
-        smaDebug("vgId:%d, cleanup qinf, success to remove %s", TD_VID(pVnode), qTaskInfoFullName);
+        smaDebug("vgId:%d, cleanup qinf, committed %" PRIi64 ", success to remove %s", TD_VID(pVnode), committed,
+                 qTaskInfoFullName);
       }
       taosArrayRemove(pFS->aQTaskInf, i);
       continue;
@@ -364,7 +366,6 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) {
   }
 
   SRSmaStat     *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv);
-  SRSmaInfoItem *pItem = NULL;
 
   // step 1: merge qTaskInfo and iQTaskInfo
   // lock

source/dnode/vnode/src/sma/smaEnv.c

@@ -62,7 +62,7 @@ int32_t smaInit() {
     }
 
     int32_t type = (8 == POINTER_BYTES) ? TSDB_DATA_TYPE_UBIGINT : TSDB_DATA_TYPE_UINT;
-    smaMgmt.refHash = taosHashInit(1, taosGetDefaultHashFunction(type), true, HASH_ENTRY_LOCK);
+    smaMgmt.refHash = taosHashInit(64, taosGetDefaultHashFunction(type), true, HASH_ENTRY_LOCK);
     if (!smaMgmt.refHash) {
       taosCloseRef(smaMgmt.rsetId);
       atomic_store_8(&smaMgmt.inited, 0);
@@ -107,6 +107,7 @@ void smaCleanUp() {
   if (old == 1) {
     taosCloseRef(smaMgmt.rsetId);
     taosHashCleanup(smaMgmt.refHash);
+    smaMgmt.refHash = NULL;
     taosTmrCleanUp(smaMgmt.tmrHandle);
     smaInfo("sma mgmt env is cleaned up, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle);
     atomic_store_8(&smaMgmt.inited, 0);
@@ -220,7 +221,7 @@ static void tRSmaInfoHashFreeNode(void *data) {
     if ((pItem = RSMA_INFO_ITEM((SRSmaInfo *)pRSmaInfo, 1)) && pItem->level) {
       taosHashRemove(smaMgmt.refHash, &pItem, POINTER_BYTES);
     }
-    tdFreeRSmaInfo(NULL, pRSmaInfo, true);
+    tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo, true);
   }
 }
 

source/dnode/vnode/src/sma/smaRollup.c

@@ -121,27 +121,27 @@ static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t l
  */
 void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) {
   if (pInfo) {
-    int32_t vid = pSma ? SMA_VID(pSma) : -1;
     for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) {
       SRSmaInfoItem *pItem = &pInfo->items[i];
 
       if (isDeepFree && pItem->tmrId) {
-        smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", vid, pItem->tmrId, pInfo->suid, i + 1);
+        smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pItem->tmrId,
+                 pInfo->suid, i + 1);
         taosTmrStopA(&pItem->tmrId);
       }
 
       if (isDeepFree && pInfo->taskInfo[i]) {
-        tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], vid, i + 1);
+        tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1);
       } else {
-        smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", vid,
+        smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma),
                  pInfo->suid, i + 1);
       }
 
       if (pInfo->iTaskInfo[i]) {
-        tdRSmaQTaskInfoFree(&pInfo->iTaskInfo[i], vid, i + 1);
+        tdRSmaQTaskInfoFree(&pInfo->iTaskInfo[i], SMA_VID(pSma), i + 1);
       } else {
-        smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo", vid,
-                 pInfo->suid, i + 1);
+        smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo",
+                 SMA_VID(pSma), pInfo->suid, i + 1);
       }
     }
     if (isDeepFree) {
@@ -377,7 +377,10 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
     terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
     goto _err;
   }
+  pRSmaInfo->pSma = pSma;
   pRSmaInfo->pTSchema = pTSchema;
+  pRSmaInfo->suid = suid;
+  T_REF_INIT_VAL(pRSmaInfo, 1);
   if (!(pRSmaInfo->queue = taosOpenQueue())) {
     goto _err;
   }
@@ -391,8 +394,6 @@ int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con
   if (!(pRSmaInfo->iQall = taosAllocateQall())) {
     goto _err;
   }
-  pRSmaInfo->suid = suid;
-  T_REF_INIT_VAL(pRSmaInfo, 1);
 
   if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) {
     goto _err;

source/dnode/vnode/src/tsdb/tsdbCommit.c

@@ -32,12 +32,12 @@ typedef struct {
       STbDataIter iter;
     };  // memory data iter
     struct {
-      int32_t    iSst;
-      SArray    *aSstBlk;
-      int32_t    iSstBlk;
+      int32_t    iStt;
+      SArray    *aSttBlk;
+      int32_t    iSttBlk;
       SBlockData bData;
       int32_t    iRow;
-    };  // sst file data iter
+    };  // stt file data iter
   };
 } SDataIter;
 
@@ -71,13 +71,13 @@ typedef struct {
     SDataIter *pIter;
     SRBTree    rbt;
     SDataIter  dataIter;
-    SDataIter  aDataIter[TSDB_MAX_LAST_FILE];
+    SDataIter  aDataIter[TSDB_MAX_STT_FILE];
     int8_t     toLastOnly;
   };
   struct {
     SDataFWriter *pWriter;
     SArray       *aBlockIdx;  // SArray<SBlockIdx>
-    SArray       *aSstBlk;    // SArray<SSstBlk>
+    SArray       *aSttBlk;    // SArray<SSttBlk>
     SMapData      mBlock;     // SMapData<SDataBlk>
     SBlockData    bData;
     SBlockData    bDatal;
@@ -92,9 +92,6 @@ typedef struct {
   SArray      *aDelData;  // SArray<SDelData>
 } SCommitter;
 
-extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *aSstBlk,
-                                  SBlockData *pBlockData);  // todo
-
 static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter);
 static int32_t tsdbCommitData(SCommitter *pCommitter);
 static int32_t tsdbCommitDel(SCommitter *pCommitter);
@@ -431,21 +428,21 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
   pCommitter->toLastOnly = 0;
   SDataFReader *pReader = pCommitter->dReader.pReader;
   if (pReader) {
-    if (pReader->pSet->nSstF >= pCommitter->maxLast) {
+    if (pReader->pSet->nSttF >= pCommitter->maxLast) {
       int8_t iIter = 0;
-      for (int32_t iSst = 0; iSst < pReader->pSet->nSstF; iSst++) {
+      for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) {
         pIter = &pCommitter->aDataIter[iIter];
         pIter->type = LAST_DATA_ITER;
-        pIter->iSst = iSst;
+        pIter->iStt = iStt;
 
-        code = tsdbReadSstBlk(pCommitter->dReader.pReader, iSst, pIter->aSstBlk);
+        code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk);
         if (code) goto _err;
 
-        if (taosArrayGetSize(pIter->aSstBlk) == 0) continue;
+        if (taosArrayGetSize(pIter->aSttBlk) == 0) continue;
 
-        pIter->iSstBlk = 0;
-        SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, 0);
-        code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, iSst, pSstBlk, &pIter->bData);
+        pIter->iSttBlk = 0;
+        SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0);
+        code = tsdbReadSttBlock(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData);
         if (code) goto _err;
 
         pIter->iRow = 0;
@@ -457,9 +454,9 @@ static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) {
         iIter++;
       }
     } else {
-      for (int32_t iSst = 0; iSst < pReader->pSet->nSstF; iSst++) {
-        SSstFile *pSstFile = pReader->pSet->aSstF[iSst];
-        if (pSstFile->size > pSstFile->offset) {
+      for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) {
+        SSttFile *pSttFile = pReader->pSet->aSttF[iStt];
+        if (pSttFile->size > pSttFile->offset) {
           pCommitter->toLastOnly = 1;
           break;
         }
@@ -515,34 +512,34 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) {
   SHeadFile fHead = {.commitID = pCommitter->commitID};
   SDataFile fData = {.commitID = pCommitter->commitID};
   SSmaFile  fSma = {.commitID = pCommitter->commitID};
-  SSstFile  fSst = {.commitID = pCommitter->commitID};
+  SSttFile  fStt = {.commitID = pCommitter->commitID};
   SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma};
   if (pRSet) {
-    ASSERT(pRSet->nSstF <= pCommitter->maxLast);
+    ASSERT(pRSet->nSttF <= pCommitter->maxLast);
     fData = *pRSet->pDataF;
     fSma = *pRSet->pSmaF;
     wSet.diskId = pRSet->diskId;
-    if (pRSet->nSstF < pCommitter->maxLast) {
-      for (int32_t iSst = 0; iSst < pRSet->nSstF; iSst++) {
-        wSet.aSstF[iSst] = pRSet->aSstF[iSst];
+    if (pRSet->nSttF < pCommitter->maxLast) {
+      for (int32_t iStt = 0; iStt < pRSet->nSttF; iStt++) {
+        wSet.aSttF[iStt] = pRSet->aSttF[iStt];
       }
-      wSet.nSstF = pRSet->nSstF + 1;
+      wSet.nSttF = pRSet->nSttF + 1;
     } else {
-      wSet.nSstF = 1;
+      wSet.nSttF = 1;
     }
   } else {
     SDiskID did = {0};
     tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did);
     tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did);
     wSet.diskId = did;
-    wSet.nSstF = 1;
+    wSet.nSttF = 1;
   }
-  wSet.aSstF[wSet.nSstF - 1] = &fSst;
+  wSet.aSttF[wSet.nSttF - 1] = &fStt;
   code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet);
   if (code) goto _err;
 
   taosArrayClear(pCommitter->dWriter.aBlockIdx);
-  taosArrayClear(pCommitter->dWriter.aSstBlk);
+  taosArrayClear(pCommitter->dWriter.aSttBlk);
   tMapDataReset(&pCommitter->dWriter.mBlock);
   tBlockDataReset(&pCommitter->dWriter.bData);
   tBlockDataReset(&pCommitter->dWriter.bDatal);
@@ -613,7 +610,7 @@ _err:
 
 static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) {
   int32_t     code = 0;
-  SSstBlk     blockL;
+  SSttBlk     blockL;
   SBlockData *pBlockData = &pCommitter->dWriter.bDatal;
 
   ASSERT(pBlockData->nRow > 0);
@@ -638,8 +635,8 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) {
   code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1);
   if (code) goto _err;
 
-  // push SSstBlk
-  if (taosArrayPush(pCommitter->dWriter.aSstBlk, &blockL) == NULL) {
+  // push SSttBlk
+  if (taosArrayPush(pCommitter->dWriter.aSttBlk, &blockL) == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
@@ -661,8 +658,8 @@ static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) {
   code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx);
   if (code) goto _err;
 
-  // write aSstBlk
-  code = tsdbWriteSstBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSstBlk);
+  // write aSttBlk
+  code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk);
   if (code) goto _err;
 
   // update file header
@@ -760,7 +757,7 @@ static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) {
   pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows;
   pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows;
   pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
-  pCommitter->maxLast = TSDB_DEFAULT_LAST_FILE;  // TODO: make it as a config
+  pCommitter->maxLast = TSDB_DEFAULT_STT_FILE;  // TODO: make it as a config
   pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem);
   if (pCommitter->aTbDataP == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
@@ -790,10 +787,10 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
   if (code) goto _exit;
 
   // merger
-  for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) {
-    SDataIter *pIter = &pCommitter->aDataIter[iSst];
-    pIter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
-    if (pIter->aSstBlk == NULL) {
+  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) {
+    SDataIter *pIter = &pCommitter->aDataIter[iStt];
+    pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+    if (pIter->aSttBlk == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _exit;
     }
@@ -809,8 +806,8 @@ static int32_t tsdbCommitDataStart(SCommitter *pCommitter) {
     goto _exit;
   }
 
-  pCommitter->dWriter.aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
-  if (pCommitter->dWriter.aSstBlk == NULL) {
+  pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+  if (pCommitter->dWriter.aSttBlk == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _exit;
   }
@@ -832,15 +829,15 @@ static void tsdbCommitDataEnd(SCommitter *pCommitter) {
   tBlockDataDestroy(&pCommitter->dReader.bData, 1);
 
   // merger
-  for (int32_t iSst = 0; iSst < TSDB_MAX_LAST_FILE; iSst++) {
-    SDataIter *pIter = &pCommitter->aDataIter[iSst];
-    taosArrayDestroy(pIter->aSstBlk);
+  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) {
+    SDataIter *pIter = &pCommitter->aDataIter[iStt];
+    taosArrayDestroy(pIter->aSttBlk);
     tBlockDataDestroy(&pIter->bData, 1);
   }
 
   // writer
   taosArrayDestroy(pCommitter->dWriter.aBlockIdx);
-  taosArrayDestroy(pCommitter->dWriter.aSstBlk);
+  taosArrayDestroy(pCommitter->dWriter.aSttBlk);
   tMapDataClear(&pCommitter->dWriter.mBlock);
   tBlockDataDestroy(&pCommitter->dWriter.bData, 1);
   tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1);
@@ -1055,11 +1052,11 @@ static int32_t tsdbNextCommitRow(SCommitter *pCommitter) {
         pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow];
         pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
       } else {
-        pIter->iSstBlk++;
-        if (pIter->iSstBlk < taosArrayGetSize(pIter->aSstBlk)) {
-          SSstBlk *pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk);
+        pIter->iSttBlk++;
+        if (pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk)) {
+          SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
 
-          code = tsdbReadSstBlockEx(pCommitter->dReader.pReader, pIter->iSst, pSstBlk, &pIter->bData);
+          code = tsdbReadSttBlock(pCommitter->dReader.pReader, pIter->iStt, pSttBlk, &pIter->bData);
           if (code) goto _exit;
 
           pIter->iRow = 0;

source/dnode/vnode/src/tsdb/tsdbFS.c

@@ -21,6 +21,9 @@ static int32_t tsdbEncodeFS(uint8_t *p, STsdbFS *pFS) {
   int8_t   hasDel = pFS->pDelFile ? 1 : 0;
   uint32_t nSet = taosArrayGetSize(pFS->aDFileSet);
 
+  // version
+  n += tPutI8(p ? p + n : p, 0);
+
   // SDelFile
   n += tPutI8(p ? p + n : p, hasDel);
   if (hasDel) {
@@ -110,7 +113,7 @@ _err:
 //       taosRemoveFile(fname);
 //     }
 
-//     // sst
+//     // stt
 //     if (isSameDisk && pFrom->pLastF->commitID == pTo->pLastF->commitID) {
 //       if (pFrom->pLastF->size > pTo->pLastF->size) {
 //         code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE);
@@ -140,7 +143,7 @@ _err:
 //     tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname);
 //     taosRemoveFile(fname);
 
-//     // sst
+//     // stt
 //     tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname);
 //     taosRemoveFile(fname);
 
@@ -255,8 +258,8 @@ void tsdbFSDestroy(STsdbFS *pFS) {
     taosMemoryFree(pSet->pHeadF);
     taosMemoryFree(pSet->pDataF);
     taosMemoryFree(pSet->pSmaF);
-    for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
-      taosMemoryFree(pSet->aSstF[iSst]);
+    for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+      taosMemoryFree(pSet->aSttF[iStt]);
     }
   }
 
@@ -292,7 +295,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
       code = TAOS_SYSTEM_ERROR(errno);
       goto _err;
     }
-    if (size != pSet->pHeadF->size) {
+    if (size != LOGIC_TO_FILE_SIZE(pSet->pHeadF->size, TSDB_DEFAULT_PAGE_SIZE)) {
       code = TSDB_CODE_FILE_CORRUPTED;
       goto _err;
     }
@@ -303,10 +306,10 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
       code = TAOS_SYSTEM_ERROR(errno);
       goto _err;
     }
-    if (size < pSet->pDataF->size) {
+    if (size < LOGIC_TO_FILE_SIZE(pSet->pDataF->size, TSDB_DEFAULT_PAGE_SIZE)) {
       code = TSDB_CODE_FILE_CORRUPTED;
       goto _err;
-    } else if (size > pSet->pDataF->size) {
+    } else if (size > LOGIC_TO_FILE_SIZE(pSet->pDataF->size, TSDB_DEFAULT_PAGE_SIZE)) {
       code = tsdbDFileRollback(pTsdb, pSet, TSDB_DATA_FILE);
       if (code) goto _err;
     }
@@ -317,22 +320,22 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) {
       code = TAOS_SYSTEM_ERROR(errno);
       goto _err;
     }
-    if (size < pSet->pSmaF->size) {
+    if (size < LOGIC_TO_FILE_SIZE(pSet->pSmaF->size, TSDB_DEFAULT_PAGE_SIZE)) {
       code = TSDB_CODE_FILE_CORRUPTED;
       goto _err;
-    } else if (size > pSet->pSmaF->size) {
+    } else if (size > LOGIC_TO_FILE_SIZE(pSet->pSmaF->size, TSDB_DEFAULT_PAGE_SIZE)) {
       code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE);
       if (code) goto _err;
     }
 
-    // sst ===========
-    for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
-      tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname);
+    // stt ===========
+    for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+      tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
       if (taosStatFile(fname, &size, NULL)) {
         code = TAOS_SYSTEM_ERROR(errno);
         goto _err;
       }
-      if (size != pSet->aSstF[iSst]->size) {
+      if (size != LOGIC_TO_FILE_SIZE(pSet->aSttF[iStt]->size, TSDB_DEFAULT_PAGE_SIZE)) {
         code = TSDB_CODE_FILE_CORRUPTED;
         goto _err;
       }
@@ -364,10 +367,12 @@ static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) {
   int32_t  code = 0;
   int8_t   hasDel;
   uint32_t nSet;
-  int32_t  n;
+  int32_t  n = 0;
+
+  // version
+  n += tGetI8(pData + n, NULL);
 
   // SDelFile
-  n = 0;
   n += tGetI8(pData + n, &hasDel);
   if (hasDel) {
     pTsdb->fs.pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile));
@@ -514,10 +519,10 @@ int32_t tsdbFSClose(STsdb *pTsdb) {
     ASSERT(pSet->pSmaF->nRef == 1);
     taosMemoryFree(pSet->pSmaF);
 
-    // sst
-    for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
-      ASSERT(pSet->aSstF[iSst]->nRef == 1);
-      taosMemoryFree(pSet->aSstF[iSst]);
+    // stt
+    for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+      ASSERT(pSet->aSttF[iStt]->nRef == 1);
+      taosMemoryFree(pSet->aSttF[iStt]);
     }
   }
 
@@ -574,14 +579,14 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) {
     }
     *fSet.pSmaF = *pSet->pSmaF;
 
-    // sst
-    for (fSet.nSstF = 0; fSet.nSstF < pSet->nSstF; fSet.nSstF++) {
-      fSet.aSstF[fSet.nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
-      if (fSet.aSstF[fSet.nSstF] == NULL) {
+    // stt
+    for (fSet.nSttF = 0; fSet.nSttF < pSet->nSttF; fSet.nSttF++) {
+      fSet.aSttF[fSet.nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+      if (fSet.aSttF[fSet.nSttF] == NULL) {
         code = TSDB_CODE_OUT_OF_MEMORY;
         goto _exit;
       }
-      *fSet.aSstF[fSet.nSstF] = *pSet->aSstF[fSet.nSstF];
+      *fSet.aSttF[fSet.nSttF] = *pSet->aSttF[fSet.nSttF];
     }
 
     if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) {
@@ -634,28 +639,28 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
       *pDFileSet->pHeadF = *pSet->pHeadF;
       *pDFileSet->pDataF = *pSet->pDataF;
       *pDFileSet->pSmaF = *pSet->pSmaF;
-      // sst
-      if (pSet->nSstF > pDFileSet->nSstF) {
-        ASSERT(pSet->nSstF == pDFileSet->nSstF + 1);
+      // stt
+      if (pSet->nSttF > pDFileSet->nSttF) {
+        ASSERT(pSet->nSttF == pDFileSet->nSttF + 1);
 
-        pDFileSet->aSstF[pDFileSet->nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
-        if (pDFileSet->aSstF[pDFileSet->nSstF] == NULL) {
+        pDFileSet->aSttF[pDFileSet->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+        if (pDFileSet->aSttF[pDFileSet->nSttF] == NULL) {
           code = TSDB_CODE_OUT_OF_MEMORY;
           goto _exit;
         }
-        *pDFileSet->aSstF[pDFileSet->nSstF] = *pSet->aSstF[pSet->nSstF - 1];
-        pDFileSet->nSstF++;
-      } else if (pSet->nSstF < pDFileSet->nSstF) {
-        ASSERT(pSet->nSstF == 1);
-        for (int32_t iSst = 1; iSst < pDFileSet->nSstF; iSst++) {
-          taosMemoryFree(pDFileSet->aSstF[iSst]);
+        *pDFileSet->aSttF[pDFileSet->nSttF] = *pSet->aSttF[pSet->nSttF - 1];
+        pDFileSet->nSttF++;
+      } else if (pSet->nSttF < pDFileSet->nSttF) {
+        ASSERT(pSet->nSttF == 1);
+        for (int32_t iStt = 1; iStt < pDFileSet->nSttF; iStt++) {
+          taosMemoryFree(pDFileSet->aSttF[iStt]);
         }
 
-        *pDFileSet->aSstF[0] = *pSet->aSstF[0];
-        pDFileSet->nSstF = 1;
+        *pDFileSet->aSttF[0] = *pSet->aSttF[0];
+        pDFileSet->nSttF = 1;
       } else {
-        for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
-          *pDFileSet->aSstF[iSst] = *pSet->aSstF[iSst];
+        for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+          *pDFileSet->aSttF[iStt] = *pSet->aSttF[iStt];
         }
       }
 
@@ -663,8 +668,8 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
     }
   }
 
-  ASSERT(pSet->nSstF == 1);
-  SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSstF = 1};
+  ASSERT(pSet->nSttF == 1);
+  SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSttF = 1};
 
   // head
   fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
@@ -690,13 +695,13 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) {
   }
   *fSet.pSmaF = *pSet->pSmaF;
 
-  // sst
-  fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
-  if (fSet.aSstF[0] == NULL) {
+  // stt
+  fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+  if (fSet.aSttF[0] == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _exit;
   }
-  *fSet.aSstF[0] = *pSet->aSstF[0];
+  *fSet.aSttF[0] = *pSet->aSttF[0];
 
   if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
@@ -864,81 +869,81 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
       pSetOld->pSmaF->size = pSetNew->pSmaF->size;
     }
 
-    // sst
+    // stt
     if (sameDisk) {
-      if (pSetNew->nSstF > pSetOld->nSstF) {
-        ASSERT(pSetNew->nSstF = pSetOld->nSstF + 1);
-        pSetOld->aSstF[pSetOld->nSstF] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
-        if (pSetOld->aSstF[pSetOld->nSstF] == NULL) {
+      if (pSetNew->nSttF > pSetOld->nSttF) {
+        ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1);
+        pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+        if (pSetOld->aSttF[pSetOld->nSttF] == NULL) {
           code = TSDB_CODE_OUT_OF_MEMORY;
           goto _err;
         }
-        *pSetOld->aSstF[pSetOld->nSstF] = *pSetNew->aSstF[pSetOld->nSstF];
-        pSetOld->aSstF[pSetOld->nSstF]->nRef = 1;
-        pSetOld->nSstF++;
-      } else if (pSetNew->nSstF < pSetOld->nSstF) {
-        ASSERT(pSetNew->nSstF == 1);
-        for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
-          SSstFile *pSstFile = pSetOld->aSstF[iSst];
-          nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1);
+        *pSetOld->aSttF[pSetOld->nSttF] = *pSetNew->aSttF[pSetOld->nSttF];
+        pSetOld->aSttF[pSetOld->nSttF]->nRef = 1;
+        pSetOld->nSttF++;
+      } else if (pSetNew->nSttF < pSetOld->nSttF) {
+        ASSERT(pSetNew->nSttF == 1);
+        for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) {
+          SSttFile *pSttFile = pSetOld->aSttF[iStt];
+          nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1);
           if (nRef == 0) {
-            tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname);
+            tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname);
             taosRemoveFile(fname);
-            taosMemoryFree(pSstFile);
+            taosMemoryFree(pSttFile);
           }
-          pSetOld->aSstF[iSst] = NULL;
+          pSetOld->aSttF[iStt] = NULL;
         }
 
-        pSetOld->nSstF = 1;
-        pSetOld->aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
-        if (pSetOld->aSstF[0] == NULL) {
+        pSetOld->nSttF = 1;
+        pSetOld->aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+        if (pSetOld->aSttF[0] == NULL) {
           code = TSDB_CODE_OUT_OF_MEMORY;
           goto _err;
         }
-        *pSetOld->aSstF[0] = *pSetNew->aSstF[0];
-        pSetOld->aSstF[0]->nRef = 1;
+        *pSetOld->aSttF[0] = *pSetNew->aSttF[0];
+        pSetOld->aSttF[0]->nRef = 1;
       } else {
-        for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
-          if (pSetOld->aSstF[iSst]->commitID != pSetNew->aSstF[iSst]->commitID) {
-            SSstFile *pSstFile = pSetOld->aSstF[iSst];
-            nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1);
+        for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) {
+          if (pSetOld->aSttF[iStt]->commitID != pSetNew->aSttF[iStt]->commitID) {
+            SSttFile *pSttFile = pSetOld->aSttF[iStt];
+            nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1);
             if (nRef == 0) {
-              tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname);
+              tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname);
               taosRemoveFile(fname);
-              taosMemoryFree(pSstFile);
+              taosMemoryFree(pSttFile);
             }
 
-            pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
-            if (pSetOld->aSstF[iSst] == NULL) {
+            pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+            if (pSetOld->aSttF[iStt] == NULL) {
               code = TSDB_CODE_OUT_OF_MEMORY;
               goto _err;
             }
-            *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst];
-            pSetOld->aSstF[iSst]->nRef = 1;
+            *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt];
+            pSetOld->aSttF[iStt]->nRef = 1;
           } else {
-            ASSERT(pSetOld->aSstF[iSst]->size == pSetOld->aSstF[iSst]->size);
-            ASSERT(pSetOld->aSstF[iSst]->offset == pSetOld->aSstF[iSst]->offset);
+            ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size);
+            ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset);
           }
         }
       }
     } else {
-      ASSERT(pSetOld->nSstF == pSetNew->nSstF);
-      for (int32_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
-        SSstFile *pSstFile = pSetOld->aSstF[iSst];
-        nRef = atomic_sub_fetch_32(&pSstFile->nRef, 1);
+      ASSERT(pSetOld->nSttF == pSetNew->nSttF);
+      for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) {
+        SSttFile *pSttFile = pSetOld->aSttF[iStt];
+        nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1);
         if (nRef == 0) {
-          tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSstFile, fname);
+          tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname);
           taosRemoveFile(fname);
-          taosMemoryFree(pSstFile);
+          taosMemoryFree(pSttFile);
         }
 
-        pSetOld->aSstF[iSst] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
-        if (pSetOld->aSstF[iSst] == NULL) {
+        pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+        if (pSetOld->aSttF[iStt] == NULL) {
           code = TSDB_CODE_OUT_OF_MEMORY;
           goto _err;
         }
-        *pSetOld->aSstF[iSst] = *pSetNew->aSstF[iSst];
-        pSetOld->aSstF[iSst]->nRef = 1;
+        *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt];
+        pSetOld->aSttF[iStt]->nRef = 1;
       }
     }
 
@@ -972,12 +977,12 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
       taosMemoryFree(pSetOld->pSmaF);
     }
 
-    for (int8_t iSst = 0; iSst < pSetOld->nSstF; iSst++) {
-      nRef = atomic_sub_fetch_32(&pSetOld->aSstF[iSst]->nRef, 1);
+    for (int8_t iStt = 0; iStt < pSetOld->nSttF; iStt++) {
+      nRef = atomic_sub_fetch_32(&pSetOld->aSttF[iStt]->nRef, 1);
       if (nRef == 0) {
-        tsdbSstFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSstF[iSst], fname);
+        tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSttF[iStt], fname);
         taosRemoveFile(fname);
-        taosMemoryFree(pSetOld->aSstF[iSst]);
+        taosMemoryFree(pSetOld->aSttF[iStt]);
       }
     }
 
@@ -985,7 +990,7 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
     continue;
 
   _add_new:
-    fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSstF = 1};
+    fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSttF = 1};
 
     // head
     fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile));
@@ -1014,15 +1019,15 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) {
     *fSet.pSmaF = *pSetNew->pSmaF;
     fSet.pSmaF->nRef = 1;
 
-    // sst
-    ASSERT(pSetNew->nSstF == 1);
-    fSet.aSstF[0] = (SSstFile *)taosMemoryMalloc(sizeof(SSstFile));
-    if (fSet.aSstF[0] == NULL) {
+    // stt
+    ASSERT(pSetNew->nSttF == 1);
+    fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile));
+    if (fSet.aSttF[0] == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
       goto _err;
     }
-    *fSet.aSstF[0] = *pSetNew->aSstF[0];
-    fSet.aSstF[0]->nRef = 1;
+    *fSet.aSttF[0] = *pSetNew->aSttF[0];
+    fSet.aSttF[0]->nRef = 1;
 
     if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) {
       code = TSDB_CODE_OUT_OF_MEMORY;
@@ -1070,8 +1075,8 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) {
     nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1);
     ASSERT(nRef > 0);
 
-    for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
-      nRef = atomic_fetch_add_32(&pSet->aSstF[iSst]->nRef, 1);
+    for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+      nRef = atomic_fetch_add_32(&pSet->aSttF[iStt]->nRef, 1);
       ASSERT(nRef > 0);
     }
 
@@ -1129,14 +1134,14 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) {
       taosMemoryFree(pSet->pSmaF);
     }
 
-    // sst
-    for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
-      nRef = atomic_sub_fetch_32(&pSet->aSstF[iSst]->nRef, 1);
+    // stt
+    for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+      nRef = atomic_sub_fetch_32(&pSet->aSttF[iStt]->nRef, 1);
       ASSERT(nRef >= 0);
       if (nRef == 0) {
-        tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname);
+        tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
         taosRemoveFile(fname);
-        taosMemoryFree(pSet->aSstF[iSst]);
+        taosMemoryFree(pSet->aSttF[iStt]);
         /* code */
       }
     }

source/dnode/vnode/src/tsdb/tsdbFile.c

@@ -53,22 +53,22 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) {
   return n;
 }
 
-int32_t tPutSstFile(uint8_t *p, SSstFile *pSstFile) {
+int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile) {
   int32_t n = 0;
 
-  n += tPutI64v(p ? p + n : p, pSstFile->commitID);
-  n += tPutI64v(p ? p + n : p, pSstFile->size);
-  n += tPutI64v(p ? p + n : p, pSstFile->offset);
+  n += tPutI64v(p ? p + n : p, pSttFile->commitID);
+  n += tPutI64v(p ? p + n : p, pSttFile->size);
+  n += tPutI64v(p ? p + n : p, pSttFile->offset);
 
   return n;
 }
 
-static int32_t tGetSstFile(uint8_t *p, SSstFile *pSstFile) {
+static int32_t tGetSttFile(uint8_t *p, SSttFile *pSttFile) {
   int32_t n = 0;
 
-  n += tGetI64v(p + n, &pSstFile->commitID);
-  n += tGetI64v(p + n, &pSstFile->size);
-  n += tGetI64v(p + n, &pSstFile->offset);
+  n += tGetI64v(p + n, &pSttFile->commitID);
+  n += tGetI64v(p + n, &pSttFile->size);
+  n += tGetI64v(p + n, &pSttFile->offset);
 
   return n;
 }
@@ -102,9 +102,9 @@ void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF,
            TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pDataF->commitID, ".data");
 }
 
-void tsdbSstFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSstFile *pSstF, char fname[]) {
+void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) {
   snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did),
-           TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pSstF->commitID, ".sst");
+           TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pSttF->commitID, ".stt");
 }
 
 void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) {
@@ -148,7 +148,7 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) {
   }
 
   // ftruncate
-  if (taosFtruncateFile(pFD, size) < 0) {
+  if (taosFtruncateFile(pFD, LOGIC_TO_FILE_SIZE(size, TSDB_DEFAULT_PAGE_SIZE)) < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
@@ -194,10 +194,10 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) {
   n += tPutDataFile(p ? p + n : p, pSet->pDataF);
   n += tPutSmaFile(p ? p + n : p, pSet->pSmaF);
 
-  // sst
-  n += tPutU8(p ? p + n : p, pSet->nSstF);
-  for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
-    n += tPutSstFile(p ? p + n : p, pSet->aSstF[iSst]);
+  // stt
+  n += tPutU8(p ? p + n : p, pSet->nSttF);
+  for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+    n += tPutSttFile(p ? p + n : p, pSet->aSttF[iStt]);
   }
 
   return n;
@@ -234,15 +234,15 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) {
   pSet->pSmaF->nRef = 1;
   n += tGetSmaFile(p + n, pSet->pSmaF);
 
-  // sst
-  n += tGetU8(p + n, &pSet->nSstF);
-  for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
-    pSet->aSstF[iSst] = (SSstFile *)taosMemoryCalloc(1, sizeof(SSstFile));
-    if (pSet->aSstF[iSst] == NULL) {
+  // stt
+  n += tGetU8(p + n, &pSet->nSttF);
+  for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+    pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile));
+    if (pSet->aSttF[iStt] == NULL) {
       return -1;
     }
-    pSet->aSstF[iSst]->nRef = 1;
-    n += tGetSstFile(p + n, pSet->aSstF[iSst]);
+    pSet->aSttF[iStt]->nRef = 1;
+    n += tGetSttFile(p + n, pSet->aSttF[iStt]);
   }
 
   return n;

source/dnode/vnode/src/tsdb/tsdbMergeTree.c

@@ -16,14 +16,14 @@
 #include "tsdb.h"
 
 // SLDataIter =================================================
-typedef struct SLDataIter {
+struct SLDataIter {
   SRBTreeNode   node;
-  SSstBlk      *pSstBlk;
+  SSttBlk      *pSttBlk;
   SDataFReader *pReader;
-  int32_t       iSst;
+  int32_t       iStt;
   int8_t        backward;
-  SArray       *aSstBlk;
-  int32_t       iSstBlk;
+  SArray       *aSttBlk;
+  int32_t       iSttBlk;
   SBlockData    bData[2];
   int32_t       loadIndex;
   int32_t       iRow;
@@ -31,18 +31,16 @@ typedef struct SLDataIter {
   uint64_t      uid;
   STimeWindow   timeWindow;
   SVersionRange verRange;
-} SLDataIter;
+};
 
-static SBlockData* getCurrentBlock(SLDataIter* pIter) {
-  return &pIter->bData[pIter->loadIndex];
-}
+static SBlockData *getCurrentBlock(SLDataIter *pIter) { return &pIter->bData[pIter->loadIndex]; }
 
-static SBlockData* getNextBlock(SLDataIter* pIter) {
+static SBlockData *getNextBlock(SLDataIter *pIter) {
   pIter->loadIndex ^= 1;
   return getCurrentBlock(pIter);
 }
 
-int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iSst, int8_t backward, uint64_t suid,
+int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid,
     uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange) {
   int32_t code = 0;
   *pIter = taosMemoryCalloc(1, sizeof(SLDataIter));
@@ -52,13 +50,11 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
   }
 
   (*pIter)->uid = uid;
-  (*pIter)->iSst = iSst;
   (*pIter)->pReader = pReader;
-  (*pIter)->verRange = *pRange;
+  (*pIter)->iStt = iStt;
   (*pIter)->backward = backward;
-  (*pIter)->timeWindow = *pTimeWindow;
-  (*pIter)->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
-  if ((*pIter)->aSstBlk == NULL) {
+  (*pIter)->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+  if ((*pIter)->aSttBlk == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _exit;
   }
@@ -73,18 +69,18 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
     goto _exit;
   }
 
-  code = tsdbReadSstBlk(pReader, iSst, (*pIter)->aSstBlk);
+  code = tsdbReadSttBlk(pReader, iStt, (*pIter)->aSttBlk);
   if (code) {
     goto _exit;
   }
 
-  size_t size = taosArrayGetSize((*pIter)->aSstBlk);
+  size_t size = taosArrayGetSize((*pIter)->aSttBlk);
 
   // find the start block
   int32_t index = -1;
   if (!backward) {  // asc
     for (int32_t i = 0; i < size; ++i) {
-      SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i);
+      SSttBlk *p = taosArrayGet((*pIter)->aSttBlk, i);
       if (p->suid != suid) {
         continue;
       }
@@ -96,7 +92,7 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
     }
   } else {  // desc
     for (int32_t i = size - 1; i >= 0; --i) {
-      SSstBlk *p = taosArrayGet((*pIter)->aSstBlk, i);
+      SSttBlk *p = taosArrayGet((*pIter)->aSttBlk, i);
       if (p->suid != suid) {
         continue;
       }
@@ -108,9 +104,9 @@ int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t
     }
   }
 
-  (*pIter)->iSstBlk = index;
+  (*pIter)->iSttBlk = index;
   if (index != -1) {
-    (*pIter)->pSstBlk = taosArrayGet((*pIter)->aSstBlk, (*pIter)->iSstBlk);
+    (*pIter)->pSttBlk = taosArrayGet((*pIter)->aSttBlk, (*pIter)->iSttBlk);
   }
 
 _exit:
@@ -120,20 +116,18 @@ _exit:
 void tLDataIterClose(SLDataIter *pIter) {
   tBlockDataDestroy(&pIter->bData[0], 1);
   tBlockDataDestroy(&pIter->bData[1], 1);
-  taosArrayDestroy(pIter->aSstBlk);
+  taosArrayDestroy(pIter->aSttBlk);
   taosMemoryFree(pIter);
 }
 
-extern int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData);
-
 void tLDataIterNextBlock(SLDataIter *pIter) {
   int32_t step = pIter->backward ? -1 : 1;
-  pIter->iSstBlk += step;
+  pIter->iSttBlk += step;
 
   int32_t index = -1;
-  size_t  size = taosArrayGetSize(pIter->aSstBlk);
-  for (int32_t i = pIter->iSstBlk; i < size && i >= 0; i += step) {
-    SSstBlk *p = taosArrayGet(pIter->aSstBlk, i);
+  size_t  size = taosArrayGetSize(pIter->aSttBlk);
+  for (int32_t i = pIter->iSttBlk; i < size && i >= 0; i += step) {
+    SSttBlk *p = taosArrayGet(pIter->aSttBlk, i);
     if ((!pIter->backward) && p->minUid > pIter->uid) {
       break;
     }
@@ -171,9 +165,9 @@ void tLDataIterNextBlock(SLDataIter *pIter) {
   }
 
   if (index == -1) {
-    pIter->pSstBlk = NULL;
+    pIter->pSttBlk = NULL;
   } else {
-    pIter->pSstBlk = (SSstBlk *)taosArrayGet(pIter->aSstBlk, pIter->iSstBlk);
+    pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
   }
 }
 
@@ -182,7 +176,7 @@ static void findNextValidRow(SLDataIter *pIter) {
 
   bool        hasVal = false;
   int32_t     i = pIter->iRow;
-  SBlockData* pBlockData = getCurrentBlock(pIter);
+  SBlockData *pBlockData = getCurrentBlock(pIter);
 
   for (; i < pBlockData->nRow && i >= 0; i += step) {
     if (pBlockData->aUid != NULL) {
@@ -238,16 +232,16 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
   int32_t step = pIter->backward ? -1 : 1;
 
   // no qualified last file block in current file, no need to fetch row
-  if (pIter->pSstBlk == NULL) {
+  if (pIter->pSttBlk == NULL) {
     return false;
   }
 
-  int32_t iBlockL = pIter->iSstBlk;
-  SBlockData* pBlockData = getCurrentBlock(pIter);
+  int32_t     iBlockL = pIter->iSttBlk;
+  SBlockData *pBlockData = getCurrentBlock(pIter);
 
-  if (pBlockData->nRow == 0 && pIter->pSstBlk != NULL) {  // current block not loaded yet
+  if (pBlockData->nRow == 0 && pIter->pSttBlk != NULL) {  // current block not loaded yet
     pBlockData = getNextBlock(pIter);
-    code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData);
+    code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlockData);
     if (code != TSDB_CODE_SUCCESS) {
       goto _exit;
     }
@@ -262,16 +256,16 @@ bool tLDataIterNextRow(SLDataIter *pIter) {
 
     if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) {
       tLDataIterNextBlock(pIter);
-      if (pIter->pSstBlk == NULL) {  // no more data
+      if (pIter->pSttBlk == NULL) {  // no more data
         goto _exit;
       }
     } else {
       break;
     }
 
-    if (iBlockL != pIter->iSstBlk) {
+    if (iBlockL != pIter->iSttBlk) {
       pBlockData = getNextBlock(pIter);
-      code = tsdbReadSstBlockEx(pIter->pReader, pIter->iSst, pIter->pSstBlk, pBlockData);
+      code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, pBlockData);
       if (code) {
         goto _exit;
       }
@@ -288,7 +282,7 @@ _exit:
     terrno = code;
   }
 
-  return (code == TSDB_CODE_SUCCESS) && (pIter->pSstBlk != NULL);
+  return (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL);
 }
 
 SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; }
@@ -328,8 +322,8 @@ int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFRead
   tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn);
   int32_t code = TSDB_CODE_OUT_OF_MEMORY;
 
-  struct SLDataIter *pIterList[TSDB_DEFAULT_LAST_FILE] = {0};
-  for (int32_t i = 0; i < pFReader->pSet->nSstF; ++i) {  // open all last file
+  struct SLDataIter *pIterList[TSDB_DEFAULT_STT_FILE] = {0};
+  for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) {  // open all last file
     code = tLDataIterOpen(&pIterList[i], pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange);
     if (code != TSDB_CODE_SUCCESS) {
       goto _end;

source/dnode/vnode/src/tsdb/tsdbRead.c

@@ -621,7 +621,7 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockN
     }
   }
 
-  pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nSstF;
+  pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nSttF;
   int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks;
 
   double el = (taosGetTimestampUs() - st) / 1000.0;
@@ -1163,7 +1163,7 @@ static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBloc
   bool overlapWithlastBlock = false;
 #if 0
   if (taosArrayGetSize(pLastBlockReader->pSstBlk) > 0 && (pLastBlockReader->currentBlockIndex != -1)) {
-    SSstBlk* pSstBlk = taosArrayGet(pLastBlockReader->pSstBlk, pLastBlockReader->currentBlockIndex);
+    SSttBlk* pSstBlk = taosArrayGet(pLastBlockReader->pSstBlk, pLastBlockReader->currentBlockIndex);
     overlapWithlastBlock = !(pBlock->maxKey.ts < pSstBlk->minKey || pBlock->minKey.ts > pSstBlk->maxKey);
   }
 #endif
@@ -1850,7 +1850,7 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum
 static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); }
 
 static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) {
-  while(1) {
+  while (1) {
     bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree);
     if (!hasVal) {
       return false;
@@ -2204,7 +2204,7 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) {
       return code;
     }
 
-    if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSstF > 0) {
+    if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSttF > 0) {
       code = doLoadFileBlock(pReader, pIndexList, pBlockNum);
       if (code != TSDB_CODE_SUCCESS) {
         taosArrayDestroy(pIndexList);
@@ -2538,7 +2538,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) {
         if (hasNext) {  // check for the next block in the block accessed order list
           initBlockDumpInfo(pReader, pBlockIter);
         } else {
-          if (pReader->status.pCurrentFileset->nSstF > 0) {
+          if (pReader->status.pCurrentFileset->nSttF > 0) {
             // data blocks in current file are exhausted, let's try the next file now
             tBlockDataReset(&pReader->status.fileBlockData);
             resetDataBlockIterator(pBlockIter, pReader->order);

source/dnode/vnode/src/tsdb/tsdbReaderWriter.c

@@ -15,804 +15,1011 @@
 
 #include "tsdb.h"
 
-// SDelFWriter ====================================================
-int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) {
+// =============== PAGE-WISE FILE ===============
+static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) {
   int32_t  code = 0;
-  char         fname[TSDB_FILENAME_LEN];
-  char         hdr[TSDB_FHDR_SIZE] = {0};
-  SDelFWriter *pDelFWriter;
-  int64_t      n;
+  STsdbFD *pFD;
 
-  // alloc
-  pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter));
-  if (pDelFWriter == NULL) {
+  *ppFD = NULL;
+
+  pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD) + strlen(path) + 1);
+  if (pFD == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
+    goto _exit;
   }
-  pDelFWriter->pTsdb = pTsdb;
-  pDelFWriter->fDel = *pFile;
 
-  tsdbDelFileName(pTsdb, pFile, fname);
-  pDelFWriter->pWriteH = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE);
-  if (pDelFWriter->pWriteH == NULL) {
+  pFD->path = (char *)&pFD[1];
+  strcpy(pFD->path, path);
+  pFD->szPage = szPage;
+  pFD->flag = flag;
+  pFD->pFD = taosOpenFile(path, flag);
+  if (pFD->pFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+    goto _exit;
   }
-
-  // update header
-  n = taosWriteFile(pDelFWriter->pWriteH, &hdr, TSDB_FHDR_SIZE);
-  if (n < 0) {
+  pFD->szPage = szPage;
+  pFD->pgno = 0;
+  pFD->pBuf = taosMemoryCalloc(1, szPage);
+  if (pFD->pBuf == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    taosMemoryFree(pFD);
+    goto _exit;
+  }
+  if (taosStatFile(path, &pFD->szFile, NULL) < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+    goto _exit;
   }
+  ASSERT(pFD->szFile % szPage == 0);
+  pFD->szFile = pFD->szFile / szPage;
+  *ppFD = pFD;
 
-  pDelFWriter->fDel.size = TSDB_FHDR_SIZE;
-  pDelFWriter->fDel.offset = 0;
-
-  *ppWriter = pDelFWriter;
+_exit:
   return code;
+}
 
-_err:
-  tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
-  *ppWriter = NULL;
-  return code;
+static void tsdbCloseFile(STsdbFD **ppFD) {
+  STsdbFD *pFD = *ppFD;
+  taosMemoryFree(pFD->pBuf);
+  taosCloseFile(&pFD->pFD);
+  taosMemoryFree(pFD);
+  *ppFD = NULL;
 }
 
-int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) {
+static int32_t tsdbWriteFilePage(STsdbFD *pFD) {
   int32_t code = 0;
-  SDelFWriter *pWriter = *ppWriter;
-  STsdb       *pTsdb = pWriter->pTsdb;
 
-  // sync
-  if (sync && taosFsyncFile(pWriter->pWriteH) < 0) {
+  if (pFD->pgno > 0) {
+    int64_t n = taosLSeekFile(pFD->pFD, PAGE_OFFSET(pFD->pgno, pFD->szPage), SEEK_SET);
+    if (n < 0) {
       code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+      goto _exit;
     }
 
-  // close
-  if (taosCloseFile(&pWriter->pWriteH) < 0) {
+    taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage);
+
+    n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage);
+    if (n < 0) {
       code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+      goto _exit;
     }
 
-  for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) {
-    tFree(pWriter->aBuf[iBuf]);
+    if (pFD->szFile < pFD->pgno) {
+      pFD->szFile = pFD->pgno;
     }
-  taosMemoryFree(pWriter);
-
-  *ppWriter = NULL;
-  return code;
+  }
+  pFD->pgno = 0;
 
-_err:
-  tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+_exit:
   return code;
 }
 
-int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) {
+static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) {
   int32_t code = 0;
-  int64_t size;
-  int64_t n;
-
-  // prepare
-  size = sizeof(uint32_t);
-  for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) {
-    size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData));
-  }
-  size += sizeof(TSCKSUM);
 
-  // alloc
-  code = tRealloc(&pWriter->aBuf[0], size);
-  if (code) goto _err;
+  ASSERT(pgno <= pFD->szFile);
 
-  // build
-  n = 0;
-  n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
-  for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) {
-    n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData));
+  // seek
+  int64_t offset = PAGE_OFFSET(pgno, pFD->szPage);
+  int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET);
+  if (n < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _exit;
   }
-  taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
 
-  ASSERT(n + sizeof(TSCKSUM) == size);
-
-  // write
-  n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size);
+  // read
+  n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage);
   if (n < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+    goto _exit;
+  } else if (n < pFD->szPage) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    goto _exit;
   }
 
-  ASSERT(n == size);
-
-  // update
-  pDelIdx->offset = pWriter->fDel.size;
-  pDelIdx->size = size;
-  pWriter->fDel.size += size;
+  // check
+  if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) {
+    code = TSDB_CODE_FILE_CORRUPTED;
+    goto _exit;
+  }
 
-  return code;
+  pFD->pgno = pgno;
 
-_err:
-  tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+_exit:
   return code;
 }
 
-int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) {
+static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) {
   int32_t code = 0;
-  int64_t  size;
-  int64_t  n;
-  SDelIdx *pDelIdx;
+  int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
+  int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
+  int64_t bOffset = fOffset % pFD->szPage;
+  int64_t n = 0;
+
+  do {
+    if (pFD->pgno != pgno) {
+      code = tsdbWriteFilePage(pFD);
+      if (code) goto _exit;
 
-  // prepare
-  size = sizeof(uint32_t);
-  for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) {
-    size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx));
+      if (pgno <= pFD->szFile) {
+        code = tsdbReadFilePage(pFD, pgno);
+        if (code) goto _exit;
+      } else {
+        pFD->pgno = pgno;
+      }
     }
-  size += sizeof(TSCKSUM);
 
-  // alloc
-  code = tRealloc(&pWriter->aBuf[0], size);
-  if (code) goto _err;
+    int64_t nWrite = TMIN(PAGE_CONTENT_SIZE(pFD->szPage) - bOffset, size - n);
+    memcpy(pFD->pBuf + bOffset, pBuf + n, nWrite);
 
-  // build
-  n = 0;
-  n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
-  for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) {
-    n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx));
-  }
-  taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
+    pgno++;
+    bOffset = 0;
+    n += nWrite;
+  } while (n < size);
 
-  ASSERT(n + sizeof(TSCKSUM) == size);
+_exit:
+  return code;
+}
 
-  // write
-  n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) {
+  int32_t code = 0;
+  int64_t n = 0;
+  int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage);
+  int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage);
+  int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage);
+  int64_t bOffset = fOffset % pFD->szPage;
+
+  ASSERT(pgno && pgno <= pFD->szFile);
+  ASSERT(bOffset < szPgCont);
+
+  while (n < size) {
+    if (pFD->pgno != pgno) {
+      code = tsdbReadFilePage(pFD, pgno);
+      if (code) goto _exit;
     }
 
-  // update
-  pWriter->fDel.offset = pWriter->fDel.size;
-  pWriter->fDel.size += size;
+    int64_t nRead = TMIN(szPgCont - bOffset, size - n);
+    memcpy(pBuf + n, pFD->pBuf + bOffset, nRead);
 
-  return code;
+    n += nRead;
+    pgno++;
+    bOffset = 0;
+  }
 
-_err:
-  tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+_exit:
   return code;
 }
 
-int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) {
+static int32_t tsdbFsyncFile(STsdbFD *pFD) {
   int32_t code = 0;
-  char    hdr[TSDB_FHDR_SIZE];
-  int64_t size = TSDB_FHDR_SIZE;
-  int64_t n;
-
-  // build
-  memset(hdr, 0, size);
-  tPutDelFile(hdr, &pWriter->fDel);
-  taosCalcChecksumAppend(0, hdr, size);
 
-  // seek
-  if (taosLSeekFile(pWriter->pWriteH, 0, SEEK_SET) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  code = tsdbWriteFilePage(pFD);
+  if (code) goto _exit;
 
-  // write
-  n = taosWriteFile(pWriter->pWriteH, hdr, size);
-  if (n < 0) {
+  if (taosFsyncFile(pFD->pFD) < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+    goto _exit;
   }
 
-  return code;
-
-_err:
-  tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+_exit:
   return code;
 }
 
-// SDelFReader ====================================================
-struct SDelFReader {
-  STsdb    *pTsdb;
-  SDelFile  fDel;
-  TdFilePtr pReadH;
-
-  uint8_t *aBuf[1];
-};
-
-int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) {
+// SDataFWriter ====================================================
+int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) {
   int32_t       code = 0;
-  char         fname[TSDB_FILENAME_LEN];
-  SDelFReader *pDelFReader;
+  int32_t       flag;
   int64_t       n;
+  int32_t       szPage = TSDB_DEFAULT_PAGE_SIZE;
+  SDataFWriter *pWriter = NULL;
+  char          fname[TSDB_FILENAME_LEN];
+  char          hdr[TSDB_FHDR_SIZE] = {0};
 
   // alloc
-  pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader));
-  if (pDelFReader == NULL) {
+  pWriter = taosMemoryCalloc(1, sizeof(*pWriter));
+  if (pWriter == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
+  pWriter->pTsdb = pTsdb;
+  pWriter->wSet = (SDFileSet){.diskId = pSet->diskId,
+                              .fid = pSet->fid,
+                              .pHeadF = &pWriter->fHead,
+                              .pDataF = &pWriter->fData,
+                              .pSmaF = &pWriter->fSma,
+                              .nSttF = pSet->nSttF};
+  pWriter->fHead = *pSet->pHeadF;
+  pWriter->fData = *pSet->pDataF;
+  pWriter->fSma = *pSet->pSmaF;
+  for (int8_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+    pWriter->wSet.aSttF[iStt] = &pWriter->fStt[iStt];
+    pWriter->fStt[iStt] = *pSet->aSttF[iStt];
+  }
 
-  // open impl
-  pDelFReader->pTsdb = pTsdb;
-  pDelFReader->fDel = *pFile;
+  // head
+  flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
+  tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname);
+  code = tsdbOpenFile(fname, szPage, flag, &pWriter->pHeadFD);
+  if (code) goto _err;
 
-  tsdbDelFileName(pTsdb, pFile, fname);
-  pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ);
-  if (pDelFReader->pReadH == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    taosMemoryFree(pDelFReader);
-    goto _err;
+  code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE);
+  if (code) goto _err;
+  pWriter->fHead.size += TSDB_FHDR_SIZE;
+
+  // data
+  if (pWriter->fData.size == 0) {
+    flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
+  } else {
+    flag = TD_FILE_READ | TD_FILE_WRITE;
+  }
+  tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname);
+  code = tsdbOpenFile(fname, szPage, flag, &pWriter->pDataFD);
+  if (code) goto _err;
+  if (pWriter->fData.size == 0) {
+    code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE);
+    if (code) goto _err;
+    pWriter->fData.size += TSDB_FHDR_SIZE;
   }
 
-_exit:
-  *ppReader = pDelFReader;
+  // sma
+  if (pWriter->fSma.size == 0) {
+    flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
+  } else {
+    flag = TD_FILE_READ | TD_FILE_WRITE;
+  }
+  tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname);
+  code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSmaFD);
+  if (code) goto _err;
+  if (pWriter->fSma.size == 0) {
+    code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE);
+    if (code) goto _err;
+
+    pWriter->fSma.size += TSDB_FHDR_SIZE;
+  }
+
+  // stt
+  ASSERT(pWriter->fStt[pSet->nSttF - 1].size == 0);
+  flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
+  tsdbSttFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fStt[pSet->nSttF - 1], fname);
+  code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSttFD);
+  if (code) goto _err;
+  code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE);
+  if (code) goto _err;
+  pWriter->fStt[pWriter->wSet.nSttF - 1].size += TSDB_FHDR_SIZE;
+
+  *ppWriter = pWriter;
   return code;
 
 _err:
-  tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
-  *ppReader = NULL;
+  tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  *ppWriter = NULL;
   return code;
 }
 
-int32_t tsdbDelFReaderClose(SDelFReader **ppReader) {
+int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) {
   int32_t code = 0;
-  SDelFReader *pReader = *ppReader;
+  STsdb  *pTsdb = NULL;
 
-  if (pReader) {
-    if (taosCloseFile(&pReader->pReadH) < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _exit;
-    }
-    for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) {
-      tFree(pReader->aBuf[iBuf]);
-    }
-    taosMemoryFree(pReader);
-  }
-  *ppReader = NULL;
+  if (*ppWriter == NULL) goto _exit;
 
-_exit:
-  return code;
-}
+  pTsdb = (*ppWriter)->pTsdb;
+  if (sync) {
+    code = tsdbFsyncFile((*ppWriter)->pHeadFD);
+    if (code) goto _err;
 
-int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) {
-  int32_t code = 0;
-  int64_t offset = pDelIdx->offset;
-  int64_t size = pDelIdx->size;
-  int64_t n;
+    code = tsdbFsyncFile((*ppWriter)->pDataFD);
+    if (code) goto _err;
 
-  taosArrayClear(aDelData);
+    code = tsdbFsyncFile((*ppWriter)->pSmaFD);
+    if (code) goto _err;
 
-  // seek
-  if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+    code = tsdbFsyncFile((*ppWriter)->pSttFD);
+    if (code) goto _err;
   }
 
-  // alloc
-  code = tRealloc(&pReader->aBuf[0], size);
-  if (code) goto _err;
+  tsdbCloseFile(&(*ppWriter)->pHeadFD);
+  tsdbCloseFile(&(*ppWriter)->pDataFD);
+  tsdbCloseFile(&(*ppWriter)->pSmaFD);
+  tsdbCloseFile(&(*ppWriter)->pSttFD);
 
-  // read
-  n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  } else if (n < size) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _err;
+  for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) {
+    tFree((*ppWriter)->aBuf[iBuf]);
   }
+  taosMemoryFree(*ppWriter);
+_exit:
+  *ppWriter = NULL;
+  return code;
 
-  // check
-  if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _err;
-  }
+_err:
+  tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  return code;
+}
 
-  // // decode
-  n = 0;
+int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) {
+  int32_t code = 0;
+  int64_t n;
+  char    hdr[TSDB_FHDR_SIZE];
 
-  uint32_t delimiter;
-  n += tGetU32(pReader->aBuf[0] + n, &delimiter);
-  while (n < size - sizeof(TSCKSUM)) {
-    SDelData delData;
-    n += tGetDelData(pReader->aBuf[0] + n, &delData);
+  // head ==============
+  memset(hdr, 0, TSDB_FHDR_SIZE);
+  tPutHeadFile(hdr, &pWriter->fHead);
+  code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE);
+  if (code) goto _err;
 
-    if (taosArrayPush(aDelData, &delData) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      goto _err;
-    }
-  }
+  // data ==============
+  memset(hdr, 0, TSDB_FHDR_SIZE);
+  tPutDataFile(hdr, &pWriter->fData);
+  code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE);
+  if (code) goto _err;
 
-  ASSERT(n == size - sizeof(TSCKSUM));
+  // sma ==============
+  memset(hdr, 0, TSDB_FHDR_SIZE);
+  tPutSmaFile(hdr, &pWriter->fSma);
+  code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE);
+  if (code) goto _err;
+
+  // stt ==============
+  memset(hdr, 0, TSDB_FHDR_SIZE);
+  tPutSttFile(hdr, &pWriter->fStt[pWriter->wSet.nSttF - 1]);
+  code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE);
+  if (code) goto _err;
 
   return code;
 
 _err:
-  tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
+int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) {
   int32_t    code = 0;
-  int32_t n;
-  int64_t offset = pReader->fDel.offset;
-  int64_t size = pReader->fDel.size - offset;
+  SHeadFile *pHeadFile = &pWriter->fHead;
+  int64_t    size;
+  int64_t    n;
 
-  taosArrayClear(aDelIdx);
+  // check
+  if (taosArrayGetSize(aBlockIdx) == 0) {
+    pHeadFile->offset = pHeadFile->size;
+    goto _exit;
+  }
 
-  // seek
-  if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+  // prepare
+  size = 0;
+  for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) {
+    size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx));
   }
 
   // alloc
-  code = tRealloc(&pReader->aBuf[0], size);
+  code = tRealloc(&pWriter->aBuf[0], size);
   if (code) goto _err;
 
-  // read
-  n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  } else if (n < size) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _err;
-  }
-
-  // check
-  if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _err;
-  }
-
-  // decode
+  // build
   n = 0;
-  uint32_t delimiter;
-  n += tGetU32(pReader->aBuf[0] + n, &delimiter);
-  ASSERT(delimiter == TSDB_FILE_DLMT);
-
-  while (n < size - sizeof(TSCKSUM)) {
-    SDelIdx delIdx;
-
-    n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx);
-
-    if (taosArrayPush(aDelIdx, &delIdx) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      goto _err;
-    }
+  for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) {
+    n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx));
   }
+  ASSERT(n == size);
+
+  // write
+  code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size);
+  if (code) goto _err;
 
-  ASSERT(n == size - sizeof(TSCKSUM));
+  // update
+  pHeadFile->offset = pHeadFile->size;
+  pHeadFile->size += size;
 
+_exit:
+  // tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d",
+  // TD_VID(pWriter->pTsdb->pVnode),
+  //           pHeadFile->offset, size, taosArrayGetSize(aBlockIdx));
   return code;
 
 _err:
-  tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-// SDataFReader ====================================================
-int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) {
+int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBlockIdx) {
   int32_t    code = 0;
-  SDataFReader *pReader;
-  char          fname[TSDB_FILENAME_LEN];
+  SHeadFile *pHeadFile = &pWriter->fHead;
+  int64_t    size;
+  int64_t    n;
 
-  // alloc
-  pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader));
-  if (pReader == NULL) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
-    goto _err;
-  }
-  pReader->pTsdb = pTsdb;
-  pReader->pSet = pSet;
+  ASSERT(mBlock->nItem > 0);
 
-  // open impl
-  // head
-  tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
-  pReader->pHeadFD = taosOpenFile(fname, TD_FILE_READ);
-  if (pReader->pHeadFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  // alloc
+  size = tPutMapData(NULL, mBlock);
+  code = tRealloc(&pWriter->aBuf[0], size);
+  if (code) goto _err;
 
-  // data
-  tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
-  pReader->pDataFD = taosOpenFile(fname, TD_FILE_READ);
-  if (pReader->pDataFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  // build
+  n = tPutMapData(pWriter->aBuf[0], mBlock);
 
-  // sma
-  tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
-  pReader->pSmaFD = taosOpenFile(fname, TD_FILE_READ);
-  if (pReader->pSmaFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  // write
+  code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size);
+  if (code) goto _err;
 
-  // sst
-  for (int32_t iSst = 0; iSst < pSet->nSstF; iSst++) {
-    tsdbSstFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSstF[iSst], fname);
-    pReader->aLastFD[iSst] = taosOpenFile(fname, TD_FILE_READ);
-    if (pReader->aLastFD[iSst] == NULL) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-  }
+  // update
+  pBlockIdx->offset = pHeadFile->size;
+  pBlockIdx->size = size;
+  pHeadFile->size += size;
 
-  *ppReader = pReader;
+  tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64
+            " size:%" PRId64 " nItem:%d",
+            TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid,
+            pBlockIdx->offset, pBlockIdx->size, mBlock->nItem);
   return code;
 
 _err:
-  tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
-  *ppReader = NULL;
+  tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
+int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) {
   int32_t   code = 0;
-  if (*ppReader == NULL) goto _exit;
+  SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1];
+  int64_t   size;
+  int64_t   n;
 
-  // head
-  if (taosCloseFile(&(*ppReader)->pHeadFD) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+  // check
+  if (taosArrayGetSize(aSttBlk) == 0) {
+    pSttFile->offset = pSttFile->size;
+    goto _exit;
   }
 
-  // data
-  if (taosCloseFile(&(*ppReader)->pDataFD) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+  // size
+  size = 0;
+  for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) {
+    size += tPutSttBlk(NULL, taosArrayGet(aSttBlk, iBlockL));
   }
 
-  // sma
-  if (taosCloseFile(&(*ppReader)->pSmaFD) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  // alloc
+  code = tRealloc(&pWriter->aBuf[0], size);
+  if (code) goto _err;
 
-  // sst
-  for (int32_t iSst = 0; iSst < (*ppReader)->pSet->nSstF; iSst++) {
-    if (taosCloseFile(&(*ppReader)->aLastFD[iSst]) < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
+  // encode
+  n = 0;
+  for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) {
+    n += tPutSttBlk(pWriter->aBuf[0] + n, taosArrayGet(aSttBlk, iBlockL));
   }
 
-  for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) {
-    tFree((*ppReader)->aBuf[iBuf]);
-  }
-  taosMemoryFree(*ppReader);
+  // write
+  code = tsdbWriteFile(pWriter->pSttFD, pSttFile->size, pWriter->aBuf[0], size);
+  if (code) goto _err;
+
+  // update
+  pSttFile->offset = pSttFile->size;
+  pSttFile->size += size;
 
 _exit:
-  *ppReader = NULL;
+  tsdbTrace("vgId:%d tsdb write stt block, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode),
+            pSttFile->offset, size);
   return code;
 
 _err:
-  tsdbError("vgId:%d, data file reader close failed since %s", TD_VID((*ppReader)->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
+static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) {
   int32_t code = 0;
-  int64_t  offset = pReader->pSet->pHeadF->offset;
-  int64_t  size = pReader->pSet->pHeadF->size - offset;
-  int64_t  n;
-  uint32_t delimiter;
 
-  taosArrayClear(aBlockIdx);
-  if (size == 0) {
-    goto _exit;
-  }
+  pSmaInfo->offset = 0;
+  pSmaInfo->size = 0;
 
-  // alloc
-  code = tRealloc(&pReader->aBuf[0], size);
-  if (code) goto _err;
+  // encode
+  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
+    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
 
-  // seek
-  if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+    if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue;
 
-  // read
-  n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  } else if (n < size) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _err;
-  }
+    SColumnDataAgg sma;
+    tsdbCalcColDataSMA(pColData, &sma);
 
-  // check
-  if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _err;
+    code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma));
+    if (code) goto _err;
+    pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma);
   }
 
-  // decode
-  n = 0;
-  n = tGetU32(pReader->aBuf[0] + n, &delimiter);
-  ASSERT(delimiter == TSDB_FILE_DLMT);
+  // write
+  if (pSmaInfo->size) {
+    code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size);
+    if (code) goto _err;
 
-  while (n < size - sizeof(TSCKSUM)) {
-    SBlockIdx blockIdx;
-    n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx);
+    code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size);
+    if (code) goto _err;
 
-    if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      goto _err;
-    }
+    pSmaInfo->offset = pWriter->fSma.size;
+    pWriter->fSma.size += pSmaInfo->size;
   }
 
-  ASSERT(n + sizeof(TSCKSUM) == size);
-
-_exit:
   return code;
 
 _err:
-  tsdbError("vgId:%d, read block idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-int32_t tsdbReadSstBlk(SDataFReader *pReader, int32_t iSst, SArray *aSstBlk) {
+int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
+                           int8_t cmprAlg, int8_t toLast) {
   int32_t code = 0;
-  int64_t  offset = pReader->pSet->aSstF[iSst]->offset;
-  int64_t  size = pReader->pSet->aSstF[iSst]->size - offset;
-  int64_t  n;
-  uint32_t delimiter;
 
-  taosArrayClear(aSstBlk);
-  if (size == 0) {
-    goto _exit;
+  ASSERT(pBlockData->nRow > 0);
+
+  if (toLast) {
+    pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size;
+  } else {
+    pBlkInfo->offset = pWriter->fData.size;
   }
+  pBlkInfo->szBlock = 0;
+  pBlkInfo->szKey = 0;
 
-  // alloc
-  code = tRealloc(&pReader->aBuf[0], size);
+  int32_t aBufN[4] = {0};
+  code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN);
   if (code) goto _err;
 
-  // seek
-  if (taosLSeekFile(pReader->aLastFD[iSst], offset, SEEK_SET) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  // write =================
+  STsdbFD *pFD = toLast ? pWriter->pSttFD : pWriter->pDataFD;
 
-  // read
-  n = taosReadFile(pReader->aLastFD[iSst], pReader->aBuf[0], size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  } else if (n < size) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _err;
-  }
+  pBlkInfo->szKey = aBufN[3] + aBufN[2];
+  pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3];
 
-  // check
-  if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _err;
-  }
+  int64_t offset = pBlkInfo->offset;
+  code = tsdbWriteFile(pFD, offset, pWriter->aBuf[3], aBufN[3]);
+  if (code) goto _err;
+  offset += aBufN[3];
 
-  // decode
-  n = 0;
-  n = tGetU32(pReader->aBuf[0] + n, &delimiter);
-  ASSERT(delimiter == TSDB_FILE_DLMT);
+  code = tsdbWriteFile(pFD, offset, pWriter->aBuf[2], aBufN[2]);
+  if (code) goto _err;
+  offset += aBufN[2];
 
-  while (n < size - sizeof(TSCKSUM)) {
-    SSstBlk blockl;
-    n += tGetSstBlk(pReader->aBuf[0] + n, &blockl);
+  if (aBufN[1]) {
+    code = tsdbWriteFile(pFD, offset, pWriter->aBuf[1], aBufN[1]);
+    if (code) goto _err;
+    offset += aBufN[1];
+  }
 
-    if (taosArrayPush(aSstBlk, &blockl) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
-      goto _err;
+  if (aBufN[0]) {
+    code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], aBufN[0]);
+    if (code) goto _err;
   }
+
+  // update info
+  if (toLast) {
+    pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock;
+  } else {
+    pWriter->fData.size += pBlkInfo->szBlock;
   }
 
-  ASSERT(n + sizeof(TSCKSUM) == size);
+  // ================= SMA ====================
+  if (pSmaInfo) {
+    code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo);
+    if (code) goto _err;
+  }
 
 _exit:
+  tsdbTrace("vgId:%d tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d",
+            TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset,
+            pBlkInfo->szBlock);
   return code;
 
 _err:
-  tsdbError("vgId:%d read blockl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock) {
+int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
   int32_t   code = 0;
-  int64_t offset = pBlockIdx->offset;
-  int64_t size = pBlockIdx->size;
   int64_t   n;
-  int64_t tn;
+  int64_t   size;
+  TdFilePtr pOutFD = NULL;  // TODO
+  TdFilePtr PInFD = NULL;   // TODO
+  char      fNameFrom[TSDB_FILENAME_LEN];
+  char      fNameTo[TSDB_FILENAME_LEN];
 
-  // alloc
-  code = tRealloc(&pReader->aBuf[0], size);
-  if (code) goto _err;
+  // head
+  tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom);
+  tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo);
 
-  // seek
-  if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) {
+  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+  if (pOutFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
 
-  // read
-  n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size);
-  if (n < 0) {
+  PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
+  if (PInFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
-  } else if (n < size) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _err;
   }
 
-  // check
-  if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
-    code = TSDB_CODE_FILE_CORRUPTED;
+  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pHeadF->size);
+  if (n < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
+  taosCloseFile(&pOutFD);
+  taosCloseFile(&PInFD);
 
-  // decode
-  n = 0;
-
-  uint32_t delimiter;
-  n += tGetU32(pReader->aBuf[0] + n, &delimiter);
-  ASSERT(delimiter == TSDB_FILE_DLMT);
+  // data
+  tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom);
+  tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo);
 
-  tn = tGetMapData(pReader->aBuf[0] + n, mBlock);
-  if (tn < 0) {
-    code = TSDB_CODE_OUT_OF_MEMORY;
+  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+  if (pOutFD == NULL) {
+    code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-  n += tn;
-  ASSERT(n + sizeof(TSCKSUM) == size);
-
-  return code;
-
-_err:
-  tsdbError("vgId:%d, read block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
-  return code;
-}
-
-int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aColumnDataAgg) {
-  int32_t   code = 0;
-  SSmaInfo *pSmaInfo = &pDataBlk->smaInfo;
 
-  ASSERT(pSmaInfo->size > 0);
+  PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
+  if (PInFD == NULL) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
 
-  taosArrayClear(aColumnDataAgg);
+  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pDataF->size);
+  if (n < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+  taosCloseFile(&pOutFD);
+  taosCloseFile(&PInFD);
 
-  // alloc
-  int32_t size = pSmaInfo->size + sizeof(TSCKSUM);
-  code = tRealloc(&pReader->aBuf[0], size);
-  if (code) goto _err;
+  // stt
+  tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[0], fNameFrom);
+  tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[0], fNameTo);
 
-  // seek
-  int64_t n = taosLSeekFile(pReader->pSmaFD, pSmaInfo->offset, SEEK_SET);
-  if (n < 0) {
+  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+  if (pOutFD == NULL) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
-  } else if (n < pSmaInfo->offset) {
-    code = TSDB_CODE_FILE_CORRUPTED;
+  }
+
+  PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
+  if (PInFD == NULL) {
+    code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
 
-  // read
-  n = taosReadFile(pReader->pSmaFD, pReader->aBuf[0], size);
+  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSttF[0]->size);
   if (n < 0) {
     code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
-  } else if (n < size) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _err;
   }
+  taosCloseFile(&pOutFD);
+  taosCloseFile(&PInFD);
 
-  // check
-  if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) {
-    code = TSDB_CODE_FILE_CORRUPTED;
+  // sma
+  tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom);
+  tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo);
+
+  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
+  if (pOutFD == NULL) {
+    code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
 
-  // decode
-  n = 0;
-  while (n < pSmaInfo->size) {
-    SColumnDataAgg sma;
-
-    n += tGetColumnDataAgg(pReader->aBuf[0] + n, &sma);
-    if (taosArrayPush(aColumnDataAgg, &sma) == NULL) {
-      code = TSDB_CODE_OUT_OF_MEMORY;
+  PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
+  if (PInFD == NULL) {
+    code = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
+
+  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pSmaF->size);
+  if (n < 0) {
+    code = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
   }
+  taosCloseFile(&pOutFD);
+  taosCloseFile(&PInFD);
 
   return code;
 
 _err:
-  tsdbError("vgId:%d tsdb read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast,
-                                     SBlockData *pBlockData) {
+// SDataFReader ====================================================
+int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) {
   int32_t       code = 0;
+  SDataFReader *pReader;
+  int32_t       szPage = TSDB_DEFAULT_PAGE_SIZE;
+  char          fname[TSDB_FILENAME_LEN];
 
-  tBlockDataClear(pBlockData);
-
-  TdFilePtr pFD = fromLast ? pReader->aLastFD[0] : pReader->pDataFD;  // (todo)
+  // alloc
+  pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader));
+  if (pReader == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  pReader->pTsdb = pTsdb;
+  pReader->pSet = pSet;
 
-  // uid + version + tskey
-  code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->aBuf[0], pBlkInfo->szKey, 1);
+  // head
+  tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname);
+  code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD);
   if (code) goto _err;
-  SDiskDataHdr hdr;
-  uint8_t     *p = pReader->aBuf[0] + tGetDiskDataHdr(pReader->aBuf[0], &hdr);
-
-  ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
-  ASSERT(pBlockData->suid == hdr.suid);
-  ASSERT(pBlockData->uid == hdr.uid);
-
-  pBlockData->nRow = hdr.nRow;
 
-  // uid
-  if (hdr.uid == 0) {
-    ASSERT(hdr.szUid);
-    code = tsdbDecmprData(p, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aUid,
-                          sizeof(int64_t) * hdr.nRow, &pReader->aBuf[1]);
+  // data
+  tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname);
+  code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD);
   if (code) goto _err;
-  } else {
-    ASSERT(!hdr.szUid);
-  }
-  p += hdr.szUid;
 
-  // version
-  code = tsdbDecmprData(p, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aVersion,
-                        sizeof(int64_t) * hdr.nRow, &pReader->aBuf[1]);
+  // sma
+  tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname);
+  code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD);
   if (code) goto _err;
-  p += hdr.szVer;
 
-  // TSKEY
-  code = tsdbDecmprData(p, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY,
-                        sizeof(TSKEY) * hdr.nRow, &pReader->aBuf[1]);
+  // stt
+  for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+    tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname);
+    code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]);
     if (code) goto _err;
-  p += hdr.szKey;
+  }
 
-  ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey - sizeof(TSCKSUM));
+  *ppReader = pReader;
+  return code;
 
-  // read and decode columns
-  if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit;
+_err:
+  tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  *ppReader = NULL;
+  return code;
+}
 
-  if (hdr.szBlkCol > 0) {
-    int64_t offset = pBlkInfo->offset + pBlkInfo->szKey;
-    code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[0], hdr.szBlkCol + sizeof(TSCKSUM), 1);
-    if (code) goto _err;
-  }
+int32_t tsdbDataFReaderClose(SDataFReader **ppReader) {
+  int32_t code = 0;
+  if (*ppReader == NULL) return code;
 
-  SBlockCol  blockCol = {.cid = 0};
-  SBlockCol *pBlockCol = &blockCol;
-  int32_t    n = 0;
+  // head
+  tsdbCloseFile(&(*ppReader)->pHeadFD);
 
-  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
-    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
+  // data
+  tsdbCloseFile(&(*ppReader)->pDataFD);
 
-    while (pBlockCol && pBlockCol->cid < pColData->cid) {
-      if (n < hdr.szBlkCol) {
-        n += tGetBlockCol(pReader->aBuf[0] + n, pBlockCol);
-      } else {
-        ASSERT(n == hdr.szBlkCol);
-        pBlockCol = NULL;
+  // sma
+  tsdbCloseFile(&(*ppReader)->pSmaFD);
+
+  // stt
+  for (int32_t iStt = 0; iStt < TSDB_MAX_STT_FILE; iStt++) {
+    if ((*ppReader)->aSttFD[iStt]) {
+      tsdbCloseFile(&(*ppReader)->aSttFD[iStt]);
     }
   }
 
-    if (pBlockCol == NULL || pBlockCol->cid > pColData->cid) {
-      // add a lot of NONE
-      for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) {
-        code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
-        if (code) goto _err;
+  for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) {
+    tFree((*ppReader)->aBuf[iBuf]);
   }
-    } else {
-      ASSERT(pBlockCol->type == pColData->type);
+  taosMemoryFree(*ppReader);
+  *ppReader = NULL;
+  return code;
+
+_err:
+  tsdbError("vgId:%d, data file reader close failed since %s", TD_VID((*ppReader)->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) {
+  int32_t    code = 0;
+  SHeadFile *pHeadFile = pReader->pSet->pHeadF;
+  int64_t    offset = pHeadFile->offset;
+  int64_t    size = pHeadFile->size - offset;
+
+  taosArrayClear(aBlockIdx);
+  if (size == 0) return code;
+
+  // alloc
+  code = tRealloc(&pReader->aBuf[0], size);
+  if (code) goto _err;
+
+  // read
+  code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size);
+  if (code) goto _err;
+
+  // decode
+  int64_t n = 0;
+  while (n < size) {
+    SBlockIdx blockIdx;
+    n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx);
+
+    if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+  }
+  ASSERT(n == size);
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d, read block idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) {
+  int32_t   code = 0;
+  SSttFile *pSttFile = pReader->pSet->aSttF[iStt];
+  int64_t   offset = pSttFile->offset;
+  int64_t   size = pSttFile->size - offset;
+
+  taosArrayClear(aSttBlk);
+  if (size == 0) return code;
+
+  // alloc
+  code = tRealloc(&pReader->aBuf[0], size);
+  if (code) goto _err;
+
+  // read
+  code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size);
+  if (code) goto _err;
+
+  // decode
+  int64_t n = 0;
+  while (n < size) {
+    SSttBlk sttBlk;
+    n += tGetSttBlk(pReader->aBuf[0] + n, &sttBlk);
+
+    if (taosArrayPush(aSttBlk, &sttBlk) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+  }
+  ASSERT(n == size);
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d read stt blk failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock) {
+  int32_t code = 0;
+  int64_t offset = pBlockIdx->offset;
+  int64_t size = pBlockIdx->size;
+
+  // alloc
+  code = tRealloc(&pReader->aBuf[0], size);
+  if (code) goto _err;
+
+  // read
+  code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size);
+  if (code) goto _err;
+
+  // decode
+  int64_t n = tGetMapData(pReader->aBuf[0], mBlock);
+  if (n < 0) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
+    goto _err;
+  }
+  ASSERT(n == size);
+
+  return code;
+
+_err:
+  tsdbError("vgId:%d, read block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aColumnDataAgg) {
+  int32_t   code = 0;
+  SSmaInfo *pSmaInfo = &pDataBlk->smaInfo;
+
+  ASSERT(pSmaInfo->size > 0);
+
+  taosArrayClear(aColumnDataAgg);
+
+  // alloc
+  code = tRealloc(&pReader->aBuf[0], pSmaInfo->size);
+  if (code) goto _err;
+
+  // read
+  code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], pSmaInfo->size);
+  if (code) goto _err;
+
+  // decode
+  int32_t n = 0;
+  while (n < pSmaInfo->size) {
+    SColumnDataAgg sma;
+    n += tGetColumnDataAgg(pReader->aBuf[0] + n, &sma);
+
+    if (taosArrayPush(aColumnDataAgg, &sma) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
+      goto _err;
+    }
+  }
+  ASSERT(n == pSmaInfo->size);
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
+  return code;
+}
+
+static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, SBlockData *pBlockData) {
+  int32_t code = 0;
+
+  tBlockDataClear(pBlockData);
+
+  STsdbFD *pFD = pReader->pDataFD;
+
+  // uid + version + tskey
+  code = tRealloc(&pReader->aBuf[0], pBlkInfo->szKey);
+  if (code) goto _err;
+
+  code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey);
+  if (code) goto _err;
+
+  SDiskDataHdr hdr;
+  uint8_t     *p = pReader->aBuf[0] + tGetDiskDataHdr(pReader->aBuf[0], &hdr);
+
+  ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
+  ASSERT(pBlockData->suid == hdr.suid);
+  ASSERT(pBlockData->uid == hdr.uid);
+
+  pBlockData->nRow = hdr.nRow;
+
+  // uid
+  if (hdr.uid == 0) {
+    ASSERT(hdr.szUid);
+    code = tsdbDecmprData(p, hdr.szUid, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aUid,
+                          sizeof(int64_t) * hdr.nRow, &pReader->aBuf[1]);
+    if (code) goto _err;
+  } else {
+    ASSERT(!hdr.szUid);
+  }
+  p += hdr.szUid;
+
+  // version
+  code = tsdbDecmprData(p, hdr.szVer, TSDB_DATA_TYPE_BIGINT, hdr.cmprAlg, (uint8_t **)&pBlockData->aVersion,
+                        sizeof(int64_t) * hdr.nRow, &pReader->aBuf[1]);
+  if (code) goto _err;
+  p += hdr.szVer;
+
+  // TSKEY
+  code = tsdbDecmprData(p, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY,
+                        sizeof(TSKEY) * hdr.nRow, &pReader->aBuf[1]);
+  if (code) goto _err;
+  p += hdr.szKey;
+
+  ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey);
+
+  // read and decode columns
+  if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit;
+
+  if (hdr.szBlkCol > 0) {
+    int64_t offset = pBlkInfo->offset + pBlkInfo->szKey;
+
+    code = tRealloc(&pReader->aBuf[0], hdr.szBlkCol);
+    if (code) goto _err;
+
+    code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol);
+    if (code) goto _err;
+  }
+
+  SBlockCol  blockCol = {.cid = 0};
+  SBlockCol *pBlockCol = &blockCol;
+  int32_t    n = 0;
+
+  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
+    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
+
+    while (pBlockCol && pBlockCol->cid < pColData->cid) {
+      if (n < hdr.szBlkCol) {
+        n += tGetBlockCol(pReader->aBuf[0] + n, pBlockCol);
+      } else {
+        ASSERT(n == hdr.szBlkCol);
+        pBlockCol = NULL;
+      }
+    }
+
+    if (pBlockCol == NULL || pBlockCol->cid > pColData->cid) {
+      // add a lot of NONE
+      for (int32_t iRow = 0; iRow < hdr.nRow; iRow++) {
+        code = tColDataAppendValue(pColData, &COL_VAL_NONE(pColData->cid, pColData->type));
+        if (code) goto _err;
+      }
+    } else {
+      ASSERT(pBlockCol->type == pColData->type);
       ASSERT(pBlockCol->flag && pBlockCol->flag != HAS_NONE);
 
       if (pBlockCol->flag == HAS_NULL) {
@@ -823,10 +1030,13 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo
         }
       } else {
         // decode from binary
-        int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + pBlockCol->offset;
-        int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM);
+        int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + pBlockCol->offset;
+        int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue;
+
+        code = tRealloc(&pReader->aBuf[1], size);
+        if (code) goto _err;
 
-        code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[1], size, 0);
+        code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size);
         if (code) goto _err;
 
         code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]);
@@ -846,7 +1056,7 @@ _err:
 int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) {
   int32_t code = 0;
 
-  code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], 0, pBlockData);
+  code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], pBlockData);
   if (code) goto _err;
 
   if (pDataBlk->nSubBlock > 1) {
@@ -864,7 +1074,7 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData
     tBlockDataInitEx(&bData2, pBlockData);
 
     for (int32_t iSubBlock = 1; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) {
-      code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], 0, &bData1);
+      code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], &bData1);
       if (code) {
         tBlockDataDestroy(&bData1, 1);
         tBlockDataDestroy(&bData2, 1);
@@ -897,330 +1107,142 @@ _err:
   return code;
 }
 
-int32_t tsdbReadSstBlock(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) {
+int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData) {
   int32_t code = 0;
 
-  code = tsdbReadBlockDataImpl(pReader, &pSstBlk->bInfo, 1, pBlockData);
+  // alloc
+  code = tRealloc(&pReader->aBuf[0], pSttBlk->bInfo.szBlock);
   if (code) goto _err;
 
-  return code;
-
-_err:
-  tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
-  return code;
-}
-
-int32_t tsdbReadSstBlockEx(SDataFReader *pReader, int32_t iSst, SSstBlk *pSstBlk, SBlockData *pBlockData) {
-  int32_t code = 0;
-
   // read
-  code = tsdbReadAndCheck(pReader->aLastFD[iSst], pSstBlk->bInfo.offset, &pReader->aBuf[0], pSstBlk->bInfo.szBlock, 0);
-  if (code) goto _exit;
+  code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock);
+  if (code) goto _err;
 
   // decmpr
-  code = tDecmprBlockData(pReader->aBuf[0], pSstBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]);
-  if (code) goto _exit;
+  code = tDecmprBlockData(pReader->aBuf[0], pSttBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]);
+  if (code) goto _err;
 
-_exit:
+  return code;
+
+_err:
+  tsdbError("vgId:%d tsdb read stt block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-// SDataFWriter ====================================================
-int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) {
+// SDelFWriter ====================================================
+int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) {
   int32_t      code = 0;
-  int32_t       flag;
-  int64_t       n;
-  SDataFWriter *pWriter = NULL;
   char         fname[TSDB_FILENAME_LEN];
-  char          hdr[TSDB_FHDR_SIZE] = {0};
+  uint8_t      hdr[TSDB_FHDR_SIZE] = {0};
+  SDelFWriter *pDelFWriter;
+  int64_t      n;
 
   // alloc
-  pWriter = taosMemoryCalloc(1, sizeof(*pWriter));
-  if (pWriter == NULL) {
+  pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter));
+  if (pDelFWriter == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
-  pWriter->pTsdb = pTsdb;
-  pWriter->wSet = (SDFileSet){
-      .diskId = pSet->diskId,
-      .fid = pSet->fid,
-      .pHeadF = &pWriter->fHead,
-      .pDataF = &pWriter->fData,
-      .pSmaF = &pWriter->fSma,
-      .nSstF = pSet->nSstF  //
-  };
-  pWriter->fHead = *pSet->pHeadF;
-  pWriter->fData = *pSet->pDataF;
-  pWriter->fSma = *pSet->pSmaF;
-  for (int8_t iSst = 0; iSst < pSet->nSstF; iSst++) {
-    pWriter->wSet.aSstF[iSst] = &pWriter->fSst[iSst];
-    pWriter->fSst[iSst] = *pSet->aSstF[iSst];
-  }
+  pDelFWriter->pTsdb = pTsdb;
+  pDelFWriter->fDel = *pFile;
 
-  // head
-  flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
-  tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname);
-  pWriter->pHeadFD = taosOpenFile(fname, flag);
-  if (pWriter->pHeadFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  tsdbDelFileName(pTsdb, pFile, fname);
+  code =
+      tsdbOpenFile(fname, TSDB_DEFAULT_PAGE_SIZE, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, &pDelFWriter->pWriteH);
+  if (code) goto _err;
 
-  n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  // update header
+  code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE);
+  if (code) goto _err;
 
-  ASSERT(n == TSDB_FHDR_SIZE);
-
-  pWriter->fHead.size += TSDB_FHDR_SIZE;
-
-  // data
-  if (pWriter->fData.size == 0) {
-    flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
-  } else {
-    flag = TD_FILE_WRITE;
-  }
-  tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname);
-  pWriter->pDataFD = taosOpenFile(fname, flag);
-  if (pWriter->pDataFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  if (pWriter->fData.size == 0) {
-    n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE);
-    if (n < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    pWriter->fData.size += TSDB_FHDR_SIZE;
-  } else {
-    n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_END);
-    if (n < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    ASSERT(n == pWriter->fData.size);
-  }
-
-  // sma
-  if (pWriter->fSma.size == 0) {
-    flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
-  } else {
-    flag = TD_FILE_WRITE;
-  }
-  tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname);
-  pWriter->pSmaFD = taosOpenFile(fname, flag);
-  if (pWriter->pSmaFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  if (pWriter->fSma.size == 0) {
-    n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE);
-    if (n < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    pWriter->fSma.size += TSDB_FHDR_SIZE;
-  } else {
-    n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_END);
-    if (n < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    ASSERT(n == pWriter->fSma.size);
-  }
-
-  // sst
-  ASSERT(pWriter->fSst[pSet->nSstF - 1].size == 0);
-  flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC;
-  tsdbSstFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSst[pSet->nSstF - 1], fname);
-  pWriter->pLastFD = taosOpenFile(fname, flag);
-  if (pWriter->pLastFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  pWriter->fSst[pWriter->wSet.nSstF - 1].size += TSDB_FHDR_SIZE;
+  pDelFWriter->fDel.size = TSDB_FHDR_SIZE;
+  pDelFWriter->fDel.offset = 0;
 
-  *ppWriter = pWriter;
+  *ppWriter = pDelFWriter;
   return code;
 
 _err:
-  tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   *ppWriter = NULL;
   return code;
 }
 
-int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) {
+int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) {
   int32_t      code = 0;
-  STsdb  *pTsdb = NULL;
-
-  if (*ppWriter == NULL) goto _exit;
+  SDelFWriter *pWriter = *ppWriter;
+  STsdb       *pTsdb = pWriter->pTsdb;
 
-  pTsdb = (*ppWriter)->pTsdb;
+  // sync
   if (sync) {
-    if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    if (taosFsyncFile((*ppWriter)->pDataFD) < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    if (taosFsyncFile((*ppWriter)->pLastFD) < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-  }
-
-  if (taosCloseFile(&(*ppWriter)->pHeadFD) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (taosCloseFile(&(*ppWriter)->pDataFD) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+    code = tsdbFsyncFile(pWriter->pWriteH);
+    if (code) goto _err;
   }
 
-  if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  // close
+  tsdbCloseFile(&pWriter->pWriteH);
 
-  if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+  for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) {
+    tFree(pWriter->aBuf[iBuf]);
   }
+  taosMemoryFree(pWriter);
 
-  for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) {
-    tFree((*ppWriter)->aBuf[iBuf]);
-  }
-  taosMemoryFree(*ppWriter);
-_exit:
   *ppWriter = NULL;
   return code;
 
 _err:
-  tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) {
+int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) {
   int32_t code = 0;
+  int64_t size;
   int64_t n;
-  char    hdr[TSDB_FHDR_SIZE];
-
-  // head ==============
-  memset(hdr, 0, TSDB_FHDR_SIZE);
-  tPutHeadFile(hdr, &pWriter->fHead);
-  taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
-
-  n = taosLSeekFile(pWriter->pHeadFD, 0, SEEK_SET);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  // data ==============
-  memset(hdr, 0, TSDB_FHDR_SIZE);
-  tPutDataFile(hdr, &pWriter->fData);
-  taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
-
-  n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_SET);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
 
-  n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+  // prepare
+  size = 0;
+  for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) {
+    size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData));
   }
 
-  // sma ==============
-  memset(hdr, 0, TSDB_FHDR_SIZE);
-  tPutSmaFile(hdr, &pWriter->fSma);
-  taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
-
-  n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_SET);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  // alloc
+  code = tRealloc(&pWriter->aBuf[0], size);
+  if (code) goto _err;
 
-  n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+  // build
+  n = 0;
+  for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) {
+    n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData));
   }
+  ASSERT(n == size);
 
-  // sst ==============
-  memset(hdr, 0, TSDB_FHDR_SIZE);
-  tPutSstFile(hdr, &pWriter->fSst[pWriter->wSet.nSstF - 1]);
-  taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE);
-
-  n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  // write
+  code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size);
+  if (code) goto _err;
 
-  n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  // update
+  pDelIdx->offset = pWriter->fDel.size;
+  pDelIdx->size = size;
+  pWriter->fDel.size += size;
 
   return code;
 
 _err:
-  tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) {
+int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) {
   int32_t  code = 0;
-  SHeadFile *pHeadFile = &pWriter->fHead;
-  int64_t    size = 0;
+  int64_t  size;
   int64_t  n;
-
-  // check
-  if (taosArrayGetSize(aBlockIdx) == 0) {
-    pHeadFile->offset = pHeadFile->size;
-    goto _exit;
-  }
+  SDelIdx *pDelIdx;
 
   // prepare
-  size = sizeof(uint32_t);
-  for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) {
-    size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx));
+  size = 0;
+  for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) {
+    size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx));
   }
-  size += sizeof(TSCKSUM);
 
   // alloc
   code = tRealloc(&pWriter->aBuf[0], size);
@@ -1228,383 +1250,170 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) {
 
   // build
   n = 0;
-  n = tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
-  for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) {
-    n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx));
+  for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) {
+    n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx));
   }
-  taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
-
-  ASSERT(n + sizeof(TSCKSUM) == size);
+  ASSERT(n == size);
 
   // write
-  n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
+  code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size);
+  if (code) goto _err;
 
   // update
-  pHeadFile->offset = pHeadFile->size;
-  pHeadFile->size += size;
+  pWriter->fDel.offset = pWriter->fDel.size;
+  pWriter->fDel.size += size;
 
-_exit:
-  tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", TD_VID(pWriter->pTsdb->pVnode),
-            pHeadFile->offset, size, taosArrayGetSize(aBlockIdx));
   return code;
 
 _err:
-  tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBlockIdx) {
+int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) {
   int32_t code = 0;
-  SHeadFile *pHeadFile = &pWriter->fHead;
-  int64_t    size;
+  char    hdr[TSDB_FHDR_SIZE] = {0};
+  int64_t size = TSDB_FHDR_SIZE;
   int64_t n;
 
-  ASSERT(mBlock->nItem > 0);
-
-  // alloc
-  size = sizeof(uint32_t) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM);
-  code = tRealloc(&pWriter->aBuf[0], size);
-  if (code) goto _err;
-
   // build
-  n = 0;
-  n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
-  n += tPutMapData(pWriter->aBuf[0] + n, mBlock);
-  taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
-
-  ASSERT(n + sizeof(TSCKSUM) == size);
+  tPutDelFile(hdr, &pWriter->fDel);
 
   // write
-  n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  // update
-  pBlockIdx->offset = pHeadFile->size;
-  pBlockIdx->size = size;
-  pHeadFile->size += size;
+  code = tsdbWriteFile(pWriter->pWriteH, 0, hdr, size);
+  if (code) goto _err;
 
-  tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64
-            " size:%" PRId64 " nItem:%d",
-            TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid,
-            pBlockIdx->offset, pBlockIdx->size, mBlock->nItem);
   return code;
 
 _err:
-  tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
   return code;
 }
+// SDelFReader ====================================================
+struct SDelFReader {
+  STsdb   *pTsdb;
+  SDelFile fDel;
+  STsdbFD *pReadH;
+  uint8_t *aBuf[1];
+};
 
-int32_t tsdbWriteSstBlk(SDataFWriter *pWriter, SArray *aSstBlk) {
+int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) {
   int32_t      code = 0;
-  SSstFile *pSstFile = &pWriter->fSst[pWriter->wSet.nSstF - 1];
-  int64_t   size;
+  char         fname[TSDB_FILENAME_LEN];
+  SDelFReader *pDelFReader;
   int64_t      n;
 
-  // check
-  if (taosArrayGetSize(aSstBlk) == 0) {
-    pSstFile->offset = pSstFile->size;
-    goto _exit;
-  }
-
-  // size
-  size = sizeof(uint32_t);  // TSDB_FILE_DLMT
-  for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) {
-    size += tPutSstBlk(NULL, taosArrayGet(aSstBlk, iBlockL));
-  }
-  size += sizeof(TSCKSUM);
-
   // alloc
-  code = tRealloc(&pWriter->aBuf[0], size);
-  if (code) goto _err;
-
-  // encode
-  n = 0;
-  n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT);
-  for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSstBlk); iBlockL++) {
-    n += tPutSstBlk(pWriter->aBuf[0] + n, taosArrayGet(aSstBlk, iBlockL));
-  }
-  taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
-
-  ASSERT(n + sizeof(TSCKSUM) == size);
-
-  // write
-  n = taosWriteFile(pWriter->pLastFD, pWriter->aBuf[0], size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
+  pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader));
+  if (pDelFReader == NULL) {
+    code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
 
-  // update
-  pSstFile->offset = pSstFile->size;
-  pSstFile->size += size;
+  // open impl
+  pDelFReader->pTsdb = pTsdb;
+  pDelFReader->fDel = *pFile;
 
-_exit:
-  tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode),
-            pSstFile->offset, size);
+  tsdbDelFileName(pTsdb, pFile, fname);
+  code = tsdbOpenFile(fname, TSDB_DEFAULT_PAGE_SIZE, TD_FILE_READ, &pDelFReader->pReadH);
+  if (code) goto _err;
+
+  *ppReader = pDelFReader;
   return code;
 
 _err:
-  tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  *ppReader = NULL;
   return code;
 }
 
-static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SDataBlk *pDataBlk) {
-  for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
-    TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]};
+int32_t tsdbDelFReaderClose(SDelFReader **ppReader) {
+  int32_t      code = 0;
+  SDelFReader *pReader = *ppReader;
 
-    if (iRow == 0) {
-      if (tsdbKeyCmprFn(&pDataBlk->minKey, &key) > 0) {
-        pDataBlk->minKey = key;
-      }
-    } else {
-      if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) {
-        pDataBlk->hasDup = 1;
-      }
+  if (pReader) {
+    tsdbCloseFile(&pReader->pReadH);
+    for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) {
+      tFree(pReader->aBuf[iBuf]);
     }
-
-    if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&pDataBlk->maxKey, &key) < 0) {
-      pDataBlk->maxKey = key;
+    taosMemoryFree(pReader);
   }
+  *ppReader = NULL;
 
-    pDataBlk->minVer = TMIN(pDataBlk->minVer, key.version);
-    pDataBlk->maxVer = TMAX(pDataBlk->maxVer, key.version);
-  }
-  pDataBlk->nRow += pBlockData->nRow;
+_exit:
+  return code;
 }
 
-static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) {
+int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) {
   int32_t code = 0;
+  int64_t offset = pDelIdx->offset;
+  int64_t size = pDelIdx->size;
+  int64_t n;
 
-  pSmaInfo->offset = 0;
-  pSmaInfo->size = 0;
-
-  // encode
-  for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
-    SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
-
-    if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue;
-
-    SColumnDataAgg sma;
-    tsdbCalcColDataSMA(pColData, &sma);
+  taosArrayClear(aDelData);
 
-    code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma));
+  // alloc
+  code = tRealloc(&pReader->aBuf[0], size);
   if (code) goto _err;
-    pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma);
-  }
-
-  // write
-  if (pSmaInfo->size) {
-    int32_t size = pSmaInfo->size + sizeof(TSCKSUM);
 
-    code = tRealloc(&pWriter->aBuf[0], size);
+  // read
+  code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size);
   if (code) goto _err;
 
-    taosCalcChecksumAppend(0, pWriter->aBuf[0], size);
+  // // decode
+  n = 0;
+  while (n < size) {
+    SDelData delData;
+    n += tGetDelData(pReader->aBuf[0] + n, &delData);
 
-    int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->aBuf[0], size);
-    if (n < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
+    if (taosArrayPush(aDelData, &delData) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
       goto _err;
     }
-
-    pSmaInfo->offset = pWriter->fSma.size;
-    pWriter->fSma.size += size;
   }
+  ASSERT(n == size);
 
   return code;
 
 _err:
-  tsdbError("vgId:%d tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
   return code;
 }
 
-int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo,
-                           int8_t cmprAlg, int8_t toLast) {
+int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) {
   int32_t code = 0;
+  int32_t n;
+  int64_t offset = pReader->fDel.offset;
+  int64_t size = pReader->fDel.size - offset;
 
-  ASSERT(pBlockData->nRow > 0);
-
-  pBlkInfo->offset = toLast ? pWriter->fSst[pWriter->wSet.nSstF - 1].size : pWriter->fData.size;
-  pBlkInfo->szBlock = 0;
-  pBlkInfo->szKey = 0;
+  taosArrayClear(aDelIdx);
 
-  int32_t aBufN[4] = {0};
-  code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN);
+  // alloc
+  code = tRealloc(&pReader->aBuf[0], size);
   if (code) goto _err;
 
-  // write =================
-  TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD;
-
-  pBlkInfo->szKey = aBufN[3] + aBufN[2];
-  pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3];
-
-  int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], aBufN[3]);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  n = taosWriteFile(pFD, pWriter->aBuf[2], aBufN[2]);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (aBufN[1]) {
-    n = taosWriteFile(pFD, pWriter->aBuf[1], aBufN[1]);
-    if (n < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-  }
-
-  if (aBufN[0]) {
-    n = taosWriteFile(pFD, pWriter->aBuf[0], aBufN[0]);
-    if (n < 0) {
-      code = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-  }
-
-  // update info
-  if (toLast) {
-    pWriter->fSst[pWriter->wSet.nSstF - 1].size += pBlkInfo->szBlock;
-  } else {
-    pWriter->fData.size += pBlkInfo->szBlock;
-  }
-
-  // ================= SMA ====================
-  if (pSmaInfo) {
-    code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo);
+  // read
+  code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size);
   if (code) goto _err;
-  }
-
-_exit:
-  tsdbTrace("vgId:%d tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d",
-            TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset,
-            pBlkInfo->szBlock);
-  return code;
-
-_err:
-  tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code));
-  return code;
-}
-
-int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) {
-  int32_t   code = 0;
-  int64_t   n;
-  int64_t   size;
-  TdFilePtr pOutFD = NULL;  // TODO
-  TdFilePtr PInFD = NULL;   // TODO
-  char      fNameFrom[TSDB_FILENAME_LEN];
-  char      fNameTo[TSDB_FILENAME_LEN];
-
-  // head
-  tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom);
-  tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo);
-
-  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
-  if (pOutFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
-  if (PInFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pHeadF->size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  taosCloseFile(&pOutFD);
-  taosCloseFile(&PInFD);
-
-  // data
-  tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom);
-  tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo);
-
-  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
-  if (pOutFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
-  if (PInFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pDataF->size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  taosCloseFile(&pOutFD);
-  taosCloseFile(&PInFD);
-
-  // sst
-  tsdbSstFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSstF[0], fNameFrom);
-  tsdbSstFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSstF[0], fNameTo);
-
-  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
-  if (pOutFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
-  if (PInFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
 
-  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->aSstF[0]->size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  taosCloseFile(&pOutFD);
-  taosCloseFile(&PInFD);
+  // decode
+  n = 0;
+  while (n < size) {
+    SDelIdx delIdx;
 
-  // sma
-  tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom);
-  tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo);
+    n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx);
 
-  pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC);
-  if (pOutFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
+    if (taosArrayPush(aDelIdx, &delIdx) == NULL) {
+      code = TSDB_CODE_OUT_OF_MEMORY;
       goto _err;
     }
-
-  PInFD = taosOpenFile(fNameFrom, TD_FILE_READ);
-  if (PInFD == NULL) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
   }
 
-  n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pSmaF->size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  taosCloseFile(&pOutFD);
-  taosCloseFile(&PInFD);
+  ASSERT(n == size);
 
   return code;
 
 _err:
-  tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code));
+  tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code));
   return code;
 }
\ No newline at end of file

source/dnode/vnode/src/tsdb/tsdbRetention.c

@@ -60,7 +60,7 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) {
     if (expLevel < 0) {
       taosMemoryFree(pSet->pHeadF);
       taosMemoryFree(pSet->pDataF);
-      taosMemoryFree(pSet->aSstF[0]);
+      taosMemoryFree(pSet->aSttF[0]);
       taosMemoryFree(pSet->pSmaF);
       taosArrayRemove(fs.aDFileSet, iSet);
       iSet--;

source/dnode/vnode/src/tsdb/tsdbSnapshot.c

@@ -27,9 +27,9 @@ struct STsdbSnapReader {
   int32_t       fid;
   SDataFReader* pDataFReader;
   SArray*       aBlockIdx;  // SArray<SBlockIdx>
-  SArray*       aSstBlk;    // SArray<SSstBlk>
+  SArray*       aSstBlk;    // SArray<SSttBlk>
   SBlockIdx*    pBlockIdx;
-  SSstBlk*      pSstBlk;
+  SSttBlk*      pSstBlk;
 
   int32_t    iBlockIdx;
   int32_t    iBlockL;
@@ -64,7 +64,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
       code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx);
       if (code) goto _err;
 
-      code = tsdbReadSstBlk(pReader->pDataFReader, 0, pReader->aSstBlk);
+      code = tsdbReadSttBlk(pReader->pDataFReader, 0, pReader->aSstBlk);
       if (code) goto _err;
 
       // init
@@ -87,7 +87,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
           break;
         }
 
-        pReader->pSstBlk = (SSstBlk*)taosArrayGet(pReader->aSstBlk, pReader->iBlockL);
+        pReader->pSstBlk = (SSttBlk*)taosArrayGet(pReader->aSstBlk, pReader->iBlockL);
         if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) {
           // TODO
           break;
@@ -151,7 +151,7 @@ static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
           // next
           pReader->iBlockL++;
           if (pReader->iBlockL < taosArrayGetSize(pReader->aSstBlk)) {
-            pReader->pSstBlk = (SSstBlk*)taosArrayGetSize(pReader->aSstBlk);
+            pReader->pSstBlk = (SSttBlk*)taosArrayGetSize(pReader->aSstBlk);
           } else {
             pReader->pSstBlk = NULL;
           }
@@ -298,7 +298,7 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
   }
-  pReader->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
+  pReader->aSstBlk = taosArrayInit(0, sizeof(SSttBlk));
   if (pReader->aSstBlk == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -431,7 +431,7 @@ struct STsdbSnapWriter {
   SBlockData*   pBlockData;
   int32_t       iRow;
   SBlockData    bDataR;
-  SArray*       aSstBlk;  // SArray<SSstBlk>
+  SArray*       aSstBlk;  // SArray<SSttBlk>
   int32_t       iBlockL;
   SBlockData    lDataR;
 
@@ -443,7 +443,7 @@ struct STsdbSnapWriter {
 
   SMapData mBlockW;     // SMapData<SDataBlk>
   SArray*  aBlockIdxW;  // SArray<SBlockIdx>
-  SArray*  aBlockLW;    // SArray<SSstBlk>
+  SArray*  aBlockLW;    // SArray<SSttBlk>
 
   // for del file
   SDelFReader* pDelFReader;
@@ -845,7 +845,7 @@ static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
 
   // write remain stuff
   if (taosArrayGetSize(pWriter->aBlockLW) > 0) {
-    code = tsdbWriteSstBlk(pWriter->pDataFWriter, pWriter->aBlockIdxW);
+    code = tsdbWriteSttBlk(pWriter->pDataFWriter, pWriter->aBlockIdxW);
     if (code) goto _err;
   }
 
@@ -911,7 +911,7 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
       code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx);
       if (code) goto _err;
 
-      code = tsdbReadSstBlk(pWriter->pDataFReader, 0, pWriter->aSstBlk);
+      code = tsdbReadSttBlk(pWriter->pDataFReader, 0, pWriter->aSstBlk);
       if (code) goto _err;
     } else {
       ASSERT(pWriter->pDataFReader == NULL);
@@ -931,25 +931,25 @@ static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint3
     // write
     SHeadFile fHead;
     SDataFile fData;
-    SSstFile  fLast;
+    SSttFile  fLast;
     SSmaFile  fSma;
-    SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aSstF[0] = &fLast, .pSmaF = &fSma};
+    SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aSttF[0] = &fLast, .pSmaF = &fSma};
 
     if (pSet) {
       wSet.diskId = pSet->diskId;
       wSet.fid = fid;
-      wSet.nSstF = 1;
+      wSet.nSttF = 1;
       fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
       fData = *pSet->pDataF;
-      fLast = (SSstFile){.commitID = pWriter->commitID, .size = 0};
+      fLast = (SSttFile){.commitID = pWriter->commitID, .size = 0};
       fSma = *pSet->pSmaF;
     } else {
       wSet.diskId = (SDiskID){.level = 0, .id = 0};
       wSet.fid = fid;
-      wSet.nSstF = 1;
+      wSet.nSttF = 1;
       fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
       fData = (SDataFile){.commitID = pWriter->commitID, .size = 0};
-      fLast = (SSstFile){.commitID = pWriter->commitID, .size = 0, .offset = 0};
+      fLast = (SSttFile){.commitID = pWriter->commitID, .size = 0, .offset = 0};
       fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0};
     }
 
@@ -1147,7 +1147,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
   code = tBlockDataCreate(&pWriter->bDataR);
   if (code) goto _err;
 
-  pWriter->aSstBlk = taosArrayInit(0, sizeof(SSstBlk));
+  pWriter->aSstBlk = taosArrayInit(0, sizeof(SSttBlk));
   if (pWriter->aSstBlk == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;
@@ -1161,7 +1161,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
   code = tBlockDataCreate(&pWriter->bDataW);
   if (code) goto _err;
 
-  pWriter->aBlockLW = taosArrayInit(0, sizeof(SSstBlk));
+  pWriter->aBlockLW = taosArrayInit(0, sizeof(SSttBlk));
   if (pWriter->aBlockLW == NULL) {
     code = TSDB_CODE_OUT_OF_MEMORY;
     goto _err;

source/dnode/vnode/src/tsdb/tsdbUtil.c

@@ -214,7 +214,7 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) {
 
 int32_t tCmprBlockL(void const *lhs, void const *rhs) {
   SBlockIdx *lBlockIdx = (SBlockIdx *)lhs;
-  SSstBlk   *rBlockL = (SSstBlk *)rhs;
+  SSttBlk   *rBlockL = (SSttBlk *)rhs;
 
   if (lBlockIdx->suid < rBlockL->suid) {
     return -1;
@@ -311,41 +311,41 @@ bool tDataBlkHasSma(SDataBlk *pDataBlk) {
   return pDataBlk->smaInfo.size > 0;
 }
 
-// SSstBlk ======================================================
-int32_t tPutSstBlk(uint8_t *p, void *ph) {
+// SSttBlk ======================================================
+int32_t tPutSttBlk(uint8_t *p, void *ph) {
   int32_t  n = 0;
-  SSstBlk *pSstBlk = (SSstBlk *)ph;
-
-  n += tPutI64(p ? p + n : p, pSstBlk->suid);
-  n += tPutI64(p ? p + n : p, pSstBlk->minUid);
-  n += tPutI64(p ? p + n : p, pSstBlk->maxUid);
-  n += tPutI64v(p ? p + n : p, pSstBlk->minKey);
-  n += tPutI64v(p ? p + n : p, pSstBlk->maxKey);
-  n += tPutI64v(p ? p + n : p, pSstBlk->minVer);
-  n += tPutI64v(p ? p + n : p, pSstBlk->maxVer);
-  n += tPutI32v(p ? p + n : p, pSstBlk->nRow);
-  n += tPutI64v(p ? p + n : p, pSstBlk->bInfo.offset);
-  n += tPutI32v(p ? p + n : p, pSstBlk->bInfo.szBlock);
-  n += tPutI32v(p ? p + n : p, pSstBlk->bInfo.szKey);
+  SSttBlk *pSttBlk = (SSttBlk *)ph;
+
+  n += tPutI64(p ? p + n : p, pSttBlk->suid);
+  n += tPutI64(p ? p + n : p, pSttBlk->minUid);
+  n += tPutI64(p ? p + n : p, pSttBlk->maxUid);
+  n += tPutI64v(p ? p + n : p, pSttBlk->minKey);
+  n += tPutI64v(p ? p + n : p, pSttBlk->maxKey);
+  n += tPutI64v(p ? p + n : p, pSttBlk->minVer);
+  n += tPutI64v(p ? p + n : p, pSttBlk->maxVer);
+  n += tPutI32v(p ? p + n : p, pSttBlk->nRow);
+  n += tPutI64v(p ? p + n : p, pSttBlk->bInfo.offset);
+  n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szBlock);
+  n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szKey);
 
   return n;
 }
 
-int32_t tGetSstBlk(uint8_t *p, void *ph) {
+int32_t tGetSttBlk(uint8_t *p, void *ph) {
   int32_t  n = 0;
-  SSstBlk *pSstBlk = (SSstBlk *)ph;
-
-  n += tGetI64(p + n, &pSstBlk->suid);
-  n += tGetI64(p + n, &pSstBlk->minUid);
-  n += tGetI64(p + n, &pSstBlk->maxUid);
-  n += tGetI64v(p + n, &pSstBlk->minKey);
-  n += tGetI64v(p + n, &pSstBlk->maxKey);
-  n += tGetI64v(p + n, &pSstBlk->minVer);
-  n += tGetI64v(p + n, &pSstBlk->maxVer);
-  n += tGetI32v(p + n, &pSstBlk->nRow);
-  n += tGetI64v(p + n, &pSstBlk->bInfo.offset);
-  n += tGetI32v(p + n, &pSstBlk->bInfo.szBlock);
-  n += tGetI32v(p + n, &pSstBlk->bInfo.szKey);
+  SSttBlk *pSttBlk = (SSttBlk *)ph;
+
+  n += tGetI64(p + n, &pSttBlk->suid);
+  n += tGetI64(p + n, &pSttBlk->minUid);
+  n += tGetI64(p + n, &pSttBlk->maxUid);
+  n += tGetI64v(p + n, &pSttBlk->minKey);
+  n += tGetI64v(p + n, &pSttBlk->maxKey);
+  n += tGetI64v(p + n, &pSttBlk->minVer);
+  n += tGetI64v(p + n, &pSttBlk->maxVer);
+  n += tGetI32v(p + n, &pSttBlk->nRow);
+  n += tGetI64v(p + n, &pSttBlk->bInfo.offset);
+  n += tGetI32v(p + n, &pSttBlk->bInfo.szBlock);
+  n += tGetI32v(p + n, &pSttBlk->bInfo.szKey);
 
   return n;
 }
@@ -1548,7 +1548,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
       if (code) goto _exit;
 
       blockCol.offset = aBufN[0];
-      aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM);
+      aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue;
     }
 
     code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol));
@@ -1556,15 +1556,8 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
     hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol);
   }
 
-  aBufN[1] = 0;
-  if (hdr.szBlkCol > 0) {
-    aBufN[1] = hdr.szBlkCol + sizeof(TSCKSUM);
-
-    code = tRealloc(&aBuf[1], aBufN[1]);
-    if (code) goto _exit;
-
-    taosCalcChecksumAppend(0, aBuf[1], aBufN[1]);
-  }
+  // SBlockCol
+  aBufN[1] = hdr.szBlkCol;
 
   // uid + version + tskey
   aBufN[2] = 0;
@@ -1585,16 +1578,11 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut,
   if (code) goto _exit;
   aBufN[2] += hdr.szKey;
 
-  aBufN[2] += sizeof(TSCKSUM);
-  code = tRealloc(&aBuf[2], aBufN[2]);
-  if (code) goto _exit;
-
   // hdr
   aBufN[3] = tPutDiskDataHdr(NULL, &hdr);
   code = tRealloc(&aBuf[3], aBufN[3]);
   if (code) goto _exit;
   tPutDiskDataHdr(aBuf[3], &hdr);
-  taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], aBufN[3]), aBuf[2], aBufN[2]);
 
   // aggragate
   if (ppOut) {
@@ -1626,10 +1614,6 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
 
   // SDiskDataHdr
   n += tGetDiskDataHdr(pIn + n, &hdr);
-  if (!taosCheckChecksumWhole(pIn, n + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM))) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _exit;
-  }
   ASSERT(hdr.delimiter == TSDB_FILE_DLMT);
 
   pBlockData->suid = hdr.suid;
@@ -1657,7 +1641,7 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
   code = tsdbDecmprData(pIn + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY,
                         sizeof(TSKEY) * hdr.nRow, &aBuf[0]);
   if (code) goto _exit;
-  n = n + hdr.szKey + sizeof(TSCKSUM);
+  n += hdr.szKey;
 
   // loop to decode each column data
   if (hdr.szBlkCol == 0) goto _exit;
@@ -1679,8 +1663,8 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin
         if (code) goto _exit;
       }
     } else {
-      code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &blockCol, hdr.cmprAlg,
-                               hdr.nRow, pColData, &aBuf[0]);
+      code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData,
+                               &aBuf[0]);
       if (code) goto _exit;
     }
   }
@@ -2062,12 +2046,6 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
   }
   size += pBlockCol->szValue;
 
-  // checksum
-  size += sizeof(TSCKSUM);
-  code = tRealloc(ppOut, nOut + size);
-  if (code) goto _exit;
-  taosCalcChecksumAppend(0, *ppOut + nOut, size);
-
 _exit:
   return code;
 }
@@ -2076,12 +2054,6 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
                           uint8_t **ppBuf) {
   int32_t code = 0;
 
-  int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM);
-  if (!taosCheckChecksumWhole(pIn, size)) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _exit;
-  }
-
   ASSERT(pColData->cid == pBlockCol->cid);
   ASSERT(pColData->type == pBlockCol->type);
   pColData->smaOn = pBlockCol->smaOn;
@@ -2153,37 +2125,3 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
 _exit:
   return code;
 }
-
-int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) {
-  int32_t code = 0;
-
-  // alloc
-  code = tRealloc(ppOut, size);
-  if (code) goto _exit;
-
-  // seek
-  int64_t n = taosLSeekFile(pFD, offset, SEEK_SET);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _exit;
-  }
-
-  // read
-  n = taosReadFile(pFD, *ppOut, size);
-  if (n < 0) {
-    code = TAOS_SYSTEM_ERROR(errno);
-    goto _exit;
-  } else if (n < size) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _exit;
-  }
-
-  // check
-  if (toCheck && !taosCheckChecksumWhole(*ppOut, size)) {
-    code = TSDB_CODE_FILE_CORRUPTED;
-    goto _exit;
-  }
-
-_exit:
-  return code;
-}

source/libs/executor/inc/executorimpl.h

@@ -591,6 +591,24 @@ typedef struct SMergeAlignedIntervalAggOperatorInfo {
   SNode*       pCondition;
 } SMergeAlignedIntervalAggOperatorInfo;
 
+typedef struct SStreamIntervalOperatorInfo {
+  // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
+  SOptrBasicInfo     binfo;              // basic info
+  SAggSupporter      aggSup;             // aggregate supporter
+  SExprSupp          scalarSupp;         // supporter for perform scalar function
+  SGroupResInfo      groupResInfo;       // multiple results build supporter
+  SInterval          interval;           // interval info
+  int32_t            primaryTsIndex;     // primary time stamp slot id from result of downstream operator.
+  STimeWindowAggSupp twAggSup;
+  bool               invertible;
+  bool               ignoreExpiredData;
+  SArray*            pRecycledPages;
+  SArray*            pDelWins;           // SWinRes
+  int32_t            delIndex;
+  SSDataBlock*       pDelRes;
+  bool               isFinal;
+} SStreamIntervalOperatorInfo;
+
 typedef struct SStreamFinalIntervalOperatorInfo {
   // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode
   SOptrBasicInfo     binfo;           // basic info
@@ -1003,6 +1021,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh
                                                   SExecTaskInfo* pTaskInfo);
 SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                        SExecTaskInfo* pTaskInfo, int32_t numOfChild);
+SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream,
+                                                SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo);
 
 SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
                                                 SExecTaskInfo* pTaskInfo);

source/libs/executor/src/executor.c

@@ -97,6 +97,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu
   }
 }
 
+static FORCE_INLINE void streamInputBlockDataDestory(void* pBlock) { blockDataDestroy((SSDataBlock*)pBlock); }
+
 void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) {
   SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo;
   if (!pTaskInfo || !pTaskInfo->pRoot || pTaskInfo->pRoot->numOfDownstream <= 0) {
@@ -107,11 +109,7 @@ void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) {
   if (pOptrInfo->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) {
     SStreamScanInfo* pInfo = pOptrInfo->info;
     if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) {
-      for (int32_t i = 0; i < taosArrayGetSize(pInfo->pBlockLists); ++i) {
-        SSDataBlock* p = *(SSDataBlock**)taosArrayGet(pInfo->pBlockLists, i);
-        taosArrayDestroy(p->pDataBlock);
-        taosMemoryFreeClear(p);
-      }
+      taosArrayClearP(pInfo->pBlockLists, streamInputBlockDataDestory);
     } else {
       ASSERT(0);
     }

source/libs/executor/src/executorimpl.c

@@ -3913,7 +3913,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
       pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions,
                                           pScalarExprInfo, numOfScalarExpr, pAggNode->mergeDataBlock, pTaskInfo);
     }
-  } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) {
+  } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) {
     SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode;
 
     SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num);
@@ -3938,6 +3938,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo
     pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode,
                                        pTaskInfo, isStream);
 
+  } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { 
+    pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo);
   } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) {
     SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode;
     pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo);

source/libs/executor/src/timewindowoperator.c

@@ -939,7 +939,7 @@ bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) {
 bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); }
 
 static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock,
-                            int32_t scanFlag, SHashObj* pUpdatedMap) {
+                            int32_t scanFlag) {
   SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info;
 
   SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
@@ -955,21 +955,11 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
 
   STimeWindow win =
       getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder);
-  int32_t ret = TSDB_CODE_SUCCESS;
-  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
-      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
-    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
+  int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
   if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
     T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
   }
-
-    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
-      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
-      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
-    }
-  }
-
   TSKEY   ekey = ascScan ? win.ekey : win.skey;
   int32_t forwardRows =
       getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
@@ -991,12 +981,9 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
     doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup);
   }
 
-  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
-      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
   updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
   doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
                     numOfOutput);
-  }
 
   doCloseWindow(pResultRowInfo, pInfo, pResult);
 
@@ -1007,13 +994,6 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
     if (startPos < 0) {
       break;
     }
-    if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
-      ekey = ascScan ? nextWin.ekey : nextWin.skey;
-      forwardRows =
-          getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
-      continue;
-    }
-
     // null data, failed to allocate more memory buffer
     int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
                                           pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
@@ -1021,11 +1001,6 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul
       T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
     }
 
-    if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
-      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
-      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
-    }
-
     ekey = ascScan ? nextWin.ekey : nextWin.skey;
     forwardRows =
         getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder);
@@ -1130,7 +1105,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) {
     setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, scanFlag, true);
     blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex);
 
-    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL);
+    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag);
   }
 
   initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder);
@@ -1581,141 +1556,6 @@ static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlo
   }
 }
 
-static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
-  SIntervalAggOperatorInfo* pInfo = pOperator->info;
-  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
-
-  pInfo->inputOrder = TSDB_ORDER_ASC;
-  SExprSupp* pSup = &pOperator->exprSupp;
-
-  if (pOperator->status == OP_EXEC_DONE) {
-    return NULL;
-  }
-
-  if (pOperator->status == OP_RES_TO_RETURN) {
-    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
-    if (pInfo->pDelRes->info.rows > 0) {
-      printDataBlock(pInfo->pDelRes, "single interval");
-      return pInfo->pDelRes;
-    }
-
-    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
-    if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
-      pOperator->status = OP_EXEC_DONE;
-      qDebug("===stream===single interval is done");
-      freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
-    }
-    printDataBlock(pInfo->binfo.pRes, "single interval");
-    return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
-  }
-
-  SOperatorInfo* downstream = pOperator->pDownstream[0];
-
-  SArray*    pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
-  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
-  SHashObj*  pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
-
-  SStreamState* pState = pTaskInfo->streamInfo.pState;
-
-  while (1) {
-    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
-    if (pBlock == NULL) {
-      break;
-    }
-    // qInfo("===stream===%ld", pBlock->info.version);
-    printDataBlock(pBlock, "single interval recv");
-
-    if (pBlock->info.type == STREAM_CLEAR) {
-      doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock,
-                     NULL);
-      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
-      continue;
-    }
-    if (pBlock->info.type == STREAM_DELETE_DATA) {
-      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval, pUpdatedMap);
-      continue;
-    } else if (pBlock->info.type == STREAM_GET_ALL) {
-      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
-      continue;
-    }
-
-    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {
-      // set input version
-      pTaskInfo->version = pBlock->info.version;
-    }
-
-    if (pInfo->scalarSupp.pExprInfo != NULL) {
-      SExprSupp* pExprSup = &pInfo->scalarSupp;
-      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
-    }
-
-    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
-    // caller. Note that all the time window are not close till now.
-    // the pDataBlock are always the same one, no need to call this again
-    setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, MAIN_SCAN, true);
-    if (pInfo->invertible) {
-      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
-    }
-
-    pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey);
-    hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap);
-  }
-
-#if 0
-  if (pState) {
-    printf(">>>>>>>> stream read backend\n");
-    SWinKey key = {
-        .ts = 1,
-        .groupId = 2,
-    };
-    char*   val = NULL;
-    int32_t sz;
-    if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) {
-      ASSERT(0);
-    }
-    printf("stream read %s %d\n", val, sz);
-    streamFreeVal(val);
-
-    SStreamStateCur* pCur = streamStateGetCur(pState, &key);
-    ASSERT(pCur);
-    while (streamStateCurNext(pState, pCur) == 0) {
-      SWinKey     key1;
-      const void* val1;
-      if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) {
-        break;
-      }
-      printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz);
-    }
-    streamStateFreeCur(pCur);
-  }
-#endif
-
-  pOperator->status = OP_RES_TO_RETURN;
-  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
-                      pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
-
-  void* pIte = NULL;
-  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
-    taosArrayPush(pUpdated, pIte);
-  }
-  taosArraySort(pUpdated, resultrowComparAsc);
-
-  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
-  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
-  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
-  removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
-  taosHashCleanup(pUpdatedMap);
-  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
-  if (pInfo->pDelRes->info.rows > 0) {
-    printDataBlock(pInfo->pDelRes, "single interval");
-    return pInfo->pDelRes;
-  }
-
-  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
-  printDataBlock(pInfo->binfo.pRes, "single interval");
-  return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
-}
-
 static void destroyStateWindowOperatorInfo(void* param) {
   SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param;
   cleanupBasicInfo(&pInfo->binfo);
@@ -1925,7 +1765,7 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo*
   pOperator->status = OP_NOT_OPENED;
   pOperator->info = pInfo;
 
-  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL,
+  pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL,
                                          destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
 
   if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) {
@@ -5627,3 +5467,311 @@ _error:
   pTaskInfo->code = code;
   return NULL;
 }
+
+static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo,
+    SSDataBlock* pBlock, int32_t scanFlag, SHashObj* pUpdatedMap) {
+  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info;
+
+  SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo;
+  SExprSupp*     pSup = &pOperatorInfo->exprSupp;
+
+  int32_t     startPos = 0;
+  int32_t     numOfOutput = pSup->numOfExprs;
+  SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex);
+  TSKEY*      tsCols = (TSKEY*)pColDataInfo->pData;
+  uint64_t    tableGroupId = pBlock->info.groupId;
+  bool        ascScan = true;
+  TSKEY       ts = getStartTsKey(&pBlock->info.window, tsCols);
+  SResultRow* pResult = NULL;
+
+  STimeWindow win =
+      getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC);
+  int32_t ret = TSDB_CODE_SUCCESS;
+  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
+      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
+    ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx,
+                                 numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
+    if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
+      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
+    }
+    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
+      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
+      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
+    }
+  }
+
+  TSKEY   ekey = ascScan ? win.ekey : win.skey;
+  int32_t forwardRows =
+      getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
+  ASSERT(forwardRows > 0);
+
+  if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) &&
+      inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) {
+    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true);
+    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
+                     numOfOutput);
+  }
+
+  STimeWindow nextWin = win;
+  while (1) {
+    int32_t prevEndPos = forwardRows - 1 + startPos;
+    startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC);
+    if (startPos < 0) {
+      break;
+    }
+    if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) {
+      ekey = ascScan ? nextWin.ekey : nextWin.skey;
+      forwardRows =
+          getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
+      continue;
+    }
+
+    // null data, failed to allocate more memory buffer
+    int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId,
+                                          pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo);
+    if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
+      T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
+    }
+
+    if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) {
+      saveWinResultRow(pResult, tableGroupId, pUpdatedMap);
+      setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur);
+    }
+
+    ekey = ascScan ? nextWin.ekey : nextWin.skey;
+    forwardRows =
+        getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC);
+    updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true);
+    doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows,
+                     numOfOutput);
+  }
+}
+
+static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) {
+  SStreamIntervalOperatorInfo* pInfo = pOperator->info;
+  SExecTaskInfo*            pTaskInfo = pOperator->pTaskInfo;
+  int64_t                   maxTs = INT64_MIN;
+  SExprSupp* pSup = &pOperator->exprSupp;
+
+  if (pOperator->status == OP_EXEC_DONE) {
+    return NULL;
+  }
+
+  if (pOperator->status == OP_RES_TO_RETURN) {
+    doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
+    if (pInfo->pDelRes->info.rows > 0) {
+      printDataBlock(pInfo->pDelRes, "single interval");
+      return pInfo->pDelRes;
+    }
+
+    doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
+    if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) {
+      pOperator->status = OP_EXEC_DONE;
+      qDebug("===stream===single interval is done");
+      freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
+    }
+    printDataBlock(pInfo->binfo.pRes, "single interval");
+    return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
+  }
+
+  SOperatorInfo* downstream = pOperator->pDownstream[0];
+
+  SArray*    pUpdated = taosArrayInit(4, POINTER_BYTES);  // SResKeyPos
+  _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY);
+  SHashObj*  pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK);
+
+  SStreamState* pState = pTaskInfo->streamInfo.pState;
+
+  while (1) {
+    SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream);
+    if (pBlock == NULL) {
+      break;
+    }
+    printDataBlock(pBlock, "single interval recv");
+
+    if (pBlock->info.type == STREAM_CLEAR) {
+      doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock,
+                     NULL);
+      qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo));
+      continue;
+    } else if (pBlock->info.type == STREAM_DELETE_DATA) {
+      doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval, pUpdatedMap);
+      continue;
+    } else if (pBlock->info.type == STREAM_GET_ALL) {
+      getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap);
+      continue;
+    }
+
+    if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) {
+      // set input version
+      pTaskInfo->version = pBlock->info.version;
+    }
+
+    if (pInfo->scalarSupp.pExprInfo != NULL) {
+      SExprSupp* pExprSup = &pInfo->scalarSupp;
+      projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL);
+    }
+
+    // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the
+    // caller. Note that all the time window are not close till now.
+    // the pDataBlock are always the same one, no need to call this again
+    setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true);
+    if (pInfo->invertible) {
+      setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type);
+    }
+
+    maxTs = TMAX(maxTs, pBlock->info.window.ekey);
+    doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap);
+  }
+  pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs);
+
+#if 0
+  if (pState) {
+    printf(">>>>>>>> stream read backend\n");
+    SWinKey key = {
+        .ts = 1,
+        .groupId = 2,
+    };
+    char*   val = NULL;
+    int32_t sz;
+    if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) {
+      ASSERT(0);
+    }
+    printf("stream read %s %d\n", val, sz);
+    streamFreeVal(val);
+
+    SStreamStateCur* pCur = streamStateGetCur(pState, &key);
+    ASSERT(pCur);
+    while (streamStateCurNext(pState, pCur) == 0) {
+      SWinKey     key1;
+      const void* val1;
+      if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) {
+        break;
+      }
+      printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz);
+    }
+    streamStateFreeCur(pCur);
+  }
+#endif
+
+  pOperator->status = OP_RES_TO_RETURN;
+  closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap,
+                      pInfo->pRecycledPages, pInfo->aggSup.pResultBuf);
+
+  void* pIte = NULL;
+  while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) {
+    taosArrayPush(pUpdated, pIte);
+  }
+  taosArraySort(pUpdated, resultrowComparAsc);
+
+  finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset);
+  initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated);
+  blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity);
+  removeDeleteResults(pUpdatedMap, pInfo->pDelWins);
+  taosHashCleanup(pUpdatedMap);
+  doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes);
+  if (pInfo->pDelRes->info.rows > 0) {
+    printDataBlock(pInfo->pDelRes, "single interval");
+    return pInfo->pDelRes;
+  }
+
+  doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf);
+  printDataBlock(pInfo->binfo.pRes, "single interval");
+  return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes;
+}
+
+void destroyStreamIntervalOperatorInfo(void* param) {
+  SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param;
+  cleanupBasicInfo(&pInfo->binfo);
+  cleanupAggSup(&pInfo->aggSup);
+  pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages);
+
+  pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins);
+  pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes);
+
+  cleanupGroupResInfo(&pInfo->groupResInfo);
+  colDataDestroy(&pInfo->twAggSup.timeWindowData);
+  taosMemoryFreeClear(param);
+}
+
+SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode,
+                                                SExecTaskInfo* pTaskInfo) {
+  SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo));
+  SOperatorInfo*            pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo));
+  if (pInfo == NULL || pOperator == NULL) {
+    goto _error;
+  }
+  SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode;
+
+  int32_t numOfCols = 0;
+  SExprInfo*   pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols);
+  ASSERT(numOfCols > 0);
+  SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc);
+  SInterval interval = {.interval = pIntervalPhyNode->interval,
+                        .sliding = pIntervalPhyNode->sliding,
+                        .intervalUnit = pIntervalPhyNode->intervalUnit,
+                        .slidingUnit = pIntervalPhyNode->slidingUnit,
+                        .offset = pIntervalPhyNode->offset,
+                        .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, };
+  STimeWindowAggSupp twAggSupp = {.waterMark = pIntervalPhyNode->window.watermark,
+                                  .calTrigger = pIntervalPhyNode->window.triggerType,
+                                  .maxTs = INT64_MIN, };
+  ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY);
+  pOperator->pTaskInfo = pTaskInfo;
+  pInfo->interval = interval;
+  pInfo->twAggSup = twAggSupp;
+  pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired;
+  pInfo->isFinal = false;
+
+  if (pIntervalPhyNode->window.pExprs != NULL) {
+    int32_t    numOfScalar = 0;
+    SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar);
+    int32_t    code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar);
+    if (code != TSDB_CODE_SUCCESS) {
+      goto _error;
+    }
+  }
+
+  pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId;;
+  initResultSizeInfo(&pOperator->resultInfo, 4096);
+  SExprSupp* pSup = &pOperator->exprSupp;
+  size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES;
+  int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  initBasicInfo(&pInfo->binfo, pResBlock);
+  initStreamFunciton(pSup->pCtx, pSup->numOfExprs);
+  initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window);
+
+  pInfo->invertible = allInvertible(pSup->pCtx, numOfCols);
+  pInfo->invertible = false;  // Todo(liuyao): Dependent TSDB API
+  pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t));
+  pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey));
+  pInfo->delIndex = 0;
+  pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT);
+  initResultRowInfo(&pInfo->binfo.resultRowInfo);
+
+  pOperator->name = "StreamIntervalOperator";
+  pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL;
+  pOperator->blocking = true;
+  pOperator->status = OP_NOT_OPENED;
+  pOperator->info = pInfo;
+  pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL,
+                                         destroyStreamIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL);
+
+  initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark);
+  code = appendDownstream(pOperator, &downstream, 1);
+  if (code != TSDB_CODE_SUCCESS) {
+    goto _error;
+  }
+
+  return pOperator;
+
+_error:
+  destroyStreamIntervalOperatorInfo(pInfo);
+  taosMemoryFreeClear(pOperator);
+  pTaskInfo->code = code;
+  return NULL;
+}

tests/script/tsim/stream/basic1.sim

@@ -588,4 +588,38 @@ if $data00 != 5 then
   goto loop3
 endi
 
+#max,min selectivity
+sql create database test3  vgroups 1;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ;
+
+sql insert into ts1 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts1 values(1648791222001,2,2,3);
+sleep 50
+
+$loop_count = 0
+loop3:
+sql select * from streamtST3;
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+# row 1
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop3
+endi
+
 system sh/exec.sh -n dnode1 -s stop -x SIGINT

tests/script/tsim/stream/distributeInterval0.sim

@@ -198,7 +198,7 @@ endi
 sql select  _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b)  c4, min(c) c5, avg(d) from st interval(10s);
 
 
-sql create database test1  vgroups 1;
+sql create database test1  vgroups 4;
 sql use test1;
 sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
 sql create table ts1 using st tags(1,1,1);
@@ -232,4 +232,43 @@ if $data11 != 2 then
   goto loop2
 endi
 
+#max,min selectivity
+sql create database test3  vgroups 4;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int);
+sql create table ts1 using st tags(1,1,1);
+sql create table ts2 using st tags(2,2,2);
+sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ;
+
+sql insert into ts1 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts1 values(1648791222001,2,2,3);
+sleep 50
+sql insert into ts2 values(1648791211000,1,2,3);
+sleep 50
+sql insert into ts2 values(1648791222001,2,2,3);
+sleep 50
+
+$loop_count = 0
+loop3:
+sql select * from streamtST3;
+
+sleep 300
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+  return -1
+endi
+
+# row 0
+if $data02 != 1 then
+  print =====data02=$data02
+  goto loop3
+endi
+
+# row 1
+if $data12 != 2 then
+  print =====data12=$data12
+  goto loop3
+endi
+
 system sh/stop_dnodes.sh

tests/script/tsim/stream/partitionbyColumn0.sim

@@ -24,7 +24,7 @@ sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop0:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -48,7 +48,7 @@ sql insert into t1 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop1:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -71,7 +71,7 @@ sql insert into t1 values(1648791213000,2,2,3,1.0);
 $loop_count = 0
 
 loop2:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -97,7 +97,7 @@ sql insert into t1 values(1648791213002,1,2,3,1.0);
 $loop_count = 0
 
 loop3:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -134,7 +134,7 @@ sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (16
 $loop_count = 0
 
 loop4:
-sleep 100
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -208,7 +208,7 @@ sql insert into t1 values(1648791213001,1,2,3,2.0);
 $loop_count = 0
 
 loop5:
-sleep 100
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -229,7 +229,7 @@ sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
 $loop_count = 0
 
 loop6:
-sleep 100
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -294,7 +294,7 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop7:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -318,7 +318,7 @@ sql insert into t2 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop8:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -342,7 +342,7 @@ sql insert into t2 values(1648791213000,2,2,3,1.0);
 $loop_count = 0
 
 loop9:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -372,7 +372,7 @@ sql insert into t2 values(1648791213002,1,2,3,1.0);
 $loop_count = 0
 
 loop10:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -414,7 +414,7 @@ sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (16
 $loop_count = 0
 
 loop11:
-sleep 100
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -492,7 +492,7 @@ sql insert into t4 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop13:
-sleep 100
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
@@ -534,7 +534,7 @@ sql insert into t1 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop14:
-sleep 100
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1

tests/script/tsim/stream/partitionbyColumn1.sim

@@ -24,11 +24,11 @@ sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop0:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -45,12 +45,14 @@ endi
 
 sql insert into t1 values(1648791213000,1,2,3,1.0);
 
+$loop_count = 0
+
 loop1:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -66,12 +68,14 @@ endi
 
 sql insert into t1 values(1648791213000,2,2,3,1.0);
 
+$loop_count = 0
+
 loop2:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -90,12 +94,14 @@ sql insert into t1 values(1648791213001,2,2,3,1.0);
 sql insert into t1 values(1648791213002,2,2,3,1.0);
 sql insert into t1 values(1648791213002,1,2,3,1.0);
 
+$loop_count = 0
+
 loop3:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -125,12 +131,14 @@ sql insert into t1 values(1648791223002,3,2,3,1.0);
 sql insert into t1 values(1648791223003,3,2,3,1.0);
 sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
 
+$loop_count = 0
+
 loop4:
-sleep 300
+sleep 50
 sql select * from streamt order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -199,11 +207,11 @@ sql insert into t1 values(1648791213001,1,2,3,2.0);
 $loop_count = 0
 
 loop5:
-sleep 300
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -217,12 +225,14 @@ sql insert into t1 values(1648791223001,1,2,5,2.0);
 sql insert into t1 values(1648791223002,1,2,5,2.0);
 sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0);
 
+$loop_count = 0
+
 loop6:
-sleep 300
+sleep 50
 sql select * from streamt1 order by c1, c4, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -282,11 +292,11 @@ sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
 $loop_count = 0
 
 loop7:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -303,12 +313,14 @@ endi
 sql insert into t1 values(1648791213000,1,2,3,1.0);
 sql insert into t2 values(1648791213000,1,2,3,1.0);
 
+$loop_count = 0
+
 loop8:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -324,12 +336,15 @@ endi
 
 sql insert into t1 values(1648791213000,2,2,3,1.0);
 sql insert into t2 values(1648791213000,2,2,3,1.0);
+
+$loop_count = 0
+
 loop9:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -352,12 +367,14 @@ sql insert into t2 values(1648791213001,2,2,3,1.0);
 sql insert into t2 values(1648791213002,2,2,3,1.0);
 sql insert into t2 values(1648791213002,1,2,3,1.0);
 
+$loop_count = 0
+
 loop10:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -373,7 +390,7 @@ endi
 
 if $data11 != 2 thenloop4
   print =====data11=$data11
-  goto loop3
+  goto loop10
 endi
 
 if $data12 != 1 then
@@ -392,12 +409,14 @@ sql insert into t2 values(1648791223002,3,2,3,1.0);
 sql insert into t2 values(1648791223003,3,2,3,1.0);
 sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
 
+$loop_count = 0
+
 loop11:
-sleep 300
+sleep 50
 sql select * from test.streamt2 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
@@ -470,17 +489,17 @@ sql insert into t4 values(1648791213000,1,2,3,1.0);
 $loop_count = 0
 
 loop13:
-sleep 300
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 
 if $rows != 2 then
   print =====rows=$rows
-  goto loop14
+  goto loop13
 endi
 
 if $data01 != 1 then
@@ -495,12 +514,12 @@ endi
 
 if $data11 != 3 then
   print =====data11=$data11
-  goto loop11
+  goto loop13
 endi
 
 if $data12 != 2 then
   print =====data12=$data12
-  goto loop11
+  goto loop13
 endi
 
 sql insert into t4 values(1648791213000,2,2,3,1.0);
@@ -509,12 +528,14 @@ sql insert into t1 values(1648791233000,2,2,3,1.0);
 
 sql insert into t1 values(1648791213000,1,2,3,1.0); 
 
+$loop_count = 0
+
 loop14:
-sleep 300
+sleep 50
 sql select * from test.streamt4 order by c1, c2, c3;
 
 $loop_count = $loop_count + 1
-if $loop_count == 10 then
+if $loop_count == 20 then
   return -1
 endi
 

tests/script/tsim/stream/partitionbyColumn2.sim

@@ -40,6 +40,8 @@ endi
 
 sql insert into t1 values(1648791213000,1,1,3,1.0);
 
+$loop_count = 0
+
 loop1:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -61,6 +63,8 @@ endi
 
 sql insert into t1 values(1648791213000,2,1,3,1.0);
 
+$loop_count = 0
+
 loop2:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -85,6 +89,8 @@ sql insert into t1 values(1648791213001,2,1,3,1.0);
 sql insert into t1 values(1648791213002,2,1,3,1.0);
 sql insert into t1 values(1648791213002,1,1,3,1.0);
 
+$loop_count = 0
+
 loop3:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -120,6 +126,8 @@ sql insert into t1 values(1648791223002,3,2,3,1.0);
 sql insert into t1 values(1648791223003,3,2,3,1.0);
 sql insert into t1 values(1648791213001,1,1,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0);
 
+$loop_count = 0
+
 loop4:
 sleep 300
 sql select * from streamt order by c1, c4, c2, c3;
@@ -212,6 +220,8 @@ sql insert into t1 values(1648791223001,1,2,2,5);
 sql insert into t1 values(1648791223002,1,2,2,6);
 sql insert into t1 values(1648791213001,1,1,1,7) (1648791223002,1,1,2,8);
 
+$loop_count = 0
+
 loop6:
 sleep 300
 sql select * from streamt1 order by c1, c4, c2, c3;

utils/test/c/sml_test.c

@@ -92,7 +92,7 @@ int smlProcess_telnet_Test() {
 int smlProcess_json1_Test() {
   TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0);
 
-  TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1");
+  TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db");
   taos_free_result(pRes);
 
   pRes = taos_query(taos, "use sml_db");
@@ -112,7 +112,7 @@ int smlProcess_json1_Test() {
       "    },"
       "    {"
       "        \"metric\": \"sys.cpu.nice\","
-      "        \"timestamp\": 1346846400,"
+      "        \"timestamp\": 1662344042,"
       "        \"value\": 9,"
       "        \"tags\": {"
       "           \"host\": \"web02\","
@@ -141,7 +141,7 @@ int smlProcess_json2_Test() {
       "{"
       "    \"metric\": \"meter_current0\","
       "    \"timestamp\": {"
-      "        \"value\"  : 1346846400,"
+      "        \"value\"  : 1662344042,"
       "        \"type\"   : \"s\""
       "    },"
       "    \"value\": {"
@@ -181,7 +181,7 @@ int smlProcess_json3_Test() {
       "{"
       "    \"metric\": \"meter_current1\","
       "    \"timestamp\": {"
-      "        \"value\"  : 1346846400,"
+      "        \"value\"  : 1662344042,"
       "        \"type\"   : \"s\""
       "    },"
       "    \"value\": {"
@@ -249,7 +249,7 @@ int smlProcess_json4_Test() {
       "{"
       "    \"metric\": \"meter_current2\","
       "    \"timestamp\": {"
-      "        \"value\"  : 1346846500000,"
+      "        \"value\"  : 1662344042000,"
       "        \"type\"   : \"ms\""
       "    },"
       "    \"value\": \"ni\","