Merge branch 'develop' into test/jenkins

CMakeLists.txt

@@ -15,6 +15,7 @@ SET(TD_ADMIN FALSE)
 SET(TD_GRANT FALSE)
 SET(TD_MQTT FALSE)
 SET(TD_TSDB_PLUGINS FALSE)
+SET(TD_STORAGE FALSE)
 
 SET(TD_COVER FALSE)
 SET(TD_MEM_CHECK FALSE)

cmake/define.inc

@@ -21,6 +21,10 @@ IF (TD_TSDB_PLUGINS)
   ADD_DEFINITIONS(-D_TSDB_PLUGINS)
 ENDIF ()
 
+IF (TD_STORAGE)
+  ADD_DEFINITIONS(-D_STORAGE)
+ENDIF ()
+
 IF (TD_GODLL)
   ADD_DEFINITIONS(-D_TD_GO_DLL_)
 ENDIF ()

src/CMakeLists.txt

@@ -5,6 +5,7 @@ PROJECT(TDengine)
 ADD_SUBDIRECTORY(os)
 ADD_SUBDIRECTORY(common)
 ADD_SUBDIRECTORY(util)
+ADD_SUBDIRECTORY(tfs)
 ADD_SUBDIRECTORY(rpc)
 ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(query)

src/client/src/tscSub.c

@@ -61,7 +61,7 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = 0};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
   if (p == NULL) {
     return dflt;
   }
@@ -76,7 +76,7 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = ts};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
   if (p != NULL) {
     p->key = ts;
     tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key);
@@ -270,7 +270,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
     STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
     SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0};
-    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
     if (p == NULL) {
       taosArrayClear(pSub->progress);
       taosArrayPush(pSub->progress, &target);

src/client/src/tscSystem.c

@@ -18,7 +18,6 @@
 #include "tref.h"
 #include "trpc.h"
 #include "tnote.h"
-#include "tsystem.h"
 #include "ttimer.h"
 #include "tutil.h"
 #include "tsched.h"
@@ -49,7 +48,7 @@ int32_t   tscNumOfThreads = 1;     // num of rpc threads
 static    pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently 
 static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
 
-void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) {
+void tscCheckDiskUsage(void *UNUSED_PARAM(para), void *UNUSED_PARAM(param)) {
   taosGetDisk();
   taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr);
 }

src/common/inc/tdataformat.h

@@ -68,9 +68,9 @@ typedef struct {
 typedef struct {
   int      version;    // version
   int      numOfCols;  // Number of columns appended
-  int      tlen;       // maximum length of a SDataRow without the header part
+  int      tlen;       // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes))
   uint16_t flen;       // First part length in a SDataRow after the header part
-  uint16_t vlen;       // pure value part length, excluded the overhead
+  uint16_t vlen;       // pure value part length, excluded the overhead (bytes only)
   STColumn columns[];
 } STSchema;
 
@@ -278,7 +278,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);
 int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
-void       tdFreeDataCols(SDataCols *pCols);
+SDataCols *tdFreeDataCols(SDataCols *pCols);
 void       tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
 int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
 

src/common/inc/tglobal.h

@@ -16,6 +16,8 @@
 #ifndef TDENGINE_COMMON_GLOBAL_H
 #define TDENGINE_COMMON_GLOBAL_H
 
+#include "taosdef.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -147,7 +149,6 @@ extern char    tsDataDir[];
 extern char    tsLogDir[];
 extern char    tsScriptDir[];
 extern int64_t tsMsPerDay[3];
-extern char    tsVnodeBakDir[];
 
 // system info
 extern char    tsOsName[];
@@ -196,6 +197,14 @@ extern int32_t wDebugFlag;
 extern int32_t cqDebugFlag;
 extern int32_t debugFlag;
 
+typedef struct {
+  char dir[TSDB_FILENAME_LEN];
+  int  level;
+  int  primary;
+} SDiskCfg;
+extern int32_t  tsDiskCfgNum;
+extern SDiskCfg tsDiskCfg[];
+
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 void    taosInitGlobalCfg();
@@ -204,6 +213,9 @@ void    taosSetAllDebugFlag();
 bool    taosCfgDynamicOptions(char *msg);
 int     taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
 bool    taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
+void    taosAddDataDir(int index, char *v1, int level, int primary);
+void    taosReadDataDirCfg(char *v1, char *v2, char *v3);
+void    taosPrintDataDirCfg();
 
 #ifdef __cplusplus
 }

src/common/src/tdataformat.c

@@ -289,23 +289,31 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
     return NULL;
   }
 
-  pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
-  if (pCols->cols == NULL) {
-    uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno));
-    tdFreeDataCols(pCols);
-    return NULL;
+  pCols->maxPoints = maxRows;
+
+  if (maxCols > 0) {
+    pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
+    if (pCols->cols == NULL) {
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols,
+             strerror(errno));
+      tdFreeDataCols(pCols);
+      return NULL;
+    }
+
+    pCols->maxCols = maxCols;
   }
 
   pCols->maxRowSize = maxRowSize;
-  pCols->maxCols = maxCols;
-  pCols->maxPoints = maxRows;
   pCols->bufSize = maxRowSize * maxRows;
 
-  pCols->buf = malloc(pCols->bufSize);
-  if (pCols->buf == NULL) {
-    uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno));
-    tdFreeDataCols(pCols);
-    return NULL;
+  if (pCols->bufSize > 0) {
+    pCols->buf = malloc(pCols->bufSize);
+    if (pCols->buf == NULL) {
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols,
+             strerror(errno));
+      tdFreeDataCols(pCols);
+      return NULL;
+    }
   }
 
   return pCols;
@@ -337,12 +345,13 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
   return 0;
 }
 
-void tdFreeDataCols(SDataCols *pCols) {
+SDataCols *tdFreeDataCols(SDataCols *pCols) {
   if (pCols) {
     tfree(pCols->buf);
     tfree(pCols->cols);
     free(pCols);
   }
+  return NULL;
 }
 
 SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {

src/common/src/tglobal.c

@@ -182,7 +182,14 @@ char   tsDnodeDir[TSDB_FILENAME_LEN] = {0};
 char   tsMnodeDir[TSDB_FILENAME_LEN] = {0};
 char   tsDataDir[TSDB_FILENAME_LEN] = {0};
 char   tsScriptDir[TSDB_FILENAME_LEN] = {0};
-char   tsVnodeBakDir[TSDB_FILENAME_LEN] = {0};
+
+int32_t  tsDiskCfgNum = 0;
+
+#ifndef _STORAGE
+SDiskCfg tsDiskCfg[1];
+#else
+SDiskCfg tsDiskCfg[TSDB_MAX_DISKS];
+#endif
 
 /*
  * minimum scale for whole system, millisecond by default
@@ -227,6 +234,7 @@ int32_t sDebugFlag = 135;
 int32_t wDebugFlag = 135;
 int32_t tsdbDebugFlag = 131;
 int32_t cqDebugFlag = 131;
+int32_t fsDebugFlag = 135;
 
 int32_t (*monStartSystemFp)() = NULL;
 void (*monStopSystemFp)() = NULL;
@@ -334,6 +342,39 @@ bool taosCfgDynamicOptions(char *msg) {
   return false;
 }
 
+void taosAddDataDir(int index, char *v1, int level, int primary) {
+  tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
+  tsDiskCfg[index].level = level;
+  tsDiskCfg[index].primary = primary;
+  uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
+}
+
+#ifndef _STORAGE
+void taosReadDataDirCfg(char *v1, char *v2, char *v3) {
+  if (tsDiskCfgNum == 1) {
+    SDiskCfg *cfg = &tsDiskCfg[0];
+    uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1);
+  }
+  taosAddDataDir(0, v1, 0, 1);
+  tsDiskCfgNum = 1;
+}
+
+void taosPrintDataDirCfg() {
+  for (int i = 0; i < tsDiskCfgNum; ++i) {
+    SDiskCfg *cfg = &tsDiskCfg[i];
+    uInfo(" dataDir: %s", cfg->dir);
+  }
+}
+#endif
+
+static void taosCheckDataDirCfg() {
+  if (tsDiskCfgNum <= 0) {
+    taosAddDataDir(0, tsDataDir, 0, 1);
+    tsDiskCfgNum = 1;
+    uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir);
+  }
+}
+
 static void doInitGlobalConfig(void) {
   osInit();
   srand(taosSafeRand());
@@ -415,7 +456,7 @@ static void doInitGlobalConfig(void) {
 
   cfg.option = "dataDir";
   cfg.ptr = tsDataDir;
-  cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
+  cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
   cfg.minValue = 0;
   cfg.maxValue = 0;
@@ -1448,6 +1489,7 @@ int32_t taosCheckGlobalCfg() {
     snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port);
   }
 
+  taosCheckDataDirCfg();
   taosGetSystemInfo();
 
   tsSetLocale();

src/dnode/CMakeLists.txt

@@ -27,7 +27,7 @@ IF (TD_GRANT)
   TARGET_LINK_LIBRARIES(taosd grant)
 ENDIF ()
 
-IF ((TD_LINUX OR TD_WINDOWS) AND TD_MQTT)
+IF (TD_MQTT)
   TARGET_LINK_LIBRARIES(taosd mqtt)
 ENDIF ()
 
@@ -43,5 +43,6 @@ ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
     COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
+    COMMAND ${CMAKE_COMMAND} -E echo monitor 0  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMENT "prepare taosd environment")
 ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD})

src/dnode/src/dnodeMain.c

@@ -21,7 +21,7 @@
 #include "tconfig.h"
 #include "tfile.h"
 #include "twal.h"
-// #include "tfs.h"
+#include "tfs.h"
 #include "tsync.h"
 #include "dnodeStep.h"
 #include "dnodePeer.h"
@@ -189,32 +189,35 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 }
 
 static int32_t dnodeInitStorage() {
-  if (dnodeCreateDir(tsDataDir) < 0) {
-   dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno));
-   return -1;
+  if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) {
+    dError("failed to init TFS since %s", tstrerror(terrno));
+    return -1;
   }
+  strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN);
   sprintf(tsMnodeDir, "%s/mnode", tsDataDir);
   sprintf(tsVnodeDir, "%s/vnode", tsDataDir);
   sprintf(tsDnodeDir, "%s/dnode", tsDataDir);
-  sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir);
+  // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir);
 
   //TODO(dengyihao): no need to init here 
   if (dnodeCreateDir(tsMnodeDir) < 0) {
    dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno));
    return -1;
   } 
-  //TODO(dengyihao): no need to init here
-  if (dnodeCreateDir(tsVnodeDir) < 0) {
-   dError("failed to create dir: %s, reason: %s", tsVnodeDir, strerror(errno));
-   return -1;
-  }
+
   if (dnodeCreateDir(tsDnodeDir) < 0) {
    dError("failed to create dir: %s, reason: %s", tsDnodeDir, strerror(errno));
    return -1;
-  } 
-  if (dnodeCreateDir(tsVnodeBakDir) < 0) {
-   dError("failed to create dir: %s, reason: %s", tsVnodeBakDir, strerror(errno));
-   return -1;
+  }
+
+  if (tfsMkdir("vnode") < 0) {
+    dError("failed to create vnode dir since %s", tstrerror(terrno));
+    return -1;
+  }
+
+  if (tfsMkdir("vnode_bak") < 0) {
+    dError("failed to create vnode_bak dir since %s", tstrerror(terrno));
+    return -1;
   }
 
   dnodeCheckDataDirOpenned(tsDnodeDir);
@@ -223,7 +226,7 @@ static int32_t dnodeInitStorage() {
   return 0;
 }
 
-static void dnodeCleanupStorage() {}
+static void dnodeCleanupStorage() { tfsDestroy(); }
 
 bool  dnodeIsFirstDeploy() {
   return strcmp(tsFirst, tsLocalEp) == 0;

src/dnode/src/dnodeVMgmt.c

@@ -174,7 +174,7 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
     vnodeRelease(pVnode);
     return code;
   } else {
-    dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
+    dInfo("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
     return TSDB_CODE_VND_INVALID_VGROUP_ID;
   }
 }

src/inc/taoserror.h

@@ -242,6 +242,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM,     0, 0x060F, "No table d
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS,      0, 0x0610, "File already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE,        0, 0x0611, "Need to reconfigure table")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO,    0, 0x0612, "Invalid information to create table")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK,            0, 0x0613, "No available disk")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG,               0, 0x0614, "TSDB messed message")
 
 // query
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE,          0, 0x0700, "Invalid handle")

src/inc/tfs.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_TFS_H
+#define TD_TFS_H
+
+#include "tglobal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  int level;
+  int id;
+} SDiskID;
+
+#define TFS_UNDECIDED_LEVEL -1
+#define TFS_UNDECIDED_ID -1
+#define TFS_PRIMARY_LEVEL 0
+#define TFS_PRIMARY_ID 0
+
+// FS APIs ====================================
+typedef struct {
+  int64_t tsize;
+  int64_t avail;
+} SFSMeta;
+
+int  tfsInit(SDiskCfg *pDiskCfg, int ndisk);
+void tfsDestroy();
+void tfsUpdateInfo(SFSMeta *pFSMeta);
+void tfsGetMeta(SFSMeta *pMeta);
+void tfsAllocDisk(int expLevel, int *level, int *id);
+
+const char *TFS_PRIMARY_PATH();
+const char *TFS_DISK_PATH(int level, int id);
+
+// TFILE APIs ====================================
+typedef struct {
+  int  level;
+  int  id;
+  char rname[TSDB_FILENAME_LEN];  // REL name
+  char aname[TSDB_FILENAME_LEN];  // ABS name
+} TFILE;
+
+#define TFILE_LEVEL(pf) ((pf)->level)
+#define TFILE_ID(pf) ((pf)->id)
+#define TFILE_NAME(pf) ((pf)->aname)
+#define TFILE_REL_NAME(pf) ((pf)->rname)
+
+#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags)
+#define tfsclose(fd) close(fd)
+#define tfsremove(pf) remove(TFILE_NAME(pf))
+#define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df))
+#define tfsrename(sf, df) rename(TFILE_NAME(sf), TFILE_NAME(df))
+
+void  tfsInitFile(TFILE *pf, int level, int id, const char *bname);
+bool  tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
+int   tfsEncodeFile(void **buf, TFILE *pf);
+void *tfsDecodeFile(void *buf, TFILE *pf);
+void  tfsbasename(const TFILE *pf, char *dest);
+void  tfsdirname(const TFILE *pf, char *dest);
+
+// DIR APIs ====================================
+int tfsMkdirAt(const char *rname, int level, int id);
+int tfsMkdirRecurAt(const char *rname, int level, int id);
+int tfsMkdir(const char *rname);
+int tfsRmdir(const char *rname);
+int tfsRename(char *orname, char *nrname);
+
+typedef struct TDIR TDIR;
+
+TDIR *       tfsOpendir(const char *rname);
+const TFILE *tfsReaddir(TDIR *tdir);
+void         tfsClosedir(TDIR *tdir);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file

src/inc/tsdb.h

@@ -40,7 +40,8 @@ extern "C" {
 
 // TSDB STATE DEFINITION
 #define TSDB_STATE_OK 0x0
-#define TSDB_STATE_BAD_FILE 0x1
+#define TSDB_STATE_BAD_META 0x1
+#define TSDB_STATE_BAD_DATA 0x2
 
 // --------- TSDB APPLICATION HANDLE DEFINITION
 typedef struct {
@@ -48,7 +49,7 @@ typedef struct {
   void *cqH;
   int (*notifyStatus)(void *, int status, int eno);
   int (*eventCallBack)(void *);
-  void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema);
+  void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char *dstTable, char *sqlStr, STSchema *pSchema);
   void (*cqDropFunc)(void *handle);
 } STsdbAppH;
 
@@ -76,17 +77,17 @@ typedef struct {
   int64_t pointsWritten;  // total data points written
 } STsdbStat;
 
-typedef void TSDB_REPO_T;  // use void to hide implementation details from outside
+typedef struct STsdbRepo STsdbRepo;
 
-STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo);
+STsdbCfg *tsdbGetCfg(const STsdbRepo *repo);
 
 // --------- TSDB REPOSITORY DEFINITION
-int          tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg);
-int32_t      tsdbDropRepo(char *rootDir);
-TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);
-int          tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit);
-int32_t      tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg);
-int          tsdbGetState(TSDB_REPO_T *repo);
+int32_t    tsdbCreateRepo(int repoid);
+int32_t    tsdbDropRepo(int repoid);
+STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH);
+int        tsdbCloseRepo(STsdbRepo *repo, int toCommit);
+int32_t    tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg);
+int        tsdbGetState(STsdbRepo *repo);
 
 // --------- TSDB TABLE DEFINITION
 typedef struct {
@@ -110,8 +111,8 @@ typedef struct {
 
 void tsdbClearTableCfg(STableCfg *config);
 
-void* tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes);
-char* tsdbGetTableName(void *pTable);
+void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type, int16_t bytes);
+char *tsdbGetTableName(void *pTable);
 
 #define TSDB_TABLEID(_table) ((STableId*) (_table))
 #define TSDB_PREV_ROW  0x1
@@ -119,12 +120,11 @@ char* tsdbGetTableName(void *pTable);
 
 STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg);
 
-int   tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg);
-int   tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId);
-int   tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg);
-// TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid);
+int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg);
+int tsdbDropTable(STsdbRepo *pRepo, STableId tableId);
+int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg);
 
-uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size);
+uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size);
 
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
@@ -134,7 +134,7 @@ typedef struct STsdbRepoInfo {
   int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
   // TODO: Other informations to add
 } STsdbRepoInfo;
-STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo);
+STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo);
 
 // the meter information report structure
 typedef struct {
@@ -152,7 +152,7 @@ typedef struct {
  *
  * @return the number of points inserted, -1 for failure and the error number is set
  */
-int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
+int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
 
 // -- FOR QUERY TIME SERIES DATA
 
@@ -168,9 +168,9 @@ typedef struct STsdbQueryCond {
 } STsdbQueryCond;
 
 typedef struct SMemRef {
-  int32_t  ref;
-  void    *mem;
-  void    *imem;
+  int32_t ref;
+  void *  mem;
+  void *  imem;
 } SMemRef;
 
 typedef struct SDataBlockInfo {
@@ -182,14 +182,14 @@ typedef struct SDataBlockInfo {
 } SDataBlockInfo;
 
 typedef struct {
-  void  *pTable;
-  TSKEY  lastKey;
+  void *pTable;
+  TSKEY lastKey;
 } STableKeyInfo;
 
 typedef struct {
   size_t    numOfTables;
-  SArray   *pGroupList;
-  SHashObj *map;         // speedup acquire the tableQueryInfo by table uid
+  SArray *  pGroupList;
+  SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
 } STableGroupInfo;
 
 /**
@@ -202,7 +202,8 @@ typedef struct {
  * @param qinfo      query info handle from query processor
  * @return
  */
-TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef);
+TsdbQueryHandleT *tsdbQueryTables(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo,
+                                  SMemRef *pRef);
 
 /**
  * Get the last row of the given query time window for all the tables in STableGroupInfo object.
@@ -214,14 +215,15 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
  * @param tableInfo  table list.
  * @return
  */
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef);
+TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo,
+                                  SMemRef *pRef);
 
 /**
  * get the queried table object list
  * @param pHandle
  * @return
  */
-SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
+SArray *tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
 
 /**
  * get the group list according to table id from client
@@ -231,8 +233,8 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
  * @param qinfo
  * @return
  */
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
-                                               void *qinfo, SMemRef* pRef);
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
+                                               void *qinfo, SMemRef *pRef);
 
 
 /**
@@ -268,7 +270,7 @@ SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t
  * @param pBlockInfo
  * @return
  */
-void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo* pBlockInfo);
+void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo);
 
 /**
  *
@@ -299,7 +301,7 @@ SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdL
  * @param stableid. super table sid
  * @param pTagCond. tag query condition
  */
-int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len,
+int32_t tsdbQuerySTableByTagCond(STsdbRepo *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len,
                                  int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList,
                                  SColIndex *pColIndex, int32_t numOfCols);
 
@@ -317,7 +319,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
  * @param pGroupInfo  the generated result
  * @return
  */
-int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
+int32_t tsdbGetOneTableGroup(STsdbRepo *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
 
 /**
  *
@@ -326,7 +328,7 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, ST
  * @param pGroupInfo
  * @return
  */
-int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo);
+int32_t tsdbGetTableGroupFromIdList(STsdbRepo *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
 
 /**
  * clean up the query handle
@@ -345,10 +347,14 @@ void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int
 
 int  tsdbInitCommitQueue();
 void tsdbDestroyCommitQueue();
-int  tsdbSyncCommit(TSDB_REPO_T *repo);
+int  tsdbSyncCommit(STsdbRepo *repo);
 void tsdbIncCommitRef(int vgId);
 void tsdbDecCommitRef(int vgId);
 
+// For TSDB file sync
+int tsdbSyncSend(void *pRepo, SOCKET socketFd);
+int tsdbSyncRecv(void *pRepo, SOCKET socketFd);
+
 #ifdef __cplusplus
 }
 #endif

src/inc/tsync.h

@@ -56,16 +56,6 @@ typedef struct {
   int32_t  role[TAOS_SYNC_MAX_REPLICA];
 } SNodesRole;
 
-/* 
-  if name is empty(name[0] is zero), get the file from index or after, but not larger than eindex. If a file
-  is found between index and eindex, index shall be updated, name shall be set, size shall be set to 
-  file size, and file magic number shall be returned. 
-
-  if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return
-  zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated.
-*/
-typedef uint32_t (*FGetFileInfo)(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); 
-
 // get the wal file from index or after
 // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
 typedef int32_t  (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); 
@@ -83,24 +73,31 @@ typedef void     (*FNotifyRole)(int32_t vgId, int8_t role);
 typedef void     (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
 
 // when data file is synced successfully, notity app
-typedef int32_t  (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion);
+typedef void     (*FStartSyncFile)(int32_t vgId);
+typedef void     (*FStopSyncFile)(int32_t vgId, uint64_t fversion);
 
 // get file version
 typedef int32_t  (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
 
+typedef int32_t  (*FSendFile)(void *tsdb, SOCKET socketFd);
+typedef int32_t  (*FRecvFile)(void *tsdb, SOCKET socketFd);
+
 typedef struct {
   int32_t  vgId;       // vgroup ID
   uint64_t version;    // initial version
   SSyncCfg syncCfg;    // configuration from mgmt
   char     path[TSDB_FILENAME_LEN];  // path to the file
-  FGetFileInfo      getFileInfo;
-  FGetWalInfo       getWalInfo;
-  FWriteToCache     writeToCache;
+  void *   pTsdb;
+  FGetWalInfo       getWalInfoFp;
+  FWriteToCache     writeToCacheFp;
   FConfirmForward   confirmForward;
-  FNotifyRole       notifyRole;
-  FNotifyFlowCtrl   notifyFlowCtrl;
-  FNotifyFileSynced notifyFileSynced;
-  FGetVersion       getVersion;
+  FNotifyRole       notifyRoleFp;
+  FNotifyFlowCtrl   notifyFlowCtrlFp;
+  FStartSyncFile    startSyncFileFp;
+  FStopSyncFile     stopSyncFileFp;
+  FGetVersion       getVersionFp;
+  FSendFile         sendFileFp;
+  FRecvFile         recvFileFp;
 } SSyncInfo;
 
 typedef void *tsync_h;

src/kit/shell/src/shellEngine.c

@@ -470,7 +470,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
 
   wordexp_t full_path;
 
-  if (wordexp(fname, &full_path, 0) != 0) {
+  if (wordexp((char *)fname, &full_path, 0) != 0) {
     fprintf(stderr, "ERROR: invalid file name: %s\n", fname);
     return -1;
   }

src/mnode/src/mnodePeer.c

@@ -17,7 +17,6 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tsched.h"
-#include "tsystem.h"
 #include "tutil.h"
 #include "tgrant.h"
 #include "tbn.h"

src/mnode/src/mnodeSdb.c

@@ -242,11 +242,6 @@ void sdbUpdateMnodeRoles() {
   mnodeUpdateMnodeEpSet(NULL);
 }
 
-static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
-  sdbUpdateMnodeRoles();
-  return 0;
-}
-
 static int32_t sdbGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) {
   return walGetWalFile(tsSdbMgmt.wal, fileName, fileId);
 }
@@ -262,7 +257,9 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) {
   sdbUpdateMnodeRoles();
 }
 
-static int32_t sdbNotifyFileSynced(int32_t vgId, uint64_t fversion) { return 0; }
+static void sdbStartFileSync(int32_t vgId) {}
+
+static void sdbStopFileSync(int32_t vgId, uint64_t fversion) {}
 
 static void sdbNotifyFlowCtrl(int32_t vgId, int32_t level) {}
 
@@ -396,14 +393,14 @@ int32_t sdbUpdateSync(void *pMnodes) {
   syncInfo.version = sdbGetVersion();
   syncInfo.syncCfg = syncCfg;
   sprintf(syncInfo.path, "%s", tsMnodeDir);
-  syncInfo.getFileInfo = sdbGetFileInfo;
-  syncInfo.getWalInfo = sdbGetWalInfo;
-  syncInfo.writeToCache = sdbWriteFwdToQueue;
+  syncInfo.getWalInfoFp = sdbGetWalInfo;
+  syncInfo.writeToCacheFp = sdbWriteFwdToQueue;
   syncInfo.confirmForward = sdbConfirmForward;
-  syncInfo.notifyRole = sdbNotifyRole;
-  syncInfo.notifyFileSynced = sdbNotifyFileSynced;
-  syncInfo.notifyFlowCtrl = sdbNotifyFlowCtrl;
-  syncInfo.getVersion = sdbGetSyncVersion;
+  syncInfo.notifyRoleFp = sdbNotifyRole;
+  syncInfo.startSyncFileFp = sdbStartFileSync;
+  syncInfo.stopSyncFileFp = sdbStopFileSync;
+  syncInfo.notifyFlowCtrlFp = sdbNotifyFlowCtrl;
+  syncInfo.getVersionFp = sdbGetSyncVersion;
   tsSdbMgmt.cfg = syncCfg;
 
   if (tsSdbMgmt.sync) {

src/os/inc/osDarwin.h

@@ -105,6 +105,8 @@ typedef int(*__compar_fn_t)(const void *, const void *);
   #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
 #endif
 
+#define TAOS_OS_FUNC_PTHREAD_RWLOCK
+
 int64_t tsosStr2int64(char *str);
 
 #include "eok.h"

src/os/inc/osFile.h

@@ -26,6 +26,7 @@ int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
 int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
 int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
 int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath);
+int64_t taosCopy(char *from, char *to);
 
 #define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
 #define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)

src/os/inc/osMemory.h

@@ -35,7 +35,7 @@ void   taosDumpMemoryLeak();
 void * taosTMalloc(size_t size);
 void * taosTCalloc(size_t nmemb, size_t size);
 void * taosTRealloc(void *ptr, size_t size);
-void   taosTZfree(void *ptr);
+void * taosTZfree(void *ptr);
 size_t taosTSizeof(void *ptr);
 void   taosTMemset(void *ptr, int c);
 

src/os/inc/osSemphone.h

@@ -28,6 +28,21 @@ extern "C" {
   #define tsem_destroy sem_destroy
 #endif
 
+#ifdef TAOS_OS_FUNC_PTHREAD_RWLOCK
+  #define pthread_rwlock_t pthread_mutex_t
+  #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL)
+  #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock)
+  #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock)
+  #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock)
+  #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock)
+
+  #define pthread_spinlock_t pthread_mutex_t
+  #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL)
+  #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock)
+  #define pthread_spin_lock(lock) pthread_mutex_lock(lock)
+  #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock)
+#endif
+
 // TAOS_OS_FUNC_SEMPHONE_PTHREAD
 bool    taosCheckPthreadValid(pthread_t thread);
 int64_t taosGetSelfPthreadId();

src/os/inc/osSysinfo.h

@@ -21,10 +21,16 @@ extern "C" {
 #endif
 
 // TAOS_OS_FUNC_SYSINFO
+typedef struct {
+  int64_t tsize;
+  int64_t avail;
+} SysDiskSize;
+
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize);
 void taosGetSystemInfo();
 bool taosGetProcIO(float *readKB, float *writeKB);
 bool taosGetBandSpeed(float *bandSpeedKb);
-bool taosGetDisk();
+void taosGetDisk();
 bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ;
 bool taosGetProcMemory(float *memoryUsedMB) ;
 bool taosGetSysMemory(float *memoryUsedMB);

src/os/inc/osWindows.h

@@ -201,13 +201,15 @@ int        gettimeofday(struct timeval *ptv, void *pTimeZone);
 
 typedef struct {
   int    we_wordc;
-  char **we_wordv;
+  char  *we_wordv[1];
   int    we_offs;
-  char   wordPos[20];
+  char   wordPos[1025];
 } wordexp_t;
-int  wordexp(const char *words, wordexp_t *pwordexp, int flags);
+int  wordexp(char *words, wordexp_t *pwordexp, int flags);
 void wordfree(wordexp_t *pwordexp);
 
+char *realpath(char *path, char *resolved_path);
+
 #define openlog(a, b, c)
 #define closelog()
 #define LOG_ERR 0

src/os/src/darwin/darwinSysInfo.c

@@ -18,6 +18,7 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
+#include "taoserror.h"
 #include <errno.h>
 #include <libproc.h>
 
@@ -70,8 +71,6 @@ void taosGetSystemInfo() {
   taosGetSystemLocale();
 }
 
-bool taosGetDisk() { return true; }
-
 bool taosGetProcIO(float *readKB, float *writeKB) {
   *readKB = 0;
   *writeKB = 0;
@@ -106,6 +105,19 @@ int taosSystem(const char *cmd) {
 
 void taosSetCoreDump() {}
 
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
+  struct statvfs info;
+  if (statvfs(tsDataDir, &info)) {
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  } else {
+    diskSize->tsize = info.f_blocks * info.f_frsize;
+    diskSize->avail = info.f_bavail * info.f_frsize;
+    return 0;
+  }
+}
+
 char cmdline[1024];
 
 char *taosGetCmdlineByPID(int pid) {

src/os/src/detail/osFile.c

@@ -119,6 +119,40 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
   return (int64_t)lseek(fd, (long)offset, whence);
 }
 
+int64_t taosCopy(char *from, char *to) {
+  char    buffer[4096];
+  int     fidto = -1, fidfrom = -1;
+  int64_t size = 0;
+  int64_t bytes;
+
+  fidfrom = open(from, O_RDONLY | O_BINARY);
+  if (fidfrom < 0) goto _err;
+
+  fidto = open(to, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755);
+  if (fidto < 0) goto _err;
+
+  while (true) {
+    bytes = taosRead(fidfrom, buffer, sizeof(buffer));
+    if (bytes < 0) goto _err;
+    if (bytes == 0) break;
+
+    size += bytes;
+
+    if (taosWrite(fidto, (void *)buffer, bytes) < bytes) goto _err;
+    if (bytes < sizeof(buffer)) break;
+  }
+
+  close(fidfrom);
+  close(fidto);
+  return size;
+
+_err:
+  if (fidfrom >= 0) close(fidfrom);
+  if (fidto >= 0) close(fidto);
+  remove(to);
+  return -1;
+}
+
 #ifndef TAOS_OS_FUNC_FILE_SENDIFLE
 
 int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) {

src/os/src/detail/osMemory.c

@@ -512,8 +512,9 @@ void * taosTRealloc(void *ptr, size_t size) {
   return (void *)((char *)tptr + sizeof(size_t));
 }
 
-void taosTZfree(void *ptr) {
+void* taosTZfree(void* ptr) {
   if (ptr) {
-    free((void *)((char *)ptr - sizeof(size_t)));
+    free((void*)((char*)ptr - sizeof(size_t)));
   }
+  return NULL;
 }
\ No newline at end of file

src/os/src/detail/osSysinfo.c

@@ -18,6 +18,7 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
+#include "taoserror.h"
 
 #ifndef TAOS_OS_FUNC_SYSINFO
 
@@ -316,37 +317,17 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   return true;
 }
 
-bool taosGetDisk() {
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
   struct statvfs info;
-  const double   unit = 1024 * 1024 * 1024;
-
-  if (tscEmbedded) {
-    if (statvfs(tsDataDir, &info)) {
-      uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
-      return false;
-    } else {
-      tsTotalDataDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
-      tsAvailDataDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
-    }
-  }
-
-  if (statvfs(tsLogDir, &info)) {
-    uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno));
-    return false;
-  } else {
-    tsTotalLogDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
-    tsAvailLogDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
-  }
-
-  if (statvfs("/tmp", &info)) {
-    uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno));
-    return false;
+  if (statvfs(tsDataDir, &info)) {
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
   } else {
-    tsTotalTmpDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
-    tsAvailTmpDirectorySpace = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
+    diskSize->tsize = info.f_blocks * info.f_frsize;
+    diskSize->avail = info.f_bavail * info.f_frsize;
+    return 0;
   }
-
-  return true;
 }
 
 static bool taosGetCardInfo(int64_t *bytes) {
@@ -510,7 +491,7 @@ void taosGetSystemInfo() {
   float tmp1, tmp2;
   taosGetSysMemory(&tmp1);
   taosGetProcMemory(&tmp2);
-  taosGetDisk();
+  // taosGetDisk();
   taosGetBandSpeed(&tmp1);
   taosGetCpuUsage(&tmp1, &tmp2);
   taosGetProcIO(&tmp1, &tmp2);
@@ -537,7 +518,6 @@ void taosPrintOsInfo() {
   uInfo(" os release:             %s", buf.release);
   uInfo(" os version:             %s", buf.version);
   uInfo(" os machine:             %s", buf.machine);
-  uInfo("==================================");
 }
 
 void taosKillSystem() {

src/os/src/windows/wSysinfo.c

@@ -21,6 +21,7 @@
 #include "ttimer.h"
 #include "tulog.h"
 #include "tutil.h"
+#include "taoserror.h"
 #if (_WIN64)
 #include <iphlpapi.h>
 #include <mswsock.h>
@@ -126,37 +127,22 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   return true;
 }
 
-bool taosGetDisk() {
-  const double    unit = 1024 * 1024 * 1024;
-  BOOL            fResult;
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
   unsigned _int64 i64FreeBytesToCaller;
   unsigned _int64 i64TotalBytes;
   unsigned _int64 i64FreeBytes;
 
-  if (tscEmbedded) {
-    fResult = GetDiskFreeSpaceExA(tsDataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-                                  (PULARGE_INTEGER)&i64FreeBytes);
-    if (fResult) {
-      tsTotalDataDirGB = (float)(i64TotalBytes / unit);
-      tsAvailDataDirGB = (float)(i64FreeBytes / unit);
-    }
-  }
-
-  fResult = GetDiskFreeSpaceExA(tsLogDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-                                (PULARGE_INTEGER)&i64FreeBytes);
+  BOOL fResult = GetDiskFreeSpaceExA(dataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
+                                     (PULARGE_INTEGER)&i64FreeBytes);
   if (fResult) {
-    tsTotalLogDirGB = (float)(i64TotalBytes / unit);
-    tsAvailLogDirGB = (float)(i64FreeBytes / unit);
-  }
-
-  fResult = GetDiskFreeSpaceExA(tsTempDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-                                (PULARGE_INTEGER)&i64FreeBytes);
-  if (fResult) {
-    tsTotalTmpDirGB = (float)(i64TotalBytes / unit);
-    tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit);
+    diskSize->tsize = (int64_t)(i64TotalBytes);
+    diskSize->avail = (int64_t)(i64FreeBytes);
+    return 0;
+  } else {
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
   }
-
-  return true;
 }
 
 bool taosGetBandSpeed(float *bandSpeedKb) {
@@ -207,7 +193,7 @@ void taosGetSystemInfo() {
   tsTotalMemoryMB = taosGetTotalMemory();
 
   float tmp1, tmp2;
-  taosGetDisk();
+  // taosGetDisk();
   taosGetBandSpeed(&tmp1);
   taosGetCpuUsage(&tmp1, &tmp2);
   taosGetProcIO(&tmp1, &tmp2);

src/os/src/windows/wWordexp.c

@@ -21,13 +21,24 @@
 #include "tulog.h"
 #include "tutil.h"
 
-int wordexp(const char *words, wordexp_t *pwordexp, int flags) {
+int wordexp(char *words, wordexp_t *pwordexp, int flags) {
   pwordexp->we_offs = 0;
   pwordexp->we_wordc = 1;
-  pwordexp->we_wordv = (char **)(pwordexp->wordPos);
-  pwordexp->we_wordv[0] = (char *)words;
+  pwordexp->we_wordv[0] = pwordexp->wordPos;
+
+  memset(pwordexp->wordPos, 0, 1025);
+  if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) {
+    pwordexp->we_wordv[0] = words;
+    uError("failed to parse relative path:%s to abs path", words);
+    return -1;
+  }
+
+  uTrace("parse relative path:%s to abs path:%s", words, pwordexp->wordPos);
   return 0;
 }
 
 void wordfree(wordexp_t *pwordexp) {}
 
+char *realpath(char *path, char *resolved_path) {
+  return _fullpath(path, resolved_path, TSDB_FILENAME_LEN - 1);
+}
\ No newline at end of file

src/plugins/monitor/src/monMain.c

@@ -20,7 +20,6 @@
 #include "tlog.h"
 #include "ttimer.h"
 #include "tutil.h"
-#include "tsystem.h"
 #include "tscUtil.h"
 #include "tsclient.h"
 #include "dnode.h"

src/rpc/src/rpcUdp.c

@@ -15,7 +15,6 @@
 
 #include "os.h"
 #include "tsocket.h"
-#include "tsystem.h"
 #include "ttimer.h"
 #include "tutil.h"
 #include "taosdef.h"

src/sync/inc/syncInt.h

@@ -108,14 +108,17 @@ typedef struct SSyncNode {
   SSyncFwds *  pSyncFwds;  // saved forward info if quorum >1
   void *       pFwdTimer;
   void *       pRoleTimer;
-  FGetFileInfo      getFileInfo;
-  FGetWalInfo       getWalInfo;
-  FWriteToCache     writeToCache;
+  void *       pTsdb;
+  FGetWalInfo       getWalInfoFp;
+  FWriteToCache     writeToCacheFp;
   FConfirmForward   confirmForward;
-  FNotifyRole       notifyRole;
-  FNotifyFlowCtrl   notifyFlowCtrl;
-  FNotifyFileSynced notifyFileSynced;
-  FGetVersion       getVersion;
+  FNotifyRole       notifyRoleFp;
+  FNotifyFlowCtrl   notifyFlowCtrlFp;
+  FStartSyncFile    startSyncFileFp;
+  FStopSyncFile     stopSyncFileFp;
+  FGetVersion       getVersionFp;
+  FSendFile         sendFileFp;
+  FRecvFile         recvFileFp;
   pthread_mutex_t   mutex;
 } SSyncNode;
 

src/sync/inc/syncMsg.h

@@ -99,16 +99,12 @@ typedef struct {
 
 typedef struct {
   SSyncHead head;
-  char      name[TSDB_FILENAME_LEN];
-  uint32_t  magic;
-  uint32_t  index;
   uint64_t  fversion;
-  int64_t   size;
-} SFileInfo;
+} SFileVersion;
 
 typedef struct {
   SSyncHead head;
-  int8_t    sync;
+  int8_t    ack;
 } SFileAck;
 
 typedef struct {
@@ -136,7 +132,7 @@ void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId);
 void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId);
 
 void syncBuildFileAck(SFileAck *pMsg, int32_t vgId);
-void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId);
+void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId);
 
 #ifdef __cplusplus
 }

src/sync/src/syncMain.c

@@ -174,19 +174,22 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
   pthread_mutex_init(&pNode->mutex, NULL);
 
-  pNode->getFileInfo = pInfo->getFileInfo;
-  pNode->getWalInfo = pInfo->getWalInfo;
-  pNode->writeToCache = pInfo->writeToCache;
-  pNode->notifyRole = pInfo->notifyRole;
+  pNode->getWalInfoFp = pInfo->getWalInfoFp;
+  pNode->writeToCacheFp = pInfo->writeToCacheFp;
+  pNode->notifyRoleFp = pInfo->notifyRoleFp;
   pNode->confirmForward = pInfo->confirmForward;
-  pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl;
-  pNode->notifyFileSynced = pInfo->notifyFileSynced;
-  pNode->getVersion = pInfo->getVersion;
+  pNode->notifyFlowCtrlFp = pInfo->notifyFlowCtrlFp;
+  pNode->startSyncFileFp = pInfo->startSyncFileFp;
+  pNode->stopSyncFileFp = pInfo->stopSyncFileFp;
+  pNode->getVersionFp = pInfo->getVersionFp;
+  pNode->sendFileFp = pInfo->sendFileFp;
+  pNode->recvFileFp = pInfo->recvFileFp;
 
   pNode->selfIndex = -1;
   pNode->vgId = pInfo->vgId;
   pNode->replica = pCfg->replica;
   pNode->quorum = pCfg->quorum;
+  pNode->pTsdb = pInfo->pTsdb;
   if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
 
   pNode->refCount = 1;
@@ -248,8 +251,8 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   syncAddArbitrator(pNode);
   taosHashPut(tsVgIdHash, &pNode->vgId, sizeof(int32_t), &pNode, sizeof(SSyncNode *));
 
-  if (pNode->notifyRole) {
-    (*pNode->notifyRole)(pNode->vgId, nodeRole);
+  if (pNode->notifyRoleFp) {
+    (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
   }
 
   syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]);  // arb
@@ -357,7 +360,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
   if (pNewCfg->replica <= 1) {
     sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId);
     nodeRole = TAOS_SYNC_ROLE_MASTER;
-    (*pNode->notifyRole)(pNode->vgId, nodeRole);
+    (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
   }
 
   syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]);  // arb
@@ -417,7 +420,7 @@ void syncRecover(int64_t rid) {
   // if take this node to unsync state, the whole system may not work
 
   nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-  (*pNode->notifyRole)(pNode->vgId, nodeRole);
+  (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
   nodeVersion = 0;
 
   pthread_mutex_lock(&pNode->mutex);
@@ -625,8 +628,8 @@ static void syncResetFlowCtrl(SSyncNode *pNode) {
     pNode->peerInfo[index]->numOfRetrieves = 0;
   }
 
-  if (pNode->notifyFlowCtrl) {
-    (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
+  if (pNode->notifyFlowCtrlFp) {
+    (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
   }
 }
 
@@ -694,7 +697,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
       taosMsleep(SYNC_WAIT_AFTER_CHOOSE_MASTER);
 
       syncResetFlowCtrl(pNode);
-      (*pNode->notifyRole)(pNode->vgId, nodeRole);
+      (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
     } else {
       pPeer = pNode->peerInfo[index];
       sInfo("%s, it shall work as master", pPeer->id);
@@ -730,7 +733,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
         nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
         sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
        }
-      (*pNode->notifyRole)(pNode->vgId, nodeRole);
+      (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
     }
   } else {
     for (int32_t index = 0; index < pNode->replica; ++index) {
@@ -742,7 +745,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
         if (masterIndex == pNode->selfIndex) {
           sError("%s, peer is master, work as slave instead", pTemp->id);
           nodeRole = TAOS_SYNC_ROLE_SLAVE;
-          (*pNode->notifyRole)(pNode->vgId, nodeRole);
+          (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
         }
       }
     }
@@ -759,7 +762,7 @@ static int32_t syncValidateMaster(SSyncPeer *pPeer) {
   if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) {
     sDebug("%s, peer has higher sver:%" PRIu64 ", restart all peer connections", pPeer->id, pPeer->version);
     nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-    (*pNode->notifyRole)(pNode->vgId, nodeRole);
+    (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
     code = -1;
 
     for (int32_t index = 0; index < pNode->replica; ++index) {
@@ -796,7 +799,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new
       } else {
         sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion);
         nodeRole = TAOS_SYNC_ROLE_SLAVE;
-        (*pNode->notifyRole)(pNode->vgId, nodeRole);
+        (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
       }
     } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) {
       sDebug("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion);
@@ -989,7 +992,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
 
   if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
     // nodeVersion = pHead->version;
-    (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
+    (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
   } else {
     if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
       syncSaveIntoBuffer(pPeer, pHead);

src/sync/src/syncMsg.c

@@ -102,9 +102,9 @@ void syncBuildFileAck(SFileAck *pMsg, int32_t vgId) {
   syncBuildHead(&pMsg->head);
 }
 
-void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId) {
+void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId) {
   pMsg->head.type = TAOS_SMSG_SYNC_FILE;
   pMsg->head.vgId = vgId;
-  pMsg->head.len = sizeof(SFileInfo) - sizeof(SSyncHead);
+  pMsg->head.len = sizeof(SFileVersion) - sizeof(SSyncHead);
   syncBuildHead(&pMsg->head);
 }
\ No newline at end of file

src/sync/src/syncRestore.c

@@ -25,139 +25,44 @@
 #include "tsync.h"
 #include "syncInt.h"
 
-static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex) {
-  char       name[TSDB_FILENAME_LEN * 2] = {0};
-  char       fname[TSDB_FILENAME_LEN * 3] = {0};
-  uint32_t   magic;
-  uint64_t   fversion;
-  int64_t    size;
-  uint32_t   index = sindex;
+static int32_t syncRecvFileVersion(SSyncPeer *pPeer, uint64_t *fversion) {
   SSyncNode *pNode = pPeer->pSyncNode;
 
-  if (sindex < 0 || eindex < sindex) return;
-
-  sDebug("%s, extra files will be removed between sindex:%d and eindex:%d", pPeer->id, sindex, eindex);
-
-  while (1) {
-    name[0] = 0;
-    magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion);
-    if (magic == 0) break;
-
-    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name);
-    (void)remove(fname);
-    sInfo("%s, %s is removed for its extra", pPeer->id, fname);
+  SFileVersion fileVersion;
+  memset(&fileVersion, 0, sizeof(SFileVersion));
+  int32_t ret = taosReadMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion));
+  if (ret != sizeof(SFileVersion)) {
+    sError("%s, failed to read fver since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
 
-    index++;
-    if (index > eindex) break;
+  SFileAck fileVersionAck;
+  memset(&fileVersionAck, 0, sizeof(SFileAck));
+  syncBuildFileAck(&fileVersionAck, pNode->vgId);
+  ret = taosWriteMsg(pPeer->syncFd, &fileVersionAck, sizeof(SFileAck));
+  if (ret != sizeof(SFileAck)) {
+    sError("%s, failed to write fver ack since %s", pPeer->id, strerror(errno));
+    return -1;
   }
+
+  *fversion = htobe64(fileVersion.fversion);
+  return 0;
 }
 
 static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  SFileInfo  minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */
-  SFileInfo  sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */
-  SFileAck   fileAck; memset(&fileAck, 0, sizeof(SFileAck));
-  int32_t    code = -1;
-  char       name[TSDB_FILENAME_LEN * 2] = {0};
-  uint32_t   pindex = 0;  // index in last restore
-  bool       fileChanged = false;
-
-  *fversion = 0;
-  sinfo.index = -1;
-  while (1) {
-    // read file info
-    minfo.index = -1;
-    int32_t ret = taosReadMsg(pPeer->syncFd, &minfo, sizeof(SFileInfo));
-    if (ret != sizeof(SFileInfo) || minfo.index == -1) {
-      sError("%s, failed to read fileinfo while restore file since %s", pPeer->id, strerror(errno));
-      break;
-    }
-
-    assert(ret == sizeof(SFileInfo));
-    ret = syncCheckHead((SSyncHead *)(&minfo));
-    if (ret != 0) {
-      sError("%s, failed to check fileinfo while restore file since %s", pPeer->id, strerror(ret));
-      break;
-    }
-
-    // if no more file from master, break;
-    if (minfo.name[0] == 0 || minfo.magic == 0) {
-      sDebug("%s, no more files to restore", pPeer->id);
 
-      // remove extra files after the current index
-      if (sinfo.index != -1) syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX);
-      code = 0;
-      break;
-    }
-
-    sDebug("%s, file:%s info is received from master, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id,
-           minfo.name, minfo.index, minfo.size, minfo.fversion, minfo.magic);
-
-    // remove extra files on slave between the current and last index
-    syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1);
-    pindex = minfo.index;
-
-    // check the file info
-    sinfo = minfo;
-    sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion);
-    sDebug("%s, local file:%s info, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id, sinfo.name,
-           sinfo.index, sinfo.size, sinfo.fversion, sinfo.magic);
-
-    // if file not there or magic is not the same, file shall be synced
-    memset(&fileAck, 0, sizeof(SFileAck));
-    syncBuildFileAck(&fileAck, pNode->vgId);
-    fileAck.sync = (sinfo.magic != minfo.magic || sinfo.size != minfo.size || sinfo.name[0] == 0) ? 1 : 0;
-
-    // send file ack
-    ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
-    if (ret != sizeof(SFileAck)) {
-      sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno));
-      break;
-    }
-
-    // if sync is not required, continue
-    if (fileAck.sync == 0) {
-      sDebug("%s, %s is the same", pPeer->id, minfo.name);
-      continue;
-    } else {
-      sDebug("%s, %s will be received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
-    }
-
-    // if sync is required, open file, receive from master, and write to file
-    // get the full path to file
-    minfo.name[sizeof(minfo.name) - 1] = 0;
-    snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name);
-
-    int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO);
-    if (dfd < 0) {
-      sError("%s, failed to open file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno));
-      break;
-    }
-
-    ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size);
-    fsync(dfd);
-    close(dfd);
-    if (ret < 0) {
-      sError("%s, failed to copy file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno));
-      break;
-    }
-
-    fileChanged = true;
-    sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
-  }
-
-  if (code == 0 && fileChanged) {
-    // data file is changed, code shall be set to 1
-    *fversion = minfo.fversion;
-    code = 1;
-    sDebug("%s, file changed after restore file, fver:%" PRIu64, pPeer->id, *fversion);
+  if (pNode->recvFileFp && (*pNode->recvFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) {
+    sError("%s, failed to restore file", pPeer->id);
+    return -1;
   }
 
-  if (code < 0) {
-    sError("%s, failed to restore %s since %s", pPeer->id, name, strerror(errno));
+  if (syncRecvFileVersion(pPeer, fversion) < 0) {
+    return -1;
   }
 
-  return code;
+  sInfo("%s, all files are restored, fver:%" PRIu64, pPeer->id, *fversion);
+  return 0;
 }
 
 static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) {
@@ -195,7 +100,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) {
     }
     lastVer = pHead->version;
 
-    ret = (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL);
+    ret = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL);
     if (ret != 0) {
       sError("%s, failed to restore record since %s, hver:%" PRIu64, pPeer->id, tstrerror(ret), pHead->version);
       break;
@@ -215,7 +120,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) {
   SSyncNode *pNode = pPeer->pSyncNode;
   SWalHead * pHead = (SWalHead *)offset;
 
-  (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
+  (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
   offset += pHead->len + sizeof(SWalHead);
 
   return offset;
@@ -315,20 +220,16 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
   sDebug("%s, send sync rsp to peer, tranId:%u", pPeer->id, rsp.tranId);
 
   sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
+  (*pNode->startSyncFileFp)(pNode->vgId);
+
   int32_t code = syncRestoreFile(pPeer, &fversion);
   if (code < 0) {
-    sError("%s, failed to restore file", pPeer->id);
+    (*pNode->stopSyncFileFp)(pNode->vgId, fversion);
+    sError("%s, failed to restore files", pPeer->id);
     return -1;
   }
 
-  // if code > 0, data file is changed, notify app, and pass the version
-  if (code > 0 && pNode->notifyFileSynced) {
-    if ((*pNode->notifyFileSynced)(pNode->vgId, fversion) < 0) {
-      sError("%s, app not in ready state", pPeer->id);
-      return -1;
-    }
-  }
-
+  (*pNode->stopSyncFileFp)(pNode->vgId, fversion);
   nodeVersion = fversion;
 
   sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion);
@@ -368,7 +269,7 @@ void *syncRestoreData(void *param) {
   atomic_add_fetch_32(&tsSyncNum, 1);
   sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
 
-  (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING);
+  (*pNode->notifyRoleFp)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING);
 
   if (syncOpenRecvBuffer(pNode) < 0) {
     sError("%s, failed to allocate recv buffer, restart connection", pPeer->id);
@@ -385,7 +286,7 @@ void *syncRestoreData(void *param) {
     }
   }
 
-  (*pNode->notifyRole)(pNode->vgId, nodeRole);
+  (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
 
   nodeSStatus = TAOS_SYNC_STATUS_INIT;
   sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);

src/sync/src/syncRetrieve.c

@@ -27,7 +27,7 @@
 
 static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
     return -1;
@@ -39,7 +39,7 @@ static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
 
 static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
     return true;
@@ -55,7 +55,7 @@ static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) {
 
 static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while get fver for retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
     return -1;
@@ -67,7 +67,7 @@ static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
 
 static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
     pPeer->fileChanged = 1;
@@ -84,104 +84,54 @@ static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) {
   return false;
 }
 
-static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
+static int32_t syncSendFileVersion(SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  SFileInfo  fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo));
-  SFileAck   fileAck; memset(&fileAck, 0, sizeof(SFileAck));
-  int32_t    code = -1;
-  char       name[TSDB_FILENAME_LEN * 2] = {0};
 
-  if (syncGetFileVersion(pNode, pPeer) < 0) {
-    pPeer->fileChanged = 1;
+  SFileVersion fileVersion;
+  memset(&fileVersion, 0, sizeof(SFileVersion));
+  syncBuildFileVersion(&fileVersion, pNode->vgId);
+
+  uint64_t fver = pPeer->lastFileVer;
+  fileVersion.fversion = htobe64(fver);
+  int32_t ret = taosWriteMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion));
+  if (ret != sizeof(SFileVersion)) {
+    sError("%s, failed to write fver:%" PRIu64 " since %s", pPeer->id, fver, strerror(errno));
     return -1;
   }
 
-  while (1) {
-    // retrieve file info
-    fileInfo.name[0] = 0;
-    fileInfo.size = 0;
-    fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
-                                           &fileInfo.size, &fileInfo.fversion);
-    syncBuildFileInfo(&fileInfo, pNode->vgId);
-    sDebug("%s, file:%s info is sent, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id, fileInfo.name,
-           fileInfo.index, fileInfo.size, fileInfo.fversion, fileInfo.magic);
-
-    // send the file info
-    int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(SFileInfo));
-    if (ret != sizeof(SFileInfo)) {
-      code = -1;
-      sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
-
-    // if no file anymore, break
-    if (fileInfo.magic == 0 || fileInfo.name[0] == 0) {
-      code = 0;
-      sDebug("%s, no more files to sync", pPeer->id);
-      break;
-    }
-
-    // wait for the ack from peer
-    ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
-    if (ret != sizeof(SFileAck)) {
-      code = -1;
-      sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
-
-    ret = syncCheckHead((SSyncHead*)(&fileAck));
-    if (ret != 0) {
-      code = -1;
-      sError("%s, failed to check file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(ret));
-      break;
-    }
-
-    // set the peer sync version
-    pPeer->sversion = fileInfo.fversion;
-
-    // if sync is not required, continue
-    if (fileAck.sync == 0) {
-      fileInfo.index++;
-      sDebug("%s, %s is the same, fver:%" PRIu64, pPeer->id, fileInfo.name, fileInfo.fversion);
-      continue;
-    } else {
-      sDebug("%s, %s will be sent, fver:%" PRIu64, pPeer->id, fileInfo.name, fileInfo.fversion);
-    }
+  SFileAck fileAck;
+  memset(&fileAck, 0, sizeof(SFileAck));
+  ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
+  if (ret != sizeof(SFileAck)) {
+    sError("%s, failed to read fver ack since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
 
-    // get the full path to file
-    snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
+  // set the peer sync version
+  pPeer->sversion = fver;
 
-    // send the file to peer
-    int32_t sfd = open(name, O_RDONLY | O_BINARY);
-    if (sfd < 0) {
-      code = -1;
-      sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
+  return 0;
+}
 
-    ret = (int32_t)taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
-    close(sfd);
-    if (ret < 0) {
-      code = -1;
-      sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
+static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
+  SSyncNode *pNode = pPeer->pSyncNode;
 
-    sDebug("%s, file:%s is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
-    fileInfo.index++;
+  if (syncGetFileVersion(pNode, pPeer) < 0) {
+    pPeer->fileChanged = 1;
+    return -1;
+  }
 
-    // check if processed files are modified
-    if (syncAreFilesModified(pNode, pPeer)) {
-      code = -1;
-      break;
-    }
+  if (pNode->sendFileFp && (*pNode->sendFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) {
+    sError("%s, failed to retrieve file", pPeer->id);
+    return -1;
   }
 
-  if (code != TSDB_CODE_SUCCESS) {
-    sError("%s, failed to retrieve file, code:0x%x", pPeer->id, code);
+  if (syncSendFileVersion(pPeer) < 0) {
+    return -1;
   }
 
-  return code;
+  sInfo("%s, all files are retrieved", pPeer->id);
+  return 0;
 }
 
 // if only a partial record is read out, upper layer will reload the file to get a complete record
@@ -345,7 +295,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
   while (1) {
     // retrieve wal info
     wname[0] = 0;
-    code = (*pNode->getWalInfo)(pNode->vgId, wname, &index);
+    code = (*pNode->getWalInfoFp)(pNode->vgId, wname, &index);
     if (code < 0) {
       sError("%s, failed to get wal info since:%s, code:0x%x", pPeer->id, strerror(errno), code);
       break;
@@ -477,7 +427,7 @@ void *syncRetrieveData(void *param) {
   sInfo("%s, start to retrieve data, sstatus:%s, numOfRetrieves:%d", pPeer->id, syncStatus[pPeer->sstatus],
         pPeer->numOfRetrieves);
 
-  if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves);
+  if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, pPeer->numOfRetrieves);
 
   pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
   if (pPeer->syncFd < 0) {
@@ -497,10 +447,10 @@ void *syncRetrieveData(void *param) {
     pPeer->numOfRetrieves++;
   } else {
     pPeer->numOfRetrieves = 0;
-    // if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
+    // if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
   }
 
-  if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
+  if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
 
   pPeer->fileChanged = 0;
   taosCloseSocket(pPeer->syncFd);

src/sync/test/syncServer.c

@@ -296,11 +296,10 @@ void initSync() {
   pCfg->replica = 1;
   pCfg->quorum = 1;
   syncInfo.vgId = 1;
-  syncInfo.getFileInfo = getFileInfo;
-  syncInfo.getWalInfo = getWalInfo;
-  syncInfo.writeToCache = writeToCache;
+  syncInfo.getWalInfoFp = getWalInfo;
+  syncInfo.writeToCacheFp = writeToCache;
   syncInfo.confirmForward = confirmForward;
-  syncInfo.notifyRole = notifyRole;
+  syncInfo.notifyRoleFp = notifyRole;
 
   pCfg->nodeInfo[0].nodeId = 1;
   pCfg->nodeInfo[0].nodePort = 7010;

src/tfs/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(inc)
+AUX_SOURCE_DIRECTORY(src SRC)
+ADD_LIBRARY(tfs ${SRC})
+TARGET_LINK_LIBRARIES(tfs tutil)
+
+IF (TD_LINUX)
+  # Someone has no gtest directory, so comment it
+  # ADD_SUBDIRECTORY(tests)
+ENDIF ()

src/tfs/inc/tfsint.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_TFSINT_H
+#define TD_TFSINT_H
+
+#include "tlog.h"
+#include "tglobal.h"
+#include "tfs.h"
+#include "tcoding.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int fsDebugFlag;
+
+// For debug purpose
+#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", 255, __VA_ARGS__); }}
+#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }}
+#define fWarn(...)  { if (fsDebugFlag & DEBUG_WARN)  { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }}
+#define fInfo(...)  { if (fsDebugFlag & DEBUG_INFO)  { taosPrintLog("TFS ", 255, __VA_ARGS__); }}
+#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }}
+#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }}
+
+// Global Definitions
+#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024
+
+// tdisk.c ======================================================
+typedef struct {
+  int64_t size;
+  int64_t free;
+} SDiskMeta;
+
+typedef struct SDisk {
+  int       level;
+  int       id;
+  char      dir[TSDB_FILENAME_LEN];
+  SDiskMeta dmeta;
+} SDisk;
+
+#define DISK_LEVEL(pd) ((pd)->level)
+#define DISK_ID(pd) ((pd)->id)
+#define DISK_DIR(pd) ((pd)->dir)
+#define DISK_META(pd) ((pd)->dmeta)
+#define DISK_SIZE(pd) ((pd)->dmeta.size)
+#define DISK_FREE_SIZE(pd) ((pd)->dmeta.free)
+
+SDisk *tfsNewDisk(int level, int id, const char *dir);
+SDisk *tfsFreeDisk(SDisk *pDisk);
+int    tfsUpdateDiskInfo(SDisk *pDisk);
+
+// ttier.c ======================================================
+typedef struct {
+  int64_t size;
+  int64_t free;
+  int16_t nAvailDisks;  // # of Available disks
+} STierMeta;
+typedef struct STier {
+  pthread_spinlock_t lock;
+  int                level;
+  int16_t            ndisk;   // # of disks mounted to this tier
+  int16_t            nextid;  // next disk id to allocate
+  STierMeta          tmeta;
+  SDisk *            disks[TSDB_MAX_DISKS_PER_TIER];
+} STier;
+
+#define TIER_LEVEL(pt) ((pt)->level)
+#define TIER_NDISKS(pt) ((pt)->ndisk)
+#define TIER_SIZE(pt) ((pt)->tmeta.size)
+#define TIER_FREE_SIZE(pt) ((pt)->tmeta.free)
+#define TIER_AVAIL_DISKS(pt) ((pt)->tmeta.nAvailDisks)
+#define DISK_AT_TIER(pt, id) ((pt)->disks[id])
+
+int    tfsInitTier(STier *pTier, int level);
+void   tfsDestroyTier(STier *pTier);
+SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg);
+void   tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta);
+int    tfsAllocDiskOnTier(STier *pTier);
+void   tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta);
+void   tfsPosNextId(STier *pTier);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file

src/tfs/src/tdisk.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "os.h"
+
+#include "taoserror.h"
+#include "tfsint.h"
+
+// PROTECTED ====================================
+SDisk *tfsNewDisk(int level, int id, const char *dir) {
+  SDisk *pDisk = (SDisk *)calloc(1, sizeof(*pDisk));
+  if (pDisk == NULL) {
+    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pDisk->level = level;
+  pDisk->id = id;
+  strncpy(pDisk->dir, dir, TSDB_FILENAME_LEN);
+
+  return pDisk;
+}
+
+SDisk *tfsFreeDisk(SDisk *pDisk) {
+  if (pDisk) {
+    free(pDisk);
+  }
+  return NULL;
+}
+
+int tfsUpdateDiskInfo(SDisk *pDisk) {
+  ASSERT(pDisk != NULL);
+
+  SysDiskSize diskSize = {0};
+
+  int code = taosGetDiskSize(pDisk->dir, &diskSize);
+  if (code != 0) {
+    fError("failed to update disk information at level %d id %d dir %s since %s", pDisk->level, pDisk->id, pDisk->dir,
+           strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+  }
+
+  pDisk->dmeta.size = diskSize.tsize;
+  pDisk->dmeta.free = diskSize.tsize - diskSize.avail;
+
+  return code;
+}
\ No newline at end of file

src/tfs/src/tfs.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+
+#include "hash.h"
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tfs.h"
+#include "tfsint.h"
+
+#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32)
+
+typedef struct {
+  pthread_spinlock_t lock;
+  SFSMeta            meta;
+  int                nlevel;
+  STier              tiers[TSDB_MAX_TIERS];
+  SHashObj *         map;  // name to did map
+} SFS;
+
+typedef struct {
+  SDisk *pDisk;
+} SDiskIter;
+
+#define TFS_META() (pfs->meta)
+#define TFS_NLEVEL() (pfs->nlevel)
+#define TFS_TIERS() (pfs->tiers)
+#define TFS_TIER_AT(level) (TFS_TIERS() + (level))
+#define TFS_DISK_AT(level, id) DISK_AT_TIER(TFS_TIER_AT(level), id)
+#define TFS_PRIMARY_DISK() TFS_DISK_AT(TFS_PRIMARY_LEVEL, TFS_PRIMARY_ID)
+#define TFS_IS_VALID_LEVEL(level) (((level) >= 0) && ((level) < TFS_NLEVEL()))
+#define TFS_IS_VALID_ID(level, id) (((id) >= 0) && ((id) < TIER_NDISKS(TFS_TIER_AT(level))))
+#define TFS_IS_VALID_DISK(level, id) (TFS_IS_VALID_LEVEL(level) && TFS_IS_VALID_ID(level, id))
+
+#define tfsLock() pthread_spin_lock(&(pfs->lock))
+#define tfsUnLock() pthread_spin_unlock(&(pfs->lock))
+
+static SFS  tfs = {0};
+static SFS *pfs = &tfs;
+
+// STATIC DECLARATION
+static int    tfsMount(SDiskCfg *pCfg);
+static int    tfsCheck();
+static int    tfsCheckAndFormatCfg(SDiskCfg *pCfg);
+static int    tfsFormatDir(char *idir, char *odir);
+static SDisk *tfsGetDiskByID(SDiskID did);
+static SDisk *tfsGetDiskByName(const char *dir);
+static int    tfsOpendirImpl(TDIR *tdir);
+static void   tfsInitDiskIter(SDiskIter *pIter);
+static SDisk *tfsNextDisk(SDiskIter *pIter);
+
+// FS APIs ====================================
+int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
+  ASSERT(ndisk > 0);
+
+  for (int level = 0; level < TSDB_MAX_TIERS; level++) {
+    if (tfsInitTier(TFS_TIER_AT(level), level) < 0) {
+      while (true) {
+        level--;
+        if (level < 0) break;
+
+        tfsDestroyTier(TFS_TIER_AT(level));
+      }
+
+      return -1;
+    }
+  }
+
+  pthread_spin_init(&(pfs->lock), 0);
+
+  pfs->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
+                          taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+  if (pfs->map == NULL) {
+    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    tfsDestroy();
+    return -1;
+  }
+
+  for (int idisk = 0; idisk < ndisk; idisk++) {
+    if (tfsMount(pDiskCfg + idisk) < 0) {
+      tfsDestroy();
+      return -1;
+    }
+  }
+
+  if (tfsCheck() < 0) {
+    tfsDestroy();
+    return -1;
+  }
+
+  tfsUpdateInfo(NULL);
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    tfsPosNextId(TFS_TIER_AT(level));
+  }
+
+  return 0;
+}
+
+void tfsDestroy() {
+  taosHashCleanup(pfs->map);
+  pfs->map = NULL;
+
+  pthread_spin_destroy(&(pfs->lock));
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    tfsDestroyTier(TFS_TIER_AT(level));
+  }
+}
+
+void tfsUpdateInfo(SFSMeta *pFSMeta) {
+  SFSMeta   fsMeta;
+  STierMeta tierMeta;
+
+  if (pFSMeta == NULL) {
+    pFSMeta = &fsMeta;
+  }
+
+  memset(pFSMeta, 0, sizeof(*pFSMeta));
+
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    tfsUpdateTierInfo(pTier, &tierMeta);
+    pFSMeta->tsize += tierMeta.size;
+    pFSMeta->avail += tierMeta.free;
+  }
+
+  tfsLock();
+  pfs->meta = *pFSMeta;
+  tfsUnLock();
+}
+
+void tfsGetMeta(SFSMeta *pMeta) {
+  ASSERT(pMeta);
+
+  tfsLock();
+  *pMeta = pfs->meta;
+  tfsUnLock();
+}
+
+/* Allocate an existing available tier level
+ */
+void tfsAllocDisk(int expLevel, int *level, int *id) {
+  ASSERT(expLevel >= 0);
+
+  *level = expLevel;
+  *id = TFS_UNDECIDED_ID;
+
+  if (*level >= TFS_NLEVEL()) {
+    *level = TFS_NLEVEL() - 1;
+  }
+
+  while (*level >= 0) {
+    *id = tfsAllocDiskOnTier(TFS_TIER_AT(*level));
+    if (*id == TFS_UNDECIDED_ID) {
+      (*level)--;
+      continue;
+    }
+
+    return;
+  }
+
+  *level = TFS_UNDECIDED_LEVEL;
+  *id = TFS_UNDECIDED_ID;
+}
+
+const char *TFS_PRIMARY_PATH() { return DISK_DIR(TFS_PRIMARY_DISK()); }
+const char *TFS_DISK_PATH(int level, int id) { return DISK_DIR(TFS_DISK_AT(level, id)); }
+
+// TFILE APIs ====================================
+void tfsInitFile(TFILE *pf, int level, int id, const char *bname) {
+  ASSERT(TFS_IS_VALID_DISK(level, id));
+
+  SDisk *pDisk = TFS_DISK_AT(level, id);
+
+  pf->level = level;
+  pf->id = id;
+  strncpy(pf->rname, bname, TSDB_FILENAME_LEN);
+
+  char tmpName[TMPNAME_LEN] = {0};
+  snprintf(tmpName, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), bname);
+  tstrncpy(pf->aname, tmpName, TSDB_FILENAME_LEN);
+}
+
+bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2) {
+  ASSERT(pf1 != NULL || pf2 != NULL);
+  if (pf1 == NULL || pf2 == NULL) return false;
+  if (pf1->level != pf2->level) return false;
+  if (pf1->id != pf2->id) return false;
+  if (strncmp(pf1->rname, pf2->rname, TSDB_FILENAME_LEN) != 0) return false;
+  return true;
+}
+
+int tfsEncodeFile(void **buf, TFILE *pf) {
+  int tlen = 0;
+
+  tlen += taosEncodeVariantI32(buf, pf->level);
+  tlen += taosEncodeVariantI32(buf, pf->id);
+  tlen += taosEncodeString(buf, pf->rname);
+
+  return tlen;
+}
+
+void *tfsDecodeFile(void *buf, TFILE *pf) {
+  int32_t level, id;
+  char *  rname;
+
+  buf = taosDecodeVariantI32(buf, &(level));
+  buf = taosDecodeVariantI32(buf, &(id));
+  buf = taosDecodeString(buf, &rname);
+
+  tfsInitFile(pf, level, id, rname);
+
+  tfree(rname);
+  return buf;
+}
+
+void tfsbasename(const TFILE *pf, char *dest) {
+  char tname[TSDB_FILENAME_LEN] = "\0";
+
+  strncpy(tname, pf->aname, TSDB_FILENAME_LEN);
+  strncpy(dest, basename(tname), TSDB_FILENAME_LEN);
+}
+
+void tfsdirname(const TFILE *pf, char *dest) {
+  char tname[TSDB_FILENAME_LEN] = "\0";
+
+  strncpy(tname, pf->aname, TSDB_FILENAME_LEN);
+  strncpy(dest, dirname(tname), TSDB_FILENAME_LEN);
+}
+
+// DIR APIs ====================================
+int tfsMkdirAt(const char *rname, int level, int id) {
+  SDisk *pDisk = TFS_DISK_AT(level, id);
+  char   aname[TMPNAME_LEN];
+
+  snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname);
+  if (taosMkDir(aname, 0755) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+int tfsMkdirRecurAt(const char *rname, int level, int id) {
+  if (tfsMkdirAt(rname, level, id) < 0) {
+    if (errno == ENOENT) {
+      // Try to create upper
+      char *s = strdup(rname);
+
+      if (tfsMkdirRecurAt(dirname(s), level, id) < 0) {
+        tfree(s);
+        return -1;
+      }
+      tfree(s);
+
+      if (tfsMkdirAt(rname, level, id) < 0) {
+        return -1;
+      }
+    } else {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tfsMkdir(const char *rname) {
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+      if (tfsMkdirAt(rname, level, id) < 0) {
+        return -1;
+      }
+    }
+  }
+
+  return 0;
+}
+
+int tfsRmdir(const char *rname) {
+  char aname[TMPNAME_LEN] = "\0";
+
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+      SDisk *pDisk = DISK_AT_TIER(pTier, id);
+
+      snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname);
+
+      taosRemoveDir(aname);
+    }
+  }
+
+  return 0;
+}
+
+int tfsRename(char *orname, char *nrname) {
+  char oaname[TMPNAME_LEN] = "\0";
+  char naname[TMPNAME_LEN] = "\0";
+
+  for (int level = 0; level < pfs->nlevel; level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+      SDisk *pDisk = DISK_AT_TIER(pTier, id);
+
+      snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname);
+      snprintf(naname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), nrname);
+
+      taosRename(oaname, naname);
+    }
+  }
+
+  return 0;
+}
+
+struct TDIR {
+  SDiskIter iter;
+  int       level;
+  int       id;
+  char      dirname[TSDB_FILENAME_LEN];
+  TFILE     tfile;
+  DIR *     dir;
+};
+
+TDIR *tfsOpendir(const char *rname) {
+  TDIR *tdir = (TDIR *)calloc(1, sizeof(*tdir));
+  if (tdir == NULL) {
+    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  tfsInitDiskIter(&(tdir->iter));
+  strncpy(tdir->dirname, rname, TSDB_FILENAME_LEN);
+
+  if (tfsOpendirImpl(tdir) < 0) {
+    free(tdir);
+    return NULL;
+  }
+
+  return tdir;
+}
+
+const TFILE *tfsReaddir(TDIR *tdir) {
+  if (tdir == NULL || tdir->dir == NULL) return NULL;
+  char bname[TMPNAME_LEN * 2] = "\0";
+
+  while (true) {
+    struct dirent *dp = NULL;
+    dp = readdir(tdir->dir);
+    if (dp != NULL) {
+      // Skip . and ..
+      if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue;
+
+      snprintf(bname, TMPNAME_LEN * 2, "%s/%s", tdir->dirname, dp->d_name);
+      tfsInitFile(&(tdir->tfile), tdir->level, tdir->id, bname);
+      return &(tdir->tfile);
+    }
+
+    if (tfsOpendirImpl(tdir) < 0) {
+      return NULL;
+    }
+
+    if (tdir->dir == NULL) {
+      terrno = TSDB_CODE_SUCCESS;
+      return NULL;
+    }
+  }
+}
+
+void tfsClosedir(TDIR *tdir) {
+  if (tdir) {
+    if (tdir->dir != NULL) {
+      closedir(tdir->dir);
+      tdir->dir = NULL;
+    }
+    free(tdir);
+  }
+}
+
+// private
+static int tfsMount(SDiskCfg *pCfg) {
+  SDiskID did;
+  SDisk * pDisk = NULL;
+
+  if (tfsCheckAndFormatCfg(pCfg) < 0) return -1;
+
+  did.level = pCfg->level;
+  pDisk = tfsMountDiskToTier(TFS_TIER_AT(did.level), pCfg);
+  if (pDisk == NULL) {
+    fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
+    return -1;
+  }
+  did.id = DISK_ID(pDisk);
+
+  taosHashPut(pfs->map, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did));
+  if (pfs->nlevel < pCfg->level + 1) pfs->nlevel = pCfg->level + 1;
+
+  return 0;
+}
+
+static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
+  char        dirName[TSDB_FILENAME_LEN] = "\0";
+  struct stat pstat;
+
+  if (pCfg->level < 0 || pCfg->level >= TSDB_MAX_TIERS) {
+    fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (pCfg->primary) {
+    if (pCfg->level != 0) {
+      fError("failed to mount %s to FS since disk is primary but level %d not 0", pCfg->dir, pCfg->level);
+      terrno = TSDB_CODE_FS_INVLD_CFG;
+      return -1;
+    }
+
+    if (TFS_PRIMARY_DISK() != NULL) {
+      fError("failed to mount %s to FS since duplicate primary mount", pCfg->dir);
+      terrno = TSDB_CODE_FS_DUP_PRIMARY;
+      return -1;
+    }
+  }
+
+  if (tfsFormatDir(pCfg->dir, dirName) < 0) {
+    fError("failed to mount %s to FS since invalid dir format", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (tfsGetDiskByName(dirName) != NULL) {
+    fError("failed to mount %s to FS since duplicate mount", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (access(dirName, W_OK | R_OK | F_OK) != 0) {
+    fError("failed to mount %s to FS since no R/W access rights", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (stat(dirName, &pstat) < 0) {
+    fError("failed to mount %s to FS since %s", pCfg->dir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (!S_ISDIR(pstat.st_mode)) {
+    fError("failed to mount %s to FS since not a directory", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  strncpy(pCfg->dir, dirName, TSDB_FILENAME_LEN);
+
+  return 0;
+}
+
+static int tfsFormatDir(char *idir, char *odir) {
+  wordexp_t wep = {0};
+
+  int code = wordexp(idir, &wep, 0);
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+
+  if (realpath(wep.we_wordv[0], odir) == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    wordfree(&wep);
+    return -1;
+  }
+
+  wordfree(&wep);
+  return 0;
+
+}
+
+static int tfsCheck() {
+  if (TFS_PRIMARY_DISK() == NULL) {
+    fError("no primary disk is set");
+    terrno = TSDB_CODE_FS_NO_PRIMARY_DISK;
+    return -1;
+  }
+
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    if (TIER_NDISKS(TFS_TIER_AT(level)) == 0) {
+      fError("no disk at level %d", level);
+      terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static SDisk *tfsGetDiskByID(SDiskID did) { return TFS_DISK_AT(did.level, did.id); }
+static SDisk *tfsGetDiskByName(const char *dir) {
+  SDiskID did;
+  SDisk * pDisk = NULL;
+  void *  pr = NULL;
+
+  pr = taosHashGet(pfs->map, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN));
+  if (pr == NULL) return NULL;
+
+  did = *(SDiskID *)pr;
+  pDisk = tfsGetDiskByID(did);
+  ASSERT(pDisk != NULL);
+
+  return pDisk;
+}
+
+static int tfsOpendirImpl(TDIR *tdir) {
+  SDisk *pDisk = NULL;
+  char   adir[TMPNAME_LEN * 2] = "\0";
+
+  if (tdir->dir != NULL) {
+    closedir(tdir->dir);
+    tdir->dir = NULL;
+  }
+
+  while (true) {
+    pDisk = tfsNextDisk(&(tdir->iter));
+    if (pDisk == NULL) return 0;
+
+    tdir->level = DISK_LEVEL(pDisk);
+    tdir->id = DISK_ID(pDisk);
+
+    snprintf(adir, TMPNAME_LEN * 2, "%s/%s", DISK_DIR(pDisk), tdir->dirname);
+    tdir->dir = opendir(adir);
+    if (tdir->dir != NULL) break;
+  }
+
+  return 0;
+}
+
+static void tfsInitDiskIter(SDiskIter *pIter) { pIter->pDisk = TFS_DISK_AT(0, 0); }
+
+static SDisk *tfsNextDisk(SDiskIter *pIter) {
+  SDisk *pDisk = pIter->pDisk;
+
+  if (pDisk == NULL) return NULL;
+
+  int level = DISK_LEVEL(pDisk);
+  int id = DISK_ID(pDisk);
+
+  id++;
+  if (id < TIER_NDISKS(TFS_TIER_AT(level))) {
+    pIter->pDisk = TFS_DISK_AT(level, id);
+    ASSERT(pIter->pDisk != NULL);
+  } else {
+    level++;
+    id = 0;
+    if (level < TFS_NLEVEL()) {
+      pIter->pDisk = TFS_DISK_AT(level, id);
+      ASSERT(pIter->pDisk != NULL);
+    } else {
+      pIter->pDisk = NULL;
+    }
+  }
+
+  return pDisk;
+}
+
+// OTHER FUNCTIONS ===================================
+void taosGetDisk() {
+  const double unit = 1024 * 1024 * 1024;
+  SysDiskSize  diskSize;
+  SFSMeta      fsMeta;
+
+  if (tscEmbedded) {
+    tfsUpdateInfo(&fsMeta);
+    tsTotalDataDirGB = (float)(fsMeta.tsize / unit);
+    tsAvailDataDirGB = (float)(fsMeta.avail / unit);
+  }
+
+  if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
+    tsTotalLogDirGB = (float)(diskSize.tsize / unit);
+    tsAvailLogDirGB = (float)(diskSize.avail / unit);
+  }
+
+  if (taosGetDiskSize("/tmp", &diskSize) == 0) {
+    tsTotalTmpDirGB = (float)(diskSize.tsize / unit);
+    tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
+  }
+}

src/tfs/src/ttier.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "os.h"
+
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tfsint.h"
+
+#define tfsLockTier(pTier) pthread_spin_lock(&((pTier)->lock))
+#define tfsUnLockTier(pTier) pthread_spin_unlock(&((pTier)->lock))
+
+// PROTECTED ==========================================
+int tfsInitTier(STier *pTier, int level) {
+  memset((void *)pTier, 0, sizeof(*pTier));
+
+  int code = pthread_spin_init(&(pTier->lock), 0);
+  if (code) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+
+  pTier->level = level;
+
+  return 0;
+}
+
+void tfsDestroyTier(STier *pTier) {
+  for (int id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) {
+    DISK_AT_TIER(pTier, id) = tfsFreeDisk(DISK_AT_TIER(pTier, id));
+  }
+
+  pTier->ndisk = 0;
+
+  pthread_spin_destroy(&(pTier->lock));
+}
+
+SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
+  ASSERT(pTier->level == pCfg->level);
+
+  int    id = 0;
+  SDisk *pDisk;
+
+  if (TIER_NDISKS(pTier) >= TSDB_MAX_DISKS_PER_TIER) {
+    terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
+    return NULL;
+  }
+
+  if (pTier->level == 0) {
+    if (DISK_AT_TIER(pTier, 0) != NULL) {
+      id = pTier->ndisk;
+    } else {
+      if (pCfg->primary) {
+        id = 0;
+      } else {
+        id = pTier->ndisk + 1;
+      }
+      if (id >= TSDB_MAX_DISKS_PER_TIER) {
+        terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
+        return NULL;
+      }
+    }
+  } else {
+    id = pTier->ndisk;
+  }
+
+  pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir);
+  if (pDisk == NULL) return NULL;
+  DISK_AT_TIER(pTier, id) = pDisk;
+  pTier->ndisk++;
+
+  fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id);
+
+  return DISK_AT_TIER(pTier, id);
+}
+
+void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta) {
+  STierMeta tmeta;
+
+  if (pTierMeta == NULL) {
+    pTierMeta = &tmeta;
+  }
+  memset(pTierMeta, 0, sizeof(*pTierMeta));
+
+  tfsLockTier(pTier);
+
+  for (int id = 0; id < pTier->ndisk; id++) {
+    if (tfsUpdateDiskInfo(DISK_AT_TIER(pTier, id)) < 0) {
+      continue;
+    }
+    pTierMeta->size += DISK_SIZE(DISK_AT_TIER(pTier, id));
+    pTierMeta->free += DISK_FREE_SIZE(DISK_AT_TIER(pTier, id));
+    pTierMeta->nAvailDisks++;
+  }
+
+  pTier->tmeta = *pTierMeta;
+
+  tfsUnLockTier(pTier);
+}
+
+// Round-Robin to allocate disk on a tier
+int tfsAllocDiskOnTier(STier *pTier) {
+  ASSERT(pTier->ndisk > 0);
+  int    id = TFS_UNDECIDED_ID;
+  SDisk *pDisk;
+
+  tfsLockTier(pTier);
+
+  if (TIER_AVAIL_DISKS(pTier) <= 0) {
+    tfsUnLockTier(pTier);
+    return id;
+  }
+
+  id = pTier->nextid;
+  while (true) {
+    pDisk = DISK_AT_TIER(pTier, id);
+    ASSERT(pDisk != NULL);
+
+    if (DISK_FREE_SIZE(pDisk) < TFS_MIN_DISK_FREE_SIZE) {
+      id = (id + 1) % pTier->ndisk;
+      if (id == pTier->nextid) {
+        tfsUnLockTier(pTier);
+        return TFS_UNDECIDED_ID;
+      } else {
+        continue;
+      }
+    } else {
+      pTier->nextid = (id + 1) % pTier->ndisk;
+      break;
+    }
+  }
+
+  tfsUnLockTier(pTier);
+  return id;
+}
+
+void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta) {
+  ASSERT(pTierMeta != NULL);
+
+  tfsLockTier(pTier);
+  *pTierMeta = pTier->tmeta;
+  tfsUnLockTier(pTier);
+}
+
+void tfsPosNextId(STier *pTier) {
+  ASSERT(pTier->ndisk > 0);
+  int nextid = 0;
+
+  for (int id = 1; id < pTier->ndisk; id++) {
+    SDisk *pLDisk = DISK_AT_TIER(pTier, nextid);
+    SDisk *pDisk = DISK_AT_TIER(pTier, id);
+    if (DISK_FREE_SIZE(pDisk) > TFS_MIN_DISK_FREE_SIZE && DISK_FREE_SIZE(pDisk) > DISK_FREE_SIZE(pLDisk)) {
+      nextid = id;
+    }
+  }
+
+  pTier->nextid = nextid;
+}
\ No newline at end of file

src/tsdb/CMakeLists.txt

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 ADD_LIBRARY(tsdb ${SRC})
-TARGET_LINK_LIBRARIES(tsdb common tutil)
+TARGET_LINK_LIBRARIES(tsdb tfs common tutil)
 
 IF (TD_LINUX)
   # Someone has no gtest directory, so comment it

src/util/inc/tkvstore.h → src/tsdb/inc/tsdbBuffer.h

@@ -12,56 +12,32 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef _TD_KVSTORE_H_
-#define _TD_KVSTORE_H_
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-
-#define KVSTORE_FILE_VERSION ((uint32_t)0)
-
-typedef int (*iterFunc)(void *, void *cont, int contLen);
-typedef void (*afterFunc)(void *);
+#ifndef _TD_TSDB_BUFFER_H_
+#define _TD_TSDB_BUFFER_H_
 
 typedef struct {
-  int64_t  size;  // including 512 bytes of header size
-  int64_t  tombSize;
-  int64_t  nRecords;
-  int64_t  nDels;
-  uint32_t magic;
-} SStoreInfo;
+  int64_t blockId;
+  int     offset;
+  int     remain;
+  char    data[];
+} STsdbBufBlock;
 
 typedef struct {
-  char *     fname;
-  int        fd;
-  char *     fsnap;
-  int        sfd;
-  char *     fnew;
-  int        nfd;
-  SHashObj * map;
-  iterFunc   iFunc;
-  afterFunc  aFunc;
-  void *     appH;
-  SStoreInfo info;
-} SKVStore;
-
-#define KVSTORE_MAGIC(s) (s)->info.magic
-
-int       tdCreateKVStore(char *fname);
-int       tdDestroyKVStore(char *fname);
-SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH);
+  pthread_cond_t poolNotEmpty;
+  int            bufBlockSize;
+  int            tBufBlocks;
+  int            nBufBlocks;
+  int64_t        index;
+  SList*         bufBlockList;
+} STsdbBufPool;
+
+#define TSDB_BUFFER_RESERVE 1024  // Reseve 1K as commit threshold
+
+STsdbBufPool* tsdbNewBufPool();
+void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
+int           tsdbOpenBufPool(STsdbRepo* pRepo);
+void          tsdbCloseBufPool(STsdbRepo* pRepo);
+SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
+
+#endif /* _TD_TSDB_BUFFER_H_ */
\ No newline at end of file
-void      tdCloseKVStore(SKVStore *pStore);
-int       tdKVStoreStartCommit(SKVStore *pStore);
-int       tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen);
-int       tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid);
-int       tdKVStoreEndCommit(SKVStore *pStore);
-void      tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
\ No newline at end of file

src/dnode/inc/dnodeMgmt.h → src/tsdb/inc/tsdbCommit.h

@@ -13,35 +13,37 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_DNODE_MGMT_H
-#define TDENGINE_DNODE_MGMT_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "trpc.h"
-
-int32_t dnodeInitMgmt();
-void    dnodeCleanupMgmt();
-int32_t dnodeInitMgmtTimer();
-void    dnodeCleanupMgmtTimer();
-void    dnodeDispatchToMgmtQueue(SRpcMsg *rpcMsg);
-
-void*   dnodeGetVnode(int32_t vgId);
-int32_t dnodeGetVnodeStatus(void *pVnode);
-void*   dnodeGetVnodeRworker(void *pVnode);
-void*   dnodeGetVnodeWworker(void *pVnode);
-void*   dnodeGetVnodeWal(void *pVnode);
-void*   dnodeGetVnodeTsdb(void *pVnode);
-void    dnodeReleaseVnode(void *pVnode);
-
-void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
-void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
-void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
-
-#ifdef __cplusplus
+#ifndef _TD_TSDB_COMMIT_H_
+#define _TD_TSDB_COMMIT_H_
+
+typedef struct {
+  int   minFid;
+  int   midFid;
+  int   maxFid;
+  TSKEY minKey;
+} SRtn;
+
+typedef struct {
+  uint64_t uid;
+  int64_t  offset;
+  int64_t  size;
+} SKVRecord;
+
+void  tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn);
+int   tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord);
+void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord);
+void *tsdbCommitData(STsdbRepo *pRepo);
+
+static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
+  if (fid >= pRtn->maxFid) {
+    return 0;
+  } else if (fid >= pRtn->midFid) {
+    return 1;
+  } else if (fid >= pRtn->minFid) {
+    return 2;
+  } else {
+    return -1;
+  }
 }
-#endif
 
-#endif
+#endif /* _TD_TSDB_COMMIT_H_ */
\ No newline at end of file

src/common/inc/tsystem.h → src/tsdb/inc/tsdbCommitQueue.h

@@ -13,26 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TSYSTEM_H
-#define TDENGINE_TSYSTEM_H
+#ifndef _TD_TSDB_COMMIT_QUEUE_H_
+#define _TD_TSDB_COMMIT_QUEUE_H_
 
-#ifdef __cplusplus
-extern "C" {
-#endif
+int tsdbScheduleCommit(STsdbRepo *pRepo);
 
-bool taosGetSysMemory(float *memoryUsedMB);
-bool taosGetProcMemory(float *memoryUsedMB);
+#endif /* _TD_TSDB_COMMIT_QUEUE_H_ */
\ No newline at end of file
-bool taosGetDisk();
-bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
-bool taosGetBandSpeed(float *bandSpeedKb);
-bool taosGetProcIO(float *readKB, float *writeKB);
-void taosGetSystemInfo();
-void taosPrintOsInfo();
-void taosKillSystem();
-void taosSetCoreDump();
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/tsdb/inc/tsdbFS.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_FS_H_
+#define _TD_TSDB_FS_H_
+
+#define TSDB_FS_VERSION 0
+
+// ================== CURRENT file header info
+typedef struct {
+  uint32_t version;  // Current file system version (relating to code)
+  uint32_t len;      // Encode content length (including checksum)
+} SFSHeader;
+
+// ================== TSDB File System Meta
+typedef struct {
+  uint32_t version;       // Commit version from 0 to increase
+  int64_t  totalPoints;   // total points
+  int64_t  totalStorage;  // Uncompressed total storage
+} STsdbFSMeta;
+
+// ==================
+typedef struct {
+  STsdbFSMeta meta;  // FS meta
+  SMFile*     pmf;   // meta file pointer
+  SMFile      mf;    // meta file
+  SArray*     df;    // data file array
+} SFSStatus;
+
+typedef struct {
+  pthread_rwlock_t lock;
+
+  SFSStatus* cstatus;    // current status
+  SHashObj*  metaCache;  // meta cache
+  bool       intxn;
+  SFSStatus* nstatus;  // new status
+} STsdbFS;
+
+#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
+#define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
+#define FS_IN_TXN(pfs) (pfs)->intxn
+#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version)
+#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version)
+
+typedef struct {
+  int        direction;
+  uint64_t   version;  // current FS version
+  STsdbFS*   pfs;
+  int        index;  // used to position next fset when version the same
+  int        fid;    // used to seek when version is changed
+  SDFileSet* pSet;
+} SFSIter;
+
+#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
+#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
+
+STsdbFS *tsdbNewFS(STsdbCfg *pCfg);
+void *   tsdbFreeFS(STsdbFS *pfs);
+int      tsdbOpenFS(STsdbRepo *pRepo);
+void     tsdbCloseFS(STsdbRepo *pRepo);
+void     tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd);
+int      tsdbEndFSTxn(STsdbRepo *pRepo);
+int      tsdbEndFSTxnWithError(STsdbFS *pfs);
+void     tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
+void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
+int      tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
+
+void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
+void       tsdbFSIterSeek(SFSIter *pIter, int fid);
+SDFileSet *tsdbFSIterNext(SFSIter *pIter);
+int        tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta);
+
+static FORCE_INLINE int tsdbRLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_rdlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbWLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_wrlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_unlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+#endif /* _TD_TSDB_FS_H_ */
\ No newline at end of file

src/tsdb/inc/tsdbFile.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TS_TSDB_FILE_H_
+#define _TS_TSDB_FILE_H_
+
+#define TSDB_FILE_HEAD_SIZE 512
+#define TSDB_FILE_DELIMITER 0xF00AFA0F
+#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
+#define TSDB_IVLD_FID INT_MIN
+
+#define TSDB_FILE_INFO(tf) (&((tf)->info))
+#define TSDB_FILE_F(tf) (&((tf)->f))
+#define TSDB_FILE_FD(tf) ((tf)->fd)
+#define TSDB_FILE_FULL_NAME(tf) TFILE_NAME(TSDB_FILE_F(tf))
+#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
+#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
+#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
+#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf))
+#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf))
+#define TSDB_FILE_FSYNC(tf) fsync(TSDB_FILE_FD(tf))
+
+typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T;
+
+// =============== SMFile
+typedef struct {
+  int64_t  size;
+  int64_t  tombSize;
+  int64_t  nRecords;
+  int64_t  nDels;
+  uint32_t magic;
+} SMFInfo;
+
+typedef struct {
+  SMFInfo info;
+  TFILE   f;
+  int     fd;
+} SMFile;
+
+void  tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
+void  tsdbInitMFileEx(SMFile* pMFile, SMFile* pOMFile);
+int   tsdbEncodeSMFile(void** buf, SMFile* pMFile);
+void* tsdbDecodeSMFile(void* buf, SMFile* pMFile);
+int   tsdbEncodeSMFileEx(void** buf, SMFile* pMFile);
+void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile);
+int   tsdbApplyMFileChange(SMFile* from, SMFile* to);
+int   tsdbCreateMFile(SMFile* pMFile, bool updateHeader);
+int   tsdbUpdateMFileHeader(SMFile* pMFile);
+int   tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo);
+int   tsdbScanAndTryFixMFile(STsdbRepo* pRepo);
+int   tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo);
+void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo);
+
+static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) {
+  ASSERT(TSDB_FILE_CLOSED(pMFile));
+
+  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags);
+  if (pMFile->fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) {
+  if (TSDB_FILE_OPENED(pMFile)) {
+    close(pMFile->fd);
+    TSDB_FILE_SET_CLOSED(pMFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t loffset = taosLSeek(TSDB_FILE_FD(pMFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t nwrite = taosWrite(pMFile->fd, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) {
+  pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pMFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pMFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); }
+
+static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t nread = taosRead(pMFile->fd, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+// =============== SDFile
+typedef struct {
+  uint32_t magic;
+  uint32_t len;
+  uint32_t totalBlocks;
+  uint32_t totalSubBlocks;
+  uint32_t offset;
+  uint64_t size;
+  uint64_t tombSize;
+} SDFInfo;
+
+typedef struct {
+  SDFInfo info;
+  TFILE   f;
+  int     fd;
+} SDFile;
+
+void  tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype);
+void  tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
+int   tsdbEncodeSDFile(void** buf, SDFile* pDFile);
+void* tsdbDecodeSDFile(void* buf, SDFile* pDFile);
+int   tsdbCreateDFile(SDFile* pDFile, bool updateHeader);
+int   tsdbUpdateDFileHeader(SDFile* pDFile);
+int   tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
+int   tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
+
+static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
+  ASSERT(!TSDB_FILE_OPENED(pDFile));
+
+  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags);
+  if (pDFile->fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
+  if (TSDB_FILE_OPENED(pDFile)) {
+    close(pDFile->fd);
+    TSDB_FILE_SET_CLOSED(pDFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t loffset = taosLSeek(TSDB_FILE_FD(pDFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nwrite = taosWrite(pDFile->fd, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) {
+  pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pDFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pDFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_FILE_F(pDFile)); }
+
+static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nread = taosRead(pDFile->fd, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
+  if (tfscopy(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
+  return 0;
+}
+
+// =============== SDFileSet
+typedef struct {
+  int    fid;
+  int    state;
+  SDFile files[TSDB_FILE_MAX];
+} SDFileSet;
+
+#define TSDB_FSET_FID(s) ((s)->fid)
+#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
+#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_SET_CLOSED(s)                                                \
+  do {                                                                         \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
+      TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype));                       \
+    }                                                                          \
+  } while (0);
+#define TSDB_FSET_FSYNC(s)                                                     \
+  do {                                                                         \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
+      TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype));                            \
+    }                                                                          \
+  } while (0);
+
+void  tsdbInitDFileSet(SDFileSet* pSet, SDiskID did, int vid, int fid, uint32_t ver);
+void  tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
+int   tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSet(void* buf, SDFileSet* pSet);
+int   tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
+int   tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
+int   tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader);
+int   tsdbUpdateDFileSetHeader(SDFileSet* pSet);
+int   tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet* pSet);
+
+static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
+      tsdbCloseDFileSet(pSet);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
+      tsdbRemoveDFileSet(pDest);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) {
+  *minKey = fid * days * tsMsPerDay[precision];
+  *maxKey = *minKey + days * tsMsPerDay[precision] - 1;
+}
+
+#endif /* _TS_TSDB_FILE_H_ */
\ No newline at end of file

src/tsdb/inc/tsdbLog.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_LOG_H_
+#define _TD_TSDB_LOG_H_
+
+extern int32_t tsdbDebugFlag;
+
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+
+#endif /* _TD_TSDB_LOG_H_ */
\ No newline at end of file

src/tsdb/inc/tsdbMain.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef _TD_TSDB_MAIN_H_
-#define _TD_TSDB_MAIN_H_
-
-#include "os.h"
-#include "hash.h"
-#include "tcoding.h"
-#include "tglobal.h"
-#include "tkvstore.h"
-#include "tlist.h"
-#include "tlog.h"
-#include "tlockfree.h"
-#include "tsdb.h"
-#include "tskiplist.h"
-#include "tutil.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern int32_t tsdbDebugFlag;
-
-#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
-#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
-#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
-#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
-#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
-#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
-
-#define TSDB_MAX_TABLE_SCHEMAS 16
-#define TSDB_FILE_HEAD_SIZE    512
-#define TSDB_FILE_DELIMITER    0xF00AFA0F
-#define TSDB_FILE_INIT_MAGIC   0xFFFFFFFF
-
-#define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax)))
-
-// NOTE: Any file format change must increase this version number by 1
-//       Also, implement the convert function
-#define TSDB_FILE_VERSION ((uint32_t)0)
-
-// Definitions
-// ------------------ tsdbMeta.c
-typedef struct STable {
-  STableId       tableId;
-  ETableType     type;
-  tstr*          name;  // NOTE: there a flexible string here
-  uint64_t       suid;
-  struct STable* pSuper;  // super table pointer
-  uint8_t        numOfSchemas;
-  STSchema*      schema[TSDB_MAX_TABLE_SCHEMAS];
-  STSchema*      tagSchema;
-  SKVRow         tagVal;
-  SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
-  void*          eventHandler;   // TODO
-  void*          streamHandler;  // TODO
-  TSKEY          lastKey;
-  SDataRow       lastRow;
-  char*          sql;
-  void*          cqhandle;
-  SRWLatch       latch;  // TODO: implementa latch functions
-  T_REF_DECLARE()
-} STable;
-
-typedef struct {
-  pthread_rwlock_t rwLock;
-
-  int32_t   nTables;
-  int32_t   maxTables;
-  STable**  tables;
-  SList*    superList;
-  SHashObj* uidMap;
-  SKVStore* pStore;
-  int       maxRowBytes;
-  int       maxCols;
-} STsdbMeta;
-
-// ------------------ tsdbBuffer.c
-typedef struct {
-  int64_t blockId;
-  int     offset;
-  int     remain;
-  char    data[];
-} STsdbBufBlock;
-
-typedef struct {
-  pthread_cond_t poolNotEmpty;
-  int            bufBlockSize;
-  int            tBufBlocks;
-  int            nBufBlocks;
-  int64_t        index;
-  SList*         bufBlockList;
-} STsdbBufPool;
-
-// ------------------ tsdbMemTable.c
-typedef struct {
-  STable *           pTable;
-  SSkipListIterator *pIter;
-} SCommitIter;
-
-typedef struct {
-  uint64_t   uid;
-  TSKEY      keyFirst;
-  TSKEY      keyLast;
-  int64_t    numOfRows;
-  SSkipList* pData;
-} STableData;
-
-typedef struct {
-  T_REF_DECLARE()
-  SRWLatch     latch;
-  TSKEY        keyFirst;
-  TSKEY        keyLast;
-  int64_t      numOfRows;
-  int32_t      maxTables;
-  STableData** tData;
-  SList*       actList;
-  SList*       extraBuffList;
-  SList*       bufBlockList;
-} SMemTable;
-
-enum { TSDB_UPDATE_META, TSDB_DROP_META };
-
-#ifdef WINDOWS
-#pragma pack(push ,1) 
-typedef struct {
-#else
-typedef struct __attribute__((packed)){
-#endif
-  char     act;
-  uint64_t uid;
-} SActObj;
-#ifdef WINDOWS
-#pragma pack(pop) 
-#endif
-
-typedef struct {
-  int  len;
-  char cont[];
-} SActCont;
-
-// ------------------ tsdbFile.c
-extern const char* tsdbFileSuffix[];
-typedef enum {
-  TSDB_FILE_TYPE_HEAD = 0,
-  TSDB_FILE_TYPE_DATA,
-  TSDB_FILE_TYPE_LAST,
-  TSDB_FILE_TYPE_STAT,
-  TSDB_FILE_TYPE_NHEAD,
-  TSDB_FILE_TYPE_NDATA,
-  TSDB_FILE_TYPE_NLAST,
-  TSDB_FILE_TYPE_NSTAT
-} TSDB_FILE_TYPE;
-
-#ifndef TDINTERNAL
-#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_LAST+1)
-#else
-#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_STAT+1)
-#endif
-
-typedef struct {
-  uint32_t magic;
-  uint32_t len;
-  uint32_t totalBlocks;
-  uint32_t totalSubBlocks;
-  uint32_t offset;
-  uint64_t size;      // total size of the file
-  uint64_t tombSize;  // unused file size
-} STsdbFileInfo;
-
-typedef struct {
-  char  fname[TSDB_FILENAME_LEN];
-  int   fd;
-
-  STsdbFileInfo info;
-} SFile;
-
-typedef struct {
-  int   fileId;
-  int   state; // 0 for health, 1 for problem
-  SFile files[TSDB_FILE_TYPE_MAX];
-} SFileGroup;
-
-typedef struct {
-  pthread_rwlock_t fhlock;
-
-  int         maxFGroups;
-  int         nFGroups;
-  SFileGroup* pFGroup;
-} STsdbFileH;
-
-typedef struct {
-  int         direction;
-  STsdbFileH* pFileH;
-  int         fileId;
-  int         index;
-} SFileGroupIter;
-
-// ------------------ tsdbMain.c
-typedef struct {
-  int32_t  totalLen;
-  int32_t  len;
-  SDataRow row;
-} SSubmitBlkIter;
-
-typedef struct {
-  int32_t totalLen;
-  int32_t len;
-  void *  pMsg;
-} SSubmitMsgIter;
-
-typedef struct {
-  int8_t state;
-
-  char*           rootDir;
-  STsdbCfg        config;
-  STsdbAppH       appH;
-  STsdbStat       stat;
-  STsdbMeta*      tsdbMeta;
-  STsdbBufPool*   pPool;
-  SMemTable*      mem;
-  SMemTable*      imem;
-  STsdbFileH*     tsdbFileH;
-  tsem_t          readyToCommit;
-  pthread_mutex_t mutex;
-  bool            repoLocked;
-  int32_t         code; // Commit code
-} STsdbRepo;
-
-// ------------------ tsdbRWHelper.c
-typedef struct {
-  int32_t  tid;
-  uint32_t len;
-  uint32_t offset;
-  uint32_t hasLast : 2;
-  uint32_t numOfBlocks : 30;
-  uint64_t uid;
-  TSKEY    maxKey;
-} SCompIdx;
-
-typedef struct {
-  int64_t last : 1;
-  int64_t offset : 63;
-  int32_t algorithm : 8;
-  int32_t numOfRows : 24;
-  int32_t len;
-  int32_t keyLen;     // key column length, keyOffset = offset+sizeof(SCompData)+sizeof(SCompCol)*numOfCols
-  int16_t numOfSubBlocks;
-  int16_t numOfCols; // not including timestamp column
-  TSKEY   keyFirst;
-  TSKEY   keyLast;
-} SCompBlock;
-
-typedef struct {
-  int32_t    delimiter;  // For recovery usage
-  int32_t    tid;
-  uint64_t   uid;
-  SCompBlock blocks[];
-} SCompInfo;
-
-typedef struct {
-  int16_t colId;
-  int32_t len;
-  int32_t type : 8;
-  int32_t offset : 24;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
-  int16_t maxIndex;
-  int16_t minIndex;
-  int16_t numOfNull;
-  char    padding[2];
-} SCompCol;
-
-typedef struct {
-  int32_t  delimiter;  // For recovery usage
-  int32_t  numOfCols;  // For recovery usage
-  uint64_t uid;        // For recovery usage
-  SCompCol cols[];
-} SCompData;
-
-typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;
-
-typedef struct {
-  TSKEY      minKey;
-  TSKEY      maxKey;
-  SFileGroup fGroup;
-  SFile      nHeadF;
-  SFile      nLastF;
-} SHelperFile;
-
-typedef struct {
-  uint64_t uid;
-  int32_t  tid;
-} SHelperTable;
-
-typedef struct {
-  SCompIdx* pIdxArray;
-  int       numOfIdx;
-  int       curIdx;
-} SIdxH;
-
-typedef struct {
-  tsdb_rw_helper_t type;
-
-  STsdbRepo* pRepo;
-  int8_t     state;
-  // For file set usage
-  SHelperFile files;
-  SIdxH       idxH;
-  SCompIdx    curCompIdx;
-  void*       pWIdx;
-  // For table set usage
-  SHelperTable tableInfo;
-  SCompInfo*   pCompInfo;
-  bool         hasOldLastBlock;
-  // For block set usage
-  SCompData* pCompData;
-  SDataCols* pDataCols[2];
-  void*      pBuffer;     // Buffer to hold the whole data block
-  void*      compBuffer;  // Buffer for temperary compress/decompress purpose
-} SRWHelper;
-
-typedef struct {
-  int   rowsInserted;
-  int   rowsUpdated;
-  int   rowsDeleteSucceed;
-  int   rowsDeleteFailed;
-  int   nOperations;
-  TSKEY keyFirst;
-  TSKEY keyLast;
-} SMergeInfo;
-// ------------------ tsdbScan.c
-typedef struct {
-  SFileGroup fGroup;
-  int        numOfIdx;
-  SCompIdx*  pCompIdx;
-  SCompInfo* pCompInfo;
-  void*      pBuf;
-  FILE*      tLogStream;
-} STsdbScanHandle;
-
-// Operations
-// ------------------ tsdbMeta.c
-#define TSDB_INIT_NTABLES 1024
-#define TABLE_TYPE(t) (t)->type
-#define TABLE_NAME(t) (t)->name
-#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
-#define TABLE_UID(t) (t)->tableId.uid
-#define TABLE_TID(t) (t)->tableId.tid
-#define TABLE_SUID(t) (t)->suid
-#define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore)
-#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch))
-#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch))
-#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch))
-#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch))
-
-STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
-void       tsdbFreeMeta(STsdbMeta* pMeta);
-int        tsdbOpenMeta(STsdbRepo* pRepo);
-int        tsdbCloseMeta(STsdbRepo* pRepo);
-STable*    tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
-STSchema*  tsdbGetTableSchemaByVersion(STable* pTable, int16_t version);
-int        tsdbWLockRepoMeta(STsdbRepo* pRepo);
-int        tsdbRLockRepoMeta(STsdbRepo* pRepo);
-int        tsdbUnlockRepoMeta(STsdbRepo* pRepo);
-void       tsdbRefTable(STable* pTable);
-void       tsdbUnRefTable(STable* pTable);
-void       tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct);
-
-static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
-  if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) {
-    return -1;
-  } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) {
-    return 1;
-  } else {
-    return 0;
-  }
-}
-
-static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) {
-  STable*   pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable;
-  STSchema* pSchema = NULL;
-  STSchema* pTSchema = NULL;
-
-  if (lock) TSDB_RLOCK_TABLE(pDTable);
-  if (version < 0) {  // get the latest version of schema
-    pTSchema = pDTable->schema[pDTable->numOfSchemas - 1];
-  } else {  // get the schema with version
-    void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*),
-                            tsdbCompareSchemaVersion, TD_EQ);
-    if (ptr == NULL) {
-      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
-      goto _exit;
-    }
-    pTSchema = *(STSchema**)ptr;
-  }
-
-  ASSERT(pTSchema != NULL);
-
-  if (copy) {
-    if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-  } else {
-    pSchema = pTSchema;
-  }
-
-_exit:
-  if (lock) TSDB_RUNLOCK_TABLE(pDTable);
-  return pSchema;
-}
-
-static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) {
-  return tsdbGetTableSchemaImpl(pTable, false, false, -1);
-}
-
-static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) {
-  if (pTable->type == TSDB_CHILD_TABLE) {  // check child table first
-    STable *pSuper = pTable->pSuper;
-    if (pSuper == NULL) return NULL;
-    return pSuper->tagSchema;
-  } else if (pTable->type == TSDB_SUPER_TABLE) {
-    return pTable->tagSchema;
-  } else {
-    return NULL;
-  }
-}
-
-static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) {
-  ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow));
-  return pTable->lastKey;
-}
-
-// ------------------ tsdbBuffer.c
-#define TSDB_BUFFER_RESERVE 1024  // Reseve 1K as commit threshold
-
-STsdbBufPool* tsdbNewBufPool();
-void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
-int           tsdbOpenBufPool(STsdbRepo* pRepo);
-void          tsdbCloseBufPool(STsdbRepo* pRepo);
-SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
-
-// ------------------ tsdbMemTable.c
-int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
-int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
-int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
-void  tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem);
-void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
-int   tsdbAsyncCommit(STsdbRepo* pRepo);
-int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
-                            TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
-void* tsdbCommitData(STsdbRepo* pRepo);
-
-static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
-  if (pIter == NULL) return NULL;
-
-  SSkipListNode* node = tSkipListIterGet(pIter);
-  if (node == NULL) return NULL;
-
-  return (SDataRow)SL_GET_NODE_DATA(node);
-}
-
-static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
-  SDataRow row = tsdbNextIterRow(pIter);
-  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
-
-  return dataRowKey(row);
-}
-
-static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
-  SDataRow row = tsdbNextIterRow(pIter);
-  if (row == NULL) return TKEY_NULL;
-
-  return dataRowTKey(row);
-}
-
-static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
-  ASSERT(pRepo != NULL);
-  if (pRepo->mem == NULL) return NULL;
-
-  SListNode* pNode = listTail(pRepo->mem->bufBlockList);
-  if (pNode == NULL) return NULL;
-
-  STsdbBufBlock* pBufBlock = NULL;
-  tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock));
-
-  return pBufBlock;
-}
-
-// ------------------ tsdbFile.c
-#define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
-#define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
-#define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId
-#define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId
-#define TSDB_IS_FILE_OPENED(f) ((f)->fd > 0)
-#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC
-#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC
-
-STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg);
-void        tsdbFreeFileH(STsdbFileH* pFileH);
-int         tsdbOpenFileH(STsdbRepo* pRepo);
-void        tsdbCloseFileH(STsdbRepo* pRepo);
-SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid);
-void        tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction);
-void        tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid);
-SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter);
-int         tsdbOpenFile(SFile* pFile, int oflag);
-void        tsdbCloseFile(SFile* pFile);
-int         tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type);
-SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags);
-void        tsdbFitRetention(STsdbRepo* pRepo);
-int         tsdbUpdateFileHeader(SFile* pFile);
-int         tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo);
-void*       tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo);
-void        tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
-int         tsdbLoadFileHeader(SFile* pFile, uint32_t* version);
-void        tsdbGetFileInfoImpl(char* fname, uint32_t* magic, int64_t* size);
-void        tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey);
-
-// ------------------ tsdbRWHelper.c
-#define TSDB_HELPER_CLEAR_STATE 0x0        // Clear state
-#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1  // File is set
-#define TSDB_HELPER_IDX_LOAD 0x2           // SCompIdx part is loaded
-#define TSDB_HELPER_TABLE_SET 0x4          // Table is set
-#define TSDB_HELPER_INFO_LOAD 0x8          // SCompInfo part is loaded
-#define TSDB_HELPER_FILE_DATA_LOAD 0x10    // SCompData part is loaded
-#define helperSetState(h, s) (((h)->state) |= (s))
-#define helperClearState(h, s) ((h)->state &= (~(s)))
-#define helperHasState(h, s) ((((h)->state) & (s)) == (s))
-#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx)
-#define TSDB_MAX_SUBBLOCKS 8
-#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
-#define helperType(h) (h)->type
-#define helperRepo(h) (h)->pRepo
-#define helperState(h) (h)->state
-#define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0)
-#define helperFileId(h) ((h)->files.fGroup.fileId)
-#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD]))
-#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA]))
-#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST]))
-#define helperNewHeadF(h) (&((h)->files.nHeadF))
-#define helperNewLastF(h) (&((h)->files.nLastF))
-
-int  tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
-int  tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
-void tsdbDestroyHelper(SRWHelper* pHelper);
-void tsdbResetHelper(SRWHelper* pHelper);
-int  tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup);
-int  tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError, SFileGroup* pGroup);
-int  tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo);
-int  tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey);
-int  tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper);
-int  tsdbWriteCompInfo(SRWHelper* pHelper);
-int  tsdbWriteCompIdx(SRWHelper* pHelper);
-int  tsdbLoadCompIdxImpl(SFile* pFile, uint32_t offset, uint32_t len, void* buffer);
-int  tsdbDecodeSCompIdxImpl(void* buffer, uint32_t len, SCompIdx** ppCompIdx, int* numOfIdx);
-int  tsdbLoadCompIdx(SRWHelper* pHelper, void* target);
-int  tsdbLoadCompInfoImpl(SFile* pFile, SCompIdx* pIdx, SCompInfo** ppCompInfo);
-int  tsdbLoadCompInfo(SRWHelper* pHelper, void* target);
-int  tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target);
-void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols);
-int  tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo, int16_t* colIds,
-                           int numOfColIds);
-int  tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo);
-
-static FORCE_INLINE int compTSKEY(const void* key1, const void* key2) {
-  if (*(TSKEY*)key1 > *(TSKEY*)key2) {
-    return 1;
-  } else if (*(TSKEY*)key1 == *(TSKEY*)key2) {
-    return 0;
-  } else {
-    return -1;
-  }
-}
-
-// ------------------ tsdbMain.c
-#define REPO_ID(r) (r)->config.tsdbId
-#define IS_REPO_LOCKED(r) (r)->repoLocked
-#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
-
-char*       tsdbGetMetaFileName(char* rootDir);
-void        tsdbGetDataFileName(char* rootDir, int vid, int fid, int type, char* fname);
-int         tsdbLockRepo(STsdbRepo* pRepo);
-int         tsdbUnlockRepo(STsdbRepo* pRepo);
-char*       tsdbGetDataDirName(char* rootDir);
-int         tsdbGetNextMaxTables(int tid);
-STsdbMeta*  tsdbGetMeta(TSDB_REPO_T* pRepo);
-STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo);
-int         tsdbCheckCommit(STsdbRepo* pRepo);
-
-// ------------------ tsdbScan.c
-int              tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid);
-STsdbScanHandle* tsdbNewScanHandle();
-void             tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream);
-int              tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid);
-int              tsdbScanSCompIdx(STsdbScanHandle* pScanHandle);
-int              tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx);
-int              tsdbCloseScanFile(STsdbScanHandle* pScanHandle);
-void             tsdbFreeScanHandle(STsdbScanHandle* pScanHandle);
-
-// ------------------ tsdbCommitQueue.c
-int tsdbScheduleCommit(STsdbRepo *pRepo);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/tsdb/inc/tsdbMemTable.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_MEMTABLE_H_
+#define _TD_TSDB_MEMTABLE_H_
+
+typedef struct {
+  int   rowsInserted;
+  int   rowsUpdated;
+  int   rowsDeleteSucceed;
+  int   rowsDeleteFailed;
+  int   nOperations;
+  TSKEY keyFirst;
+  TSKEY keyLast;
+} SMergeInfo;
+
+typedef struct {
+  STable *           pTable;
+  SSkipListIterator *pIter;
+} SCommitIter;
+
+typedef struct {
+  uint64_t   uid;
+  TSKEY      keyFirst;
+  TSKEY      keyLast;
+  int64_t    numOfRows;
+  SSkipList* pData;
+} STableData;
+
+typedef struct {
+  T_REF_DECLARE()
+  SRWLatch     latch;
+  TSKEY        keyFirst;
+  TSKEY        keyLast;
+  int64_t      numOfRows;
+  int32_t      maxTables;
+  STableData** tData;
+  SList*       actList;
+  SList*       extraBuffList;
+  SList*       bufBlockList;
+  int64_t      pointsAdd;   // TODO
+  int64_t      storageAdd;  // TODO
+} SMemTable;
+
+enum { TSDB_UPDATE_META, TSDB_DROP_META };
+
+#ifdef WINDOWS
+#pragma pack(push ,1) 
+typedef struct {
+#else
+typedef struct __attribute__((packed)){
+#endif
+  char     act;
+  uint64_t uid;
+} SActObj;
+#ifdef WINDOWS
+#pragma pack(pop) 
+#endif
+
+typedef struct {
+  int  len;
+  char cont[];
+} SActCont;
+
+int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
+int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
+int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
+void  tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem);
+void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
+int   tsdbAsyncCommit(STsdbRepo* pRepo);
+int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
+                            TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
+void* tsdbCommitData(STsdbRepo* pRepo);
+
+static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
+  if (pIter == NULL) return NULL;
+
+  SSkipListNode* node = tSkipListIterGet(pIter);
+  if (node == NULL) return NULL;
+
+  return (SDataRow)SL_GET_NODE_DATA(node);
+}
+
+static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
+  SDataRow row = tsdbNextIterRow(pIter);
+  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
+
+  return dataRowKey(row);
+}
+
+static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
+  SDataRow row = tsdbNextIterRow(pIter);
+  if (row == NULL) return TKEY_NULL;
+
+  return dataRowTKey(row);
+}
+
+#endif /* _TD_TSDB_MEMTABLE_H_ */
\ No newline at end of file

src/tsdb/inc/tsdbMeta.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_META_H_
+#define _TD_TSDB_META_H_
+
+#define TSDB_MAX_TABLE_SCHEMAS 16
+
+typedef struct STable {
+  STableId       tableId;
+  ETableType     type;
+  tstr*          name;  // NOTE: there a flexible string here
+  uint64_t       suid;
+  struct STable* pSuper;  // super table pointer
+  uint8_t        numOfSchemas;
+  STSchema*      schema[TSDB_MAX_TABLE_SCHEMAS];
+  STSchema*      tagSchema;
+  SKVRow         tagVal;
+  SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
+  void*          eventHandler;   // TODO
+  void*          streamHandler;  // TODO
+  TSKEY          lastKey;
+  SDataRow       lastRow;
+  char*          sql;
+  void*          cqhandle;
+  SRWLatch       latch;  // TODO: implementa latch functions
+  T_REF_DECLARE()
+} STable;
+
+typedef struct {
+  pthread_rwlock_t rwLock;
+
+  int32_t   nTables;
+  int32_t   maxTables;
+  STable**  tables;
+  SList*    superList;
+  SHashObj* uidMap;
+  int       maxRowBytes;
+  int       maxCols;
+} STsdbMeta;
+
+#define TSDB_INIT_NTABLES 1024
+#define TABLE_TYPE(t) (t)->type
+#define TABLE_NAME(t) (t)->name
+#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
+#define TABLE_UID(t) (t)->tableId.uid
+#define TABLE_TID(t) (t)->tableId.tid
+#define TABLE_SUID(t) (t)->suid
+// #define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore)
+#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch))
+#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch))
+#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch))
+#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch))
+
+STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
+void       tsdbFreeMeta(STsdbMeta* pMeta);
+int        tsdbOpenMeta(STsdbRepo* pRepo);
+int        tsdbCloseMeta(STsdbRepo* pRepo);
+STable*    tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
+STSchema*  tsdbGetTableSchemaByVersion(STable* pTable, int16_t version);
+int        tsdbWLockRepoMeta(STsdbRepo* pRepo);
+int        tsdbRLockRepoMeta(STsdbRepo* pRepo);
+int        tsdbUnlockRepoMeta(STsdbRepo* pRepo);
+void       tsdbRefTable(STable* pTable);
+void       tsdbUnRefTable(STable* pTable);
+void       tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct);
+int        tsdbRestoreTable(STsdbRepo* pRepo, void* cont, int contLen);
+void       tsdbOrgMeta(STsdbRepo* pRepo);
+
+static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
+  if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) {
+    return -1;
+  } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) {
+  STable*   pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable;
+  STSchema* pSchema = NULL;
+  STSchema* pTSchema = NULL;
+
+  if (lock) TSDB_RLOCK_TABLE(pDTable);
+  if (version < 0) {  // get the latest version of schema
+    pTSchema = pDTable->schema[pDTable->numOfSchemas - 1];
+  } else {  // get the schema with version
+    void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*),
+                            tsdbCompareSchemaVersion, TD_EQ);
+    if (ptr == NULL) {
+      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
+      goto _exit;
+    }
+    pTSchema = *(STSchema**)ptr;
+  }
+
+  ASSERT(pTSchema != NULL);
+
+  if (copy) {
+    if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+  } else {
+    pSchema = pTSchema;
+  }
+
+_exit:
+  if (lock) TSDB_RUNLOCK_TABLE(pDTable);
+  return pSchema;
+}
+
+static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) {
+  return tsdbGetTableSchemaImpl(pTable, false, false, -1);
+}
+
+static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) {
+  if (pTable->type == TSDB_CHILD_TABLE) {  // check child table first
+    STable *pSuper = pTable->pSuper;
+    if (pSuper == NULL) return NULL;
+    return pSuper->tagSchema;
+  } else if (pTable->type == TSDB_SUPER_TABLE) {
+    return pTable->tagSchema;
+  } else {
+    return NULL;
+  }
+}
+
+static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) {
+  ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow));
+  return pTable->lastKey;
+}
+
+#endif /* _TD_TSDB_META_H_ */
\ No newline at end of file

src/tsdb/inc/tsdbReadImpl.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_READ_IMPL_H_
+#define _TD_TSDB_READ_IMPL_H_
+
+typedef struct SReadH SReadH;
+
+typedef struct {
+  int32_t  tid;
+  uint32_t len;
+  uint32_t offset;
+  uint32_t hasLast : 2;
+  uint32_t numOfBlocks : 30;
+  uint64_t uid;
+  TSKEY    maxKey;
+} SBlockIdx;
+
+typedef struct {
+  int64_t last : 1;
+  int64_t offset : 63;
+  int32_t algorithm : 8;
+  int32_t numOfRows : 24;
+  int32_t len;
+  int32_t keyLen;  // key column length, keyOffset = offset+sizeof(SBlockData)+sizeof(SBlockCol)*numOfCols
+  int16_t numOfSubBlocks;
+  int16_t numOfCols;  // not including timestamp column
+  TSKEY   keyFirst;
+  TSKEY   keyLast;
+} SBlock;
+
+typedef struct {
+  int32_t  delimiter;  // For recovery usage
+  int32_t  tid;
+  uint64_t uid;
+  SBlock   blocks[];
+} SBlockInfo;
+
+typedef struct {
+  int16_t colId;
+  int32_t len;
+  int32_t type : 8;
+  int32_t offset : 24;
+  int64_t sum;
+  int64_t max;
+  int64_t min;
+  int16_t maxIndex;
+  int16_t minIndex;
+  int16_t numOfNull;
+  char    padding[2];
+} SBlockCol;
+
+typedef struct {
+  int32_t   delimiter;  // For recovery usage
+  int32_t   numOfCols;  // For recovery usage
+  uint64_t  uid;        // For recovery usage
+  SBlockCol cols[];
+} SBlockData;
+
+struct SReadH {
+  STsdbRepo * pRepo;
+  SDFileSet   rSet;     // FSET to read
+  SArray *    aBlkIdx;  // SBlockIdx array
+  STable *    pTable;   // table to read
+  SBlockIdx * pBlkIdx;  // current reading table SBlockIdx
+  int         cidx;
+  SBlockInfo *pBlkInfo;
+  SBlockData *pBlkData;  // Block info
+  SDataCols * pDCols[2];
+  void *      pBuf;   // buffer
+  void *      pCBuf;  // compression buffer
+};
+
+#define TSDB_READ_REPO(rh) ((rh)->pRepo)
+#define TSDB_READ_REPO_ID(rh) REPO_ID(TSDB_READ_REPO(rh))
+#define TSDB_READ_FSET(rh) (&((rh)->rSet))
+#define TSDB_READ_TABLE(rh) ((rh)->pTable)
+#define TSDB_READ_HEAD_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_HEAD)
+#define TSDB_READ_DATA_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_DATA)
+#define TSDB_READ_LAST_FILE(rh) TSDB_DFILE_IN_SET(TSDB_READ_FSET(rh), TSDB_FILE_LAST)
+#define TSDB_READ_BUF(rh) ((rh)->pBuf)
+#define TSDB_READ_COMP_BUF(rh) ((rh)->pCBuf)
+
+#define TSDB_BLOCK_STATIS_SIZE(ncols) (sizeof(SBlockData) + sizeof(SBlockCol) * (ncols) + sizeof(TSCKSUM))
+
+int   tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo);
+void  tsdbDestroyReadH(SReadH *pReadh);
+int   tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet);
+void  tsdbCloseAndUnsetFSet(SReadH *pReadh);
+int   tsdbLoadBlockIdx(SReadH *pReadh);
+int   tsdbSetReadTable(SReadH *pReadh, STable *pTable);
+int   tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget);
+int   tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlockInfo);
+int   tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds);
+int   tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock);
+int   tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx);
+void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx);
+void  tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols);
+
+static FORCE_INLINE int tsdbMakeRoom(void **ppBuf, size_t size) {
+  void * pBuf = *ppBuf;
+  size_t tsize = taosTSizeof(pBuf);
+
+  if (tsize < size) {
+    if (tsize == 0) tsize = 1024;
+
+    while (tsize < size) {
+      tsize *= 2;
+    }
+
+    *ppBuf = taosTRealloc(pBuf, tsize);
+    if (*ppBuf == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+#endif /*_TD_TSDB_READ_IMPL_H_*/
\ No newline at end of file

src/tsdb/inc/tsdbint.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_INT_H_
+#define _TD_TSDB_INT_H_
+
+// // TODO: remove the include
+// #include <errno.h>
+// #include <fcntl.h>
+// #include <limits.h>
+// #include <inttypes.h>
+// #include <sys/stat.h>
+// #include <sys/types.h>
+// #include <semaphore.h>
+// #include <dirent.h>
+
+#include "os.h"
+#include "tlog.h"
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tchecksum.h"
+#include "tskiplist.h"
+#include "tdataformat.h"
+#include "tcoding.h"
+#include "tscompression.h"
+#include "tlockfree.h"
+#include "tlist.h"
+#include "hash.h"
+#include "tarray.h"
+#include "tfs.h"
+#include "tsocket.h"
+
+#include "tsdb.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+// Log
+#include "tsdbLog.h"
+// Meta
+#include "tsdbMeta.h"
+// Buffer
+#include "tsdbBuffer.h"
+// MemTable
+#include "tsdbMemTable.h"
+// File
+#include "tsdbFile.h"
+// FS
+#include "tsdbFS.h"
+// ReadImpl
+#include "tsdbReadImpl.h"
+// Commit
+#include "tsdbCommit.h"
+// Commit Queue
+#include "tsdbCommitQueue.h"
+// Main definitions
+struct STsdbRepo {
+  uint8_t state;
+
+  STsdbCfg        config;
+  STsdbAppH       appH;
+  STsdbStat       stat;
+  STsdbMeta*      tsdbMeta;
+  STsdbBufPool*   pPool;
+  SMemTable*      mem;
+  SMemTable*      imem;
+  STsdbFS*        fs;
+  tsem_t          readyToCommit;
+  pthread_mutex_t mutex;
+  bool            repoLocked;
+  int32_t         code;  // Commit code
+};
+
+#define REPO_ID(r) (r)->config.tsdbId
+#define REPO_CFG(r) (&((r)->config))
+#define REPO_FS(r) ((r)->fs)
+#define IS_REPO_LOCKED(r) (r)->repoLocked
+#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
+
+int        tsdbLockRepo(STsdbRepo* pRepo);
+int        tsdbUnlockRepo(STsdbRepo* pRepo);
+STsdbMeta* tsdbGetMeta(STsdbRepo* pRepo);
+int        tsdbCheckCommit(STsdbRepo* pRepo);
+int        tsdbRestoreInfo(STsdbRepo* pRepo);
+void       tsdbGetRootDir(int repoid, char dirName[]);
+void       tsdbGetDataDir(int repoid, char dirName[]);
+
+static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
+  ASSERT(pRepo != NULL);
+  if (pRepo->mem == NULL) return NULL;
+
+  SListNode* pNode = listTail(pRepo->mem->bufBlockList);
+  if (pNode == NULL) return NULL;
+
+  STsdbBufBlock* pBufBlock = NULL;
+  tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock));
+
+  return pBufBlock;
+}
+
+static FORCE_INLINE int tsdbGetNextMaxTables(int tid) {
+  ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES);
+  int maxTables = TSDB_INIT_NTABLES;
+  while (true) {
+    maxTables = MIN(maxTables, TSDB_MAX_TABLES);
+    if (tid <= maxTables) break;
+    maxTables *= 2;
+  }
+
+  return maxTables + 1;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _TD_TSDB_INT_H_ */
\ No newline at end of file

src/tsdb/src/tsdbBuffer.c

@@ -13,8 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdb.h"
-#include "tsdbMain.h"
+#include "tsdbint.h"
 
 #define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0)
 

src/tsdb/src/tsdbCommit.c

@@ -12,23 +12,82 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include "tsdbMain.h"
+#include "tsdbint.h"
+
+#define TSDB_MAX_SUBBLOCKS 8
+#define TSDB_KEY_FID(key, days, precision) ((key) / tsMsPerDay[(precision)] / (days))
+
+typedef struct {
+  SRtn         rtn;     // retention snapshot
+  SFSIter      fsIter;  // tsdb file iterator
+  int          niters;  // memory iterators
+  SCommitIter *iters;
+  bool         isRFileSet; // read and commit FSET
+  SReadH       readh;
+  SDFileSet    wSet;
+  bool         isDFileSame;
+  bool         isLFileSame;
+  TSKEY        minKey;
+  TSKEY        maxKey;
+  SArray *     aBlkIdx;  // SBlockIdx array
+  STable *     pTable;
+  SArray *     aSupBlk;  // Table super-block array
+  SArray *     aSubBlk;  // table sub-block array
+  SDataCols *  pDataCols;
+} SCommitH;
+
+#define TSDB_COMMIT_REPO(ch) TSDB_READ_REPO(&(ch->readh))
+#define TSDB_COMMIT_REPO_ID(ch) REPO_ID(TSDB_READ_REPO(&(ch->readh)))
+#define TSDB_COMMIT_WRITE_FSET(ch) (&((ch)->wSet))
+#define TSDB_COMMIT_TABLE(ch) ((ch)->pTable)
+#define TSDB_COMMIT_HEAD_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_HEAD)
+#define TSDB_COMMIT_DATA_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_DATA)
+#define TSDB_COMMIT_LAST_FILE(ch) TSDB_DFILE_IN_SET(TSDB_COMMIT_WRITE_FSET(ch), TSDB_FILE_LAST)
+#define TSDB_COMMIT_BUF(ch) TSDB_READ_BUF(&((ch)->readh))
+#define TSDB_COMMIT_COMP_BUF(ch) TSDB_READ_COMP_BUF(&((ch)->readh))
+#define TSDB_COMMIT_DEFAULT_ROWS(ch) (TSDB_COMMIT_REPO(ch)->config.maxRowsPerFileBlock * 4 / 5)
+#define TSDB_COMMIT_TXN_VERSION(ch) FS_TXN_VERSION(REPO_FS(TSDB_COMMIT_REPO(ch)))
 
-static int  tsdbCommitTSData(STsdbRepo *pRepo);
 static int  tsdbCommitMeta(STsdbRepo *pRepo);
+static int  tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen);
+static int  tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid);
+static int  tsdbCommitTSData(STsdbRepo *pRepo);
+static int  tsdbStartCommit(STsdbRepo *pRepo);
 static void tsdbEndCommit(STsdbRepo *pRepo, int eno);
-static int  tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey);
-static int  tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols);
-static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo);
-static void         tsdbDestroyCommitIters(SCommitIter *iters, int maxTables);
+static int  tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
+static int  tsdbCreateCommitIters(SCommitH *pCommith);
+static void tsdbDestroyCommitIters(SCommitH *pCommith);
+static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key);
+static int  tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo);
+static void tsdbDestroyCommitH(SCommitH *pCommith);
+static int  tsdbGetFidLevel(int fid, SRtn *pRtn);
+static int  tsdbNextCommitFid(SCommitH *pCommith);
+static int  tsdbCommitToTable(SCommitH *pCommith, int tid);
+static int  tsdbSetCommitTable(SCommitH *pCommith, STable *pTable);
+static int  tsdbComparKeyBlock(const void *arg1, const void *arg2);
+static int  tsdbWriteBlockInfo(SCommitH *pCommih);
+static int  tsdbWriteBlockIdx(SCommitH *pCommih);
+static int  tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData);
+static int  tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx);
+static int  tsdbMoveBlock(SCommitH *pCommith, int bidx);
+static int  tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks);
+static int  tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit,
+                               bool isLastOneBlock);
+static void tsdbResetCommitFile(SCommitH *pCommith);
+static void tsdbResetCommitTable(SCommitH *pCommith);
+static int  tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid);
+static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError);
+static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo);
+static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
+                                      TSKEY maxKey, int maxRows, int8_t update);
+static int  tsdbApplyRtn(STsdbRepo *pRepo);
+static int  tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn);
 
 void *tsdbCommitData(STsdbRepo *pRepo) {
-  SMemTable *  pMem = pRepo->imem;
-
-  tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d",
-           REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList));
-
-  pRepo->code = TSDB_CODE_SUCCESS;
+  if (tsdbStartCommit(pRepo) < 0) {
+    tsdbError("vgId:%d failed to commit data while startting to commit since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
 
   // Commit to update meta file
   if (tsdbCommitMeta(pRepo) < 0) {
@@ -42,125 +101,279 @@ void *tsdbCommitData(STsdbRepo *pRepo) {
     goto _err;
   }
 
-  tsdbFitRetention(pRepo);
-
-  tsdbInfo("vgId:%d commit over, succeed", REPO_ID(pRepo));
   tsdbEndCommit(pRepo, TSDB_CODE_SUCCESS);
-
   return NULL;
 
 _err:
   ASSERT(terrno != TSDB_CODE_SUCCESS);
   pRepo->code = terrno;
-  tsdbInfo("vgId:%d commit over, failed", REPO_ID(pRepo));
-  tsdbEndCommit(pRepo, terrno);
 
+  tsdbEndCommit(pRepo, terrno);
   return NULL;
 }
 
-static int tsdbCommitTSData(STsdbRepo *pRepo) {
-  SMemTable *  pMem = pRepo->imem;
-  SDataCols *  pDataCols = NULL;
-  STsdbMeta *  pMeta = pRepo->tsdbMeta;
-  SCommitIter *iters = NULL;
-  SRWHelper    whelper = {0};
-  STsdbCfg *   pCfg = &(pRepo->config);
-
-  if (pMem->numOfRows <= 0) return 0;
-
-  iters = tsdbCreateCommitIters(pRepo);
-  if (iters == NULL) {
-    tsdbError("vgId:%d failed to create commit iterator since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+// =================== Commit Meta Data
+static int tsdbCommitMeta(STsdbRepo *pRepo) {
+  STsdbFS *  pfs = REPO_FS(pRepo);
+  SMemTable *pMem = pRepo->imem;
+  SMFile *   pOMFile = pfs->cstatus->pmf;
+  SMFile     mf;
+  SActObj *  pAct = NULL;
+  SActCont * pCont = NULL;
+  SListNode *pNode = NULL;
+  SDiskID    did;
+
+  ASSERT(pOMFile != NULL || listNEles(pMem->actList) > 0);
+
+  if (listNEles(pMem->actList) <= 0) {
+    // no meta data to commit, just keep the old meta file
+    tsdbUpdateMFile(pfs, pOMFile);
+    return 0;
+  } else {
+    // Create/Open a meta file or open the existing file
+    if (pOMFile == NULL) {
+      // Create a new meta file
+      did.level = TFS_PRIMARY_LEVEL;
+      did.id = TFS_PRIMARY_ID;
+      tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
+
+      if (tsdbCreateMFile(&mf, true) < 0) {
+        return -1;
+      }
+    } else {
+      tsdbInitMFileEx(&mf, pOMFile);
+      if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
+        return -1;
+      }
+    }
   }
 
-  if (tsdbInitWriteHelper(&whelper, pRepo) < 0) {
-    tsdbError("vgId:%d failed to init write helper since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+  // Loop to write
+  while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
+    pAct = (SActObj *)pNode->data;
+    if (pAct->act == TSDB_UPDATE_META) {
+      pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
+      if (tsdbUpdateMetaRecord(pfs, &mf, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
+        tsdbCloseMFile(&mf);
+        return -1;
+      }
+    } else if (pAct->act == TSDB_DROP_META) {
+      if (tsdbDropMetaRecord(pfs, &mf, pAct->uid) < 0) {
+        tsdbCloseMFile(&mf);
+        return -1;
+      }
+    } else {
+      ASSERT(false);
+    }
   }
 
-  if ((pDataCols = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pCfg->maxRowsPerFileBlock)) == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    tsdbError("vgId:%d failed to init data cols with maxRowBytes %d maxCols %d maxRowsPerFileBlock %d since %s",
-              REPO_ID(pRepo), pMeta->maxCols, pMeta->maxRowBytes, pCfg->maxRowsPerFileBlock, tstrerror(terrno));
-    goto _err;
+  if (tsdbUpdateMFileHeader(&mf) < 0) {
+    return -1;
   }
 
-  int sfid = (int)(TSDB_KEY_FILEID(pMem->keyFirst, pCfg->daysPerFile, pCfg->precision));
-  int efid = (int)(TSDB_KEY_FILEID(pMem->keyLast, pCfg->daysPerFile, pCfg->precision));
+  TSDB_FILE_FSYNC(&mf);
+  tsdbCloseMFile(&mf);
+  tsdbUpdateMFile(pfs, &mf);
 
-  // Loop to commit to each file
-  for (int fid = sfid; fid <= efid; fid++) {
-    if (tsdbCommitToFile(pRepo, fid, iters, &whelper, pDataCols) < 0) {
-      tsdbError("vgId:%d failed to commit to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
-      goto _err;
-    }
+  return 0;
+}
+
+int tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord) {
+  int tlen = 0;
+  tlen += taosEncodeFixedU64(buf, pRecord->uid);
+  tlen += taosEncodeFixedI64(buf, pRecord->offset);
+  tlen += taosEncodeFixedI64(buf, pRecord->size);
+
+  return tlen;
+}
+
+void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord) {
+  buf = taosDecodeFixedU64(buf, &(pRecord->uid));
+  buf = taosDecodeFixedI64(buf, &(pRecord->offset));
+  buf = taosDecodeFixedI64(buf, &(pRecord->size));
+
+  return buf;
+}
+
+void tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn) {
+  STsdbCfg *pCfg = REPO_CFG(pRepo);
+  TSKEY     minKey, midKey, maxKey, now;
+
+  now = taosGetTimestamp(pCfg->precision);
+  minKey = now - pCfg->keep * tsMsPerDay[pCfg->precision];
+  midKey = now - pCfg->keep2 * tsMsPerDay[pCfg->precision];
+  maxKey = now - pCfg->keep1 * tsMsPerDay[pCfg->precision];
+
+  pRtn->minKey = minKey;
+  pRtn->minFid = (int)(TSDB_KEY_FID(minKey, pCfg->daysPerFile, pCfg->precision));
+  pRtn->midFid = (int)(TSDB_KEY_FID(midKey, pCfg->daysPerFile, pCfg->precision));
+  pRtn->maxFid = (int)(TSDB_KEY_FID(maxKey, pCfg->daysPerFile, pCfg->precision));
+}
+
+static int tsdbUpdateMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid, void *cont, int contLen) {
+  char      buf[64] = "\0";
+  void *    pBuf = buf;
+  SKVRecord rInfo;
+  int64_t   offset;
+
+  // Seek to end of meta file
+  offset = tsdbSeekMFile(pMFile, 0, SEEK_END);
+  if (offset < 0) {
+    return -1;
   }
 
-  tdFreeDataCols(pDataCols);
-  tsdbDestroyCommitIters(iters, pMem->maxTables);
-  tsdbDestroyHelper(&whelper);
+  rInfo.offset = offset;
+  rInfo.uid = uid;
+  rInfo.size = contLen;
+
+  int tlen = tsdbEncodeKVRecord((void **)(&pBuf), &rInfo);
+  if (tsdbAppendMFile(pMFile, buf, tlen, NULL) < tlen) {
+    return -1;
+  }
+
+  if (tsdbAppendMFile(pMFile, cont, contLen, NULL) < contLen) {
+    return -1;
+  }
+
+  tsdbUpdateMFileMagic(pMFile, POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)));
+  SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)&uid, sizeof(uid));
+  if (pRecord != NULL) {
+    pMFile->info.tombSize += pRecord->size;
+  } else {
+    pMFile->info.nRecords++;
+  }
+  taosHashPut(pfs->metaCache, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo));
 
   return 0;
+}
 
-_err:
-  tdFreeDataCols(pDataCols);
-  tsdbDestroyCommitIters(iters, pMem->maxTables);
-  tsdbDestroyHelper(&whelper);
+static int tsdbDropMetaRecord(STsdbFS *pfs, SMFile *pMFile, uint64_t uid) {
+  SKVRecord rInfo = {0};
+  char      buf[128] = "\0";
+
+  SKVRecord *pRecord = taosHashGet(pfs->metaCache, (void *)(&uid), sizeof(uid));
+  if (pRecord == NULL) {
+    tsdbError("failed to drop KV store record with key %" PRIu64 " since not find", uid);
+    return -1;
+  }
+
+  rInfo.offset = -pRecord->offset;
+  rInfo.uid = pRecord->uid;
+  rInfo.size = pRecord->size;
 
-  return -1;
+  void *pBuf = buf;
+  tsdbEncodeKVRecord(&pBuf, &rInfo);
+
+  if (tsdbAppendMFile(pMFile, buf, POINTER_DISTANCE(pBuf, buf), NULL) < 0) {
+    return -1;
+  }
+
+  pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t *)buf, (uint32_t)POINTER_DISTANCE(pBuf, buf));
+  pMFile->info.nDels++;
+  pMFile->info.nRecords--;
+  pMFile->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
+
+  taosHashRemove(pfs->metaCache, (void *)(&uid), sizeof(uid));
+  return 0;
 }
 
-static int tsdbCommitMeta(STsdbRepo *pRepo) {
+// =================== Commit Time-Series Data
+static int tsdbCommitTSData(STsdbRepo *pRepo) {
   SMemTable *pMem = pRepo->imem;
-  STsdbMeta *pMeta = pRepo->tsdbMeta;
-  SActObj *  pAct = NULL;
-  SActCont * pCont = NULL;
+  SCommitH   commith;
+  SDFileSet *pSet = NULL;
+  int        fid;
 
-  if (listNEles(pMem->actList) <= 0) return 0;
+  memset(&commith, 0, sizeof(SMemTable *));
 
-  if (tdKVStoreStartCommit(pMeta->pStore) < 0) {
-    tsdbError("vgId:%d failed to commit data while start commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+  if (pMem->numOfRows <= 0) {
+    // No memory data, just apply retention on each file on disk
+    if (tsdbApplyRtn(pRepo) < 0) {
+      return -1;
+    }
+    return 0;
   }
 
-  SListNode *pNode = NULL;
+  // Resource initialization
+  if (tsdbInitCommitH(&commith, pRepo) < 0) {
+    return -1;
+  }
 
-  while ((pNode = tdListPopHead(pMem->actList)) != NULL) {
-    pAct = (SActObj *)pNode->data;
-    if (pAct->act == TSDB_UPDATE_META) {
-      pCont = (SActCont *)POINTER_SHIFT(pAct, sizeof(SActObj));
-      if (tdUpdateKVStoreRecord(pMeta->pStore, pAct->uid, (void *)(pCont->cont), pCont->len) < 0) {
-        tsdbError("vgId:%d failed to update meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
-                  tstrerror(terrno));
-        tdKVStoreEndCommit(pMeta->pStore);
-        goto _err;
-      }
-    } else if (pAct->act == TSDB_DROP_META) {
-      if (tdDropKVStoreRecord(pMeta->pStore, pAct->uid) < 0) {
-        tsdbError("vgId:%d failed to drop meta with uid %" PRIu64 " since %s", REPO_ID(pRepo), pAct->uid,
-                  tstrerror(terrno));
-        tdKVStoreEndCommit(pMeta->pStore);
-        goto _err;
+  // Skip expired memory data and expired FSET
+  tsdbSeekCommitIter(&commith, commith.rtn.minKey);
+  while ((pSet = tsdbFSIterNext(&(commith.fsIter)))) {
+    if (pSet->fid >= commith.rtn.minFid) break;
+  }
+
+  // Loop to commit to each file
+  fid = tsdbNextCommitFid(&(commith));
+  while (true) {
+    // Loop over both on disk and memory
+    if (pSet == NULL && fid == TSDB_IVLD_FID) break;
+
+    if (pSet && (fid == TSDB_IVLD_FID || pSet->fid < fid)) {
+      // Only has existing FSET but no memory data to commit in this
+      // existing FSET, only check if file in correct retention
+      if (tsdbApplyRtnOnFSet(pRepo, pSet, &(commith.rtn)) < 0) {
+        tsdbDestroyCommitH(&commith);
+        return -1;
       }
+
+      pSet = tsdbFSIterNext(&(commith.fsIter));
     } else {
-      ASSERT(false);
-    }
-  }
+      // Has memory data to commit
+      SDFileSet *pCSet;
+      int        cfid;
+
+      if (pSet == NULL || pSet->fid > fid) {
+        // Commit to a new FSET with fid: fid
+        pCSet = NULL;
+        cfid = fid;
+      } else {
+        // Commit to an existing FSET
+        pCSet = pSet;
+        cfid = pSet->fid;
+        pSet = tsdbFSIterNext(&(commith.fsIter));
+      }
 
-  if (tdKVStoreEndCommit(pMeta->pStore) < 0) {
-    tsdbError("vgId:%d failed to commit data while end commit meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+      if (tsdbCommitToFile(&commith, pCSet, cfid) < 0) {
+        tsdbDestroyCommitH(&commith);
+        return -1;
+      }
+
+      fid = tsdbNextCommitFid(&commith);
+    }
   }
 
+  tsdbDestroyCommitH(&commith);
   return 0;
+}
 
-_err:
-  return -1;
+static int tsdbStartCommit(STsdbRepo *pRepo) {
+  SMemTable *pMem = pRepo->imem;
+
+  ASSERT(pMem->numOfRows > 0 || listNEles(pMem->actList) > 0);
+
+  tsdbInfo("vgId:%d start to commit! keyFirst %" PRId64 " keyLast %" PRId64 " numOfRows %" PRId64 " meta rows: %d",
+           REPO_ID(pRepo), pMem->keyFirst, pMem->keyLast, pMem->numOfRows, listNEles(pMem->actList));
+
+  tsdbStartFSTxn(pRepo, pMem->pointsAdd, pMem->storageAdd);
+
+  pRepo->code = TSDB_CODE_SUCCESS;
+  return 0;
 }
 
 static void tsdbEndCommit(STsdbRepo *pRepo, int eno) {
+  if (eno != TSDB_CODE_SUCCESS) {
+    tsdbEndFSTxnWithError(REPO_FS(pRepo));
+  } else {
+    tsdbEndFSTxn(pRepo);
+  }
+
+  tsdbInfo("vgId:%d commit over, %s", REPO_ID(pRepo), (eno == TSDB_CODE_SUCCESS) ? "succeed" : "failed");
+
   if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno);
+
   SMemTable *pIMem = pRepo->imem;
   tsdbLockRepo(pRepo);
   pRepo->imem = NULL;
@@ -169,177 +382,1042 @@ static void tsdbEndCommit(STsdbRepo *pRepo, int eno) {
   tsem_post(&(pRepo->readyToCommit));
 }
 
-static int tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
+#if 0
+static bool tsdbHasDataToCommit(SCommitIter *iters, int nIters, TSKEY minKey, TSKEY maxKey) {
   for (int i = 0; i < nIters; i++) {
     TSKEY nextKey = tsdbNextIterKey((iters + i)->pIter);
-    if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return 1;
+    if (nextKey != TSDB_DATA_TIMESTAMP_NULL && (nextKey >= minKey && nextKey <= maxKey)) return true;
+  }
+  return false;
+}
+#endif
+
+static int tsdbCommitToFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
+  STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith);
+  STsdbCfg * pCfg = REPO_CFG(pRepo);
+
+  ASSERT(pSet == NULL || pSet->fid == fid);
+
+  tsdbResetCommitFile(pCommith);
+  tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &(pCommith->minKey), &(pCommith->maxKey));
+
+  // Set and open files
+  if (tsdbSetAndOpenCommitFile(pCommith, pSet, fid) < 0) {
+    return -1;
+  }
+
+  // Loop to commit each table data
+  for (int tid = 1; tid < pCommith->niters; tid++) {
+    SCommitIter *pIter = pCommith->iters + tid;
+
+    if (pIter->pTable == NULL) continue;
+
+    if (tsdbCommitToTable(pCommith, tid) < 0) {
+      // TODO: revert the file change
+      tsdbCloseCommitFile(pCommith, true);
+      return -1;
+    }
+  }
+
+  if (tsdbWriteBlockIdx(pCommith) < 0) {
+    tsdbCloseCommitFile(pCommith, true);
+    return -1;
   }
+
+  // Close commit file
+  tsdbCloseCommitFile(pCommith, false);
+
+  if (tsdbUpdateDFileSet(REPO_FS(pRepo), &(pCommith->wSet)) < 0) {
+    return -1;
+  }
+
   return 0;
 }
 
-static int tsdbCommitToFile(STsdbRepo *pRepo, int fid, SCommitIter *iters, SRWHelper *pHelper, SDataCols *pDataCols) {
-  char *      dataDir = NULL;
-  STsdbCfg *  pCfg = &pRepo->config;
-  STsdbFileH *pFileH = pRepo->tsdbFileH;
-  SFileGroup *pGroup = NULL;
-  SMemTable * pMem = pRepo->imem;
-  bool        newLast = false;
+static int tsdbCreateCommitIters(SCommitH *pCommith) {
+  STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith);
+  SMemTable *pMem = pRepo->imem;
+  STsdbMeta *pMeta = pRepo->tsdbMeta;
 
-  TSKEY minKey = 0, maxKey = 0;
-  tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, fid, &minKey, &maxKey);
+  pCommith->niters = pMem->maxTables;
+  pCommith->iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter));
+  if (pCommith->iters == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
 
-  // Check if there are data to commit to this file
-  int hasDataToCommit = tsdbHasDataToCommit(iters, pMem->maxTables, minKey, maxKey);
-  if (!hasDataToCommit) {
-    tsdbDebug("vgId:%d no data to commit to file %d", REPO_ID(pRepo), fid);
-    return 0;
+  if (tsdbRLockRepoMeta(pRepo) < 0) return -1;
+
+  // reference all tables
+  for (int i = 0; i < pMem->maxTables; i++) {
+    if (pMeta->tables[i] != NULL) {
+      tsdbRefTable(pMeta->tables[i]);
+      pCommith->iters[i].pTable = pMeta->tables[i];
+    }
   }
 
-  // Create and open files for commit
-  dataDir = tsdbGetDataDirName(pRepo->rootDir);
-  if (dataDir == NULL) {
+  if (tsdbUnlockRepoMeta(pRepo) < 0) return -1;
+
+  for (int i = 0; i < pMem->maxTables; i++) {
+    if ((pCommith->iters[i].pTable != NULL) && (pMem->tData[i] != NULL) &&
+        (TABLE_UID(pCommith->iters[i].pTable) == pMem->tData[i]->uid)) {
+      if ((pCommith->iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        return -1;
+      }
+
+      tSkipListIterNext(pCommith->iters[i].pIter);
+    }
+  }
+
+  return 0;
+}
+
+static void tsdbDestroyCommitIters(SCommitH *pCommith) {
+  if (pCommith->iters == NULL) return;
+
+  for (int i = 1; i < pCommith->niters; i++) {
+    if (pCommith->iters[i].pTable != NULL) {
+      tsdbUnRefTable(pCommith->iters[i].pTable);
+      tSkipListDestroyIter(pCommith->iters[i].pIter);
+    }
+  }
+
+  free(pCommith->iters);
+  pCommith->iters = NULL;
+  pCommith->niters = 0;
+}
+
+// Skip all keys until key (not included)
+static void tsdbSeekCommitIter(SCommitH *pCommith, TSKEY key) {
+  for (int i = 0; i < pCommith->niters; i++) {
+    SCommitIter *pIter = pCommith->iters + i;
+    if (pIter->pTable == NULL || pIter->pIter == NULL) continue;
+
+    tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, key - 1, INT32_MAX, NULL, NULL, 0, true, NULL);
+  }
+}
+
+static int tsdbInitCommitH(SCommitH *pCommith, STsdbRepo *pRepo) {
+  STsdbCfg *pCfg = REPO_CFG(pRepo);
+
+  memset(pCommith, 0, sizeof(*pCommith));
+  tsdbGetRtnSnap(pRepo, &(pCommith->rtn));
+
+  TSDB_FSET_SET_CLOSED(TSDB_COMMIT_WRITE_FSET(pCommith));
+
+  // Init read handle
+  if (tsdbInitReadH(&(pCommith->readh), pRepo) < 0) {
+    return -1;
+  }
+
+  // Init file iterator
+  tsdbFSIterInit(&(pCommith->fsIter), REPO_FS(pRepo), TSDB_FS_ITER_FORWARD);
+
+  if (tsdbCreateCommitIters(pCommith) < 0) {
+    tsdbDestroyCommitH(pCommith);
+    return -1;
+  }
+
+  pCommith->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx));
+  if (pCommith->aBlkIdx == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbDestroyCommitH(pCommith);
     return -1;
   }
 
-  if ((pGroup = tsdbCreateFGroupIfNeed(pRepo, dataDir, fid)) == NULL) {
-    tsdbError("vgId:%d failed to create file group %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
-    goto _err;
+  pCommith->aSupBlk = taosArrayInit(1024, sizeof(SBlock));
+  if (pCommith->aSupBlk == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbDestroyCommitH(pCommith);
+    return -1;
   }
 
-  // Open files for write/read
-  if (tsdbSetAndOpenHelperFile(pHelper, pGroup) < 0) {
-    tsdbError("vgId:%d failed to set helper file since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+  pCommith->aSubBlk = taosArrayInit(1024, sizeof(SBlock));
+  if (pCommith->aSubBlk == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbDestroyCommitH(pCommith);
+    return -1;
   }
 
-  newLast = TSDB_NLAST_FILE_OPENED(pHelper);
+  pCommith->pDataCols = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
+  if (pCommith->pDataCols == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbDestroyCommitH(pCommith);
+    return -1;
+  }
 
-  if (tsdbLoadCompIdx(pHelper, NULL) < 0) {
-    tsdbError("vgId:%d failed to load SCompIdx part since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+  return 0;
+}
+
+static void tsdbDestroyCommitH(SCommitH *pCommith) {
+  pCommith->pDataCols = tdFreeDataCols(pCommith->pDataCols);
+  pCommith->aSubBlk = taosArrayDestroy(pCommith->aSubBlk);
+  pCommith->aSupBlk = taosArrayDestroy(pCommith->aSupBlk);
+  pCommith->aBlkIdx = taosArrayDestroy(pCommith->aBlkIdx);
+  tsdbDestroyCommitIters(pCommith);
+  tsdbDestroyReadH(&(pCommith->readh));
+  tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith));
+}
+
+static int tsdbNextCommitFid(SCommitH *pCommith) {
+  STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith);
+  STsdbCfg * pCfg = REPO_CFG(pRepo);
+  int        fid = TSDB_IVLD_FID;
+
+  for (int i = 0; i < pCommith->niters; i++) {
+    SCommitIter *pIter = pCommith->iters + i;
+    if (pIter->pTable == NULL || pIter->pIter == NULL) continue;
+
+    TSKEY nextKey = tsdbNextIterKey(pIter->pIter);
+    if (nextKey == TSDB_DATA_TIMESTAMP_NULL) {
+      continue;
+    } else {
+      int tfid = (int)(TSDB_KEY_FID(nextKey, pCfg->daysPerFile, pCfg->precision));
+      if (fid == TSDB_IVLD_FID || fid > tfid) {
+        fid = tfid;
+      }
+    }
   }
 
-  // Loop to commit data in each table
-  for (int tid = 1; tid < pMem->maxTables; tid++) {
-    SCommitIter *pIter = iters + tid;
-    if (pIter->pTable == NULL) continue;
+  return fid;
+}
 
-    TSDB_RLOCK_TABLE(pIter->pTable);
+static int tsdbCommitToTable(SCommitH *pCommith, int tid) {
+  SCommitIter *pIter = pCommith->iters + tid;
+  TSKEY        nextKey = tsdbNextIterKey(pIter->pIter);
 
-    if (tsdbSetHelperTable(pHelper, pIter->pTable, pRepo) < 0) goto _err;
+  tsdbResetCommitTable(pCommith);
 
-    if (pIter->pIter != NULL) {
-      if (tdInitDataCols(pDataCols, tsdbGetTableSchemaImpl(pIter->pTable, false, false, -1)) < 0) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        goto _err;
+  TSDB_RLOCK_TABLE(pIter->pTable);
+
+  // Set commit table
+  if (tsdbSetCommitTable(pCommith, pIter->pTable) < 0) {
+    TSDB_RUNLOCK_TABLE(pIter->pTable);
+    return -1;
+  }
+
+  // No disk data and no memory data, just return
+  if (pCommith->readh.pBlkIdx == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) {
+    TSDB_RUNLOCK_TABLE(pIter->pTable);
+    return 0;
+  }
+
+  // Must has disk data or has memory data
+  int     nBlocks;
+  int     bidx = 0;
+  SBlock *pBlock;
+
+  if (pCommith->readh.pBlkIdx) {
+    if (tsdbLoadBlockInfo(&(pCommith->readh), NULL) < 0) {
+      TSDB_RUNLOCK_TABLE(pIter->pTable);
+      return -1;
+    }
+
+    nBlocks = pCommith->readh.pBlkIdx->numOfBlocks;
+  } else {
+    nBlocks = 0;
+  }
+
+  if (bidx < nBlocks) {
+    pBlock = pCommith->readh.pBlkInfo->blocks + bidx;
+  } else {
+    pBlock = NULL;
+  }
+
+  while (true) {
+    if (pBlock == NULL && (nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey)) break;
+
+    if ((nextKey == TSDB_DATA_TIMESTAMP_NULL || nextKey > pCommith->maxKey) ||
+        (pBlock && (!pBlock->last) && tsdbComparKeyBlock((void *)(&nextKey), pBlock) > 0)) {
+      if (tsdbMoveBlock(pCommith, bidx) < 0) {
+        TSDB_RUNLOCK_TABLE(pIter->pTable);
+        return -1;
       }
 
-      if (tsdbCommitTableData(pHelper, pIter, pDataCols, maxKey) < 0) {
+      bidx++;
+      if (bidx < nBlocks) {
+        pBlock = pCommith->readh.pBlkInfo->blocks + bidx;
+      } else {
+        pBlock = NULL;
+      }
+    } else if (pBlock && (pBlock->last || tsdbComparKeyBlock((void *)(&nextKey), pBlock) == 0)) {
+      // merge pBlock data and memory data
+      if (tsdbMergeMemData(pCommith, pIter, bidx) < 0) {
         TSDB_RUNLOCK_TABLE(pIter->pTable);
-        tsdbError("vgId:%d failed to write data of table %s tid %d uid %" PRIu64 " since %s", REPO_ID(pRepo),
-                  TABLE_CHAR_NAME(pIter->pTable), TABLE_TID(pIter->pTable), TABLE_UID(pIter->pTable),
-                  tstrerror(terrno));
-        goto _err;
+        return -1;
+      }
+
+      bidx++;
+      if (bidx < nBlocks) {
+        pBlock = pCommith->readh.pBlkInfo->blocks + bidx;
+      } else {
+        pBlock = NULL;
       }
+      nextKey = tsdbNextIterKey(pIter->pIter);
+    } else {
+      // Only commit memory data
+      if (pBlock == NULL) {
+        if (tsdbCommitMemData(pCommith, pIter, pCommith->maxKey, false) < 0) {
+          TSDB_RUNLOCK_TABLE(pIter->pTable);
+          return -1;
+        }
+      } else {
+        if (tsdbCommitMemData(pCommith, pIter, pBlock->keyFirst - 1, true) < 0) {
+          TSDB_RUNLOCK_TABLE(pIter->pTable);
+          return -1;
+        }
+      }
+      nextKey = tsdbNextIterKey(pIter->pIter);
     }
+  }
 
-    TSDB_RUNLOCK_TABLE(pIter->pTable);
+  TSDB_RUNLOCK_TABLE(pIter->pTable);
+
+  if (tsdbWriteBlockInfo(pCommith) < 0) return -1;
+
+  return 0;
+}
 
-    // Move the last block to the new .l file if neccessary
-    if (tsdbMoveLastBlockIfNeccessary(pHelper) < 0) {
-      tsdbError("vgId:%d, failed to move last block, since %s", REPO_ID(pRepo), tstrerror(terrno));
-      goto _err;
+static int tsdbSetCommitTable(SCommitH *pCommith, STable *pTable) {
+  STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
+
+  pCommith->pTable = pTable;
+
+  if (tdInitDataCols(pCommith->pDataCols, pSchema) < 0) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (pCommith->isRFileSet) {
+    if (tsdbSetReadTable(&(pCommith->readh), pTable) < 0) {
+      return -1;
     }
+  } else {
+    pCommith->readh.pBlkIdx = NULL;
+  }
+  return 0;
+}
+
+static int tsdbComparKeyBlock(const void *arg1, const void *arg2) {
+  TSKEY   key = *(TSKEY *)arg1;
+  SBlock *pBlock = (SBlock *)arg2;
+
+  if (key < pBlock->keyFirst) {
+    return -1;
+  } else if (key > pBlock->keyLast) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+static int tsdbWriteBlock(SCommitH *pCommith, SDFile *pDFile, SDataCols *pDataCols, SBlock *pBlock, bool isLast,
+                          bool isSuper) {
+  STsdbRepo * pRepo = TSDB_COMMIT_REPO(pCommith);
+  STsdbCfg *  pCfg = REPO_CFG(pRepo);
+  SBlockData *pBlockData;
+  int64_t     offset = 0;
+  STable *    pTable = TSDB_COMMIT_TABLE(pCommith);
+  int         rowsToWrite = pDataCols->numOfRows;
+
+  ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock);
+  ASSERT((!isLast) || rowsToWrite < pCfg->minRowsPerFileBlock);
+
+  // Make buffer space
+  if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommith)), TSDB_BLOCK_STATIS_SIZE(pDataCols->numOfCols)) < 0) {
+    return -1;
+  }
+  pBlockData = (SBlockData *)TSDB_COMMIT_BUF(pCommith);
 
-    // Write the SCompBlock part
-    if (tsdbWriteCompInfo(pHelper) < 0) {
-      tsdbError("vgId:%d, failed to write compInfo part since %s", REPO_ID(pRepo), tstrerror(terrno));
-      goto _err;
+  // Get # of cols not all NULL(not including key column)
+  int nColsNotAllNull = 0;
+  for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) {  // ncol from 1, we skip the timestamp column
+    SDataCol * pDataCol = pDataCols->cols + ncol;
+    SBlockCol *pBlockCol = pBlockData->cols + nColsNotAllNull;
+
+    if (isNEleNull(pDataCol, rowsToWrite)) {  // all data to commit are NULL, just ignore it
+      continue;
+    }
+
+    memset(pBlockCol, 0, sizeof(*pBlockCol));
+
+    pBlockCol->colId = pDataCol->colId;
+    pBlockCol->type = pDataCol->type;
+    if (tDataTypes[pDataCol->type].statisFunc) {
+      (*tDataTypes[pDataCol->type].statisFunc)(pDataCol->pData, rowsToWrite, &(pBlockCol->min), &(pBlockCol->max),
+                                               &(pBlockCol->sum), &(pBlockCol->minIndex), &(pBlockCol->maxIndex),
+                                               &(pBlockCol->numOfNull));
     }
+    nColsNotAllNull++;
   }
 
-  if (tsdbWriteCompIdx(pHelper) < 0) {
-    tsdbError("vgId:%d failed to write compIdx part to file %d since %s", REPO_ID(pRepo), fid, tstrerror(terrno));
-    goto _err;
+  ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols);
+
+  // Compress the data if neccessary
+  int     tcol = 0;  // counter of not all NULL and written columns
+  int32_t toffset = 0;
+  int32_t tsize = TSDB_BLOCK_STATIS_SIZE(nColsNotAllNull);
+  int32_t lsize = tsize;
+  int32_t keyLen = 0;
+  for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
+    // All not NULL columns finish
+    if (ncol != 0 && tcol >= nColsNotAllNull) break;
+
+    SDataCol * pDataCol = pDataCols->cols + ncol;
+    SBlockCol *pBlockCol = pBlockData->cols + tcol;
+
+    if (ncol != 0 && (pDataCol->colId != pBlockCol->colId)) continue;
+
+    int32_t flen;  // final length
+    int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite);
+    void *  tptr;
+
+    // Make room
+    if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommith)), lsize + tlen + COMP_OVERFLOW_BYTES + sizeof(TSCKSUM)) < 0) {
+      return -1;
+    }
+    pBlockData = (SBlockData *)TSDB_COMMIT_BUF(pCommith);
+    pBlockCol = pBlockData->cols + tcol;
+    tptr = POINTER_SHIFT(pBlockData, lsize);
+
+    if (pCfg->compression == TWO_STAGE_COMP &&
+        tsdbMakeRoom((void **)(&TSDB_COMMIT_COMP_BUF(pCommith)), tlen + COMP_OVERFLOW_BYTES) < 0) {
+      return -1;
+    }
+
+    // Compress or just copy
+    if (pCfg->compression) {
+      flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr,
+                                                      tlen + COMP_OVERFLOW_BYTES, pCfg->compression,
+                                                      TSDB_COMMIT_COMP_BUF(pCommith), tlen + COMP_OVERFLOW_BYTES);
+    } else {
+      flen = tlen;
+      memcpy(tptr, pDataCol->pData, flen);
+    }
+
+    // Add checksum
+    ASSERT(flen > 0);
+    flen += sizeof(TSCKSUM);
+    taosCalcChecksumAppend(0, (uint8_t *)tptr, flen);
+    tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)));
+
+    if (ncol != 0) {
+      pBlockCol->offset = toffset;
+      pBlockCol->len = flen;
+      tcol++;
+    } else {
+      keyLen = flen;
+    }
+
+    toffset += flen;
+    lsize += flen;
   }
 
-  tfree(dataDir);
-  tsdbCloseHelperFile(pHelper, 0, pGroup);
+  pBlockData->delimiter = TSDB_FILE_DELIMITER;
+  pBlockData->uid = TABLE_UID(pTable);
+  pBlockData->numOfCols = nColsNotAllNull;
+
+  taosCalcChecksumAppend(0, (uint8_t *)pBlockData, tsize);
+  tsdbUpdateDFileMagic(pDFile, POINTER_SHIFT(pBlockData, tsize - sizeof(TSCKSUM)));
 
-  pthread_rwlock_wrlock(&(pFileH->fhlock));
+  // Write the whole block to file
+  if (tsdbAppendDFile(pDFile, (void *)pBlockData, lsize, &offset) < lsize) {
+    return -1;
+  }
 
-  (void)taosRename(helperNewHeadF(pHelper)->fname, helperHeadF(pHelper)->fname);
-  pGroup->files[TSDB_FILE_TYPE_HEAD].info = helperNewHeadF(pHelper)->info;
+  // Update pBlock membership vairables
+  pBlock->last = isLast;
+  pBlock->offset = offset;
+  pBlock->algorithm = pCfg->compression;
+  pBlock->numOfRows = rowsToWrite;
+  pBlock->len = lsize;
+  pBlock->keyLen = keyLen;
+  pBlock->numOfSubBlocks = isSuper ? 1 : 0;
+  pBlock->numOfCols = nColsNotAllNull;
+  pBlock->keyFirst = dataColsKeyFirst(pDataCols);
+  pBlock->keyLast = dataColsKeyLast(pDataCols);
+
+  tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64
+            " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64,
+            REPO_ID(pRepo), TABLE_TID(pTable), TSDB_FILE_FULL_NAME(pDFile), offset, rowsToWrite, pBlock->len,
+            pBlock->numOfCols, pBlock->keyFirst, pBlock->keyLast);
 
-  if (newLast) {
-    (void)taosRename(helperNewLastF(pHelper)->fname, helperLastF(pHelper)->fname);
-    pGroup->files[TSDB_FILE_TYPE_LAST].info = helperNewLastF(pHelper)->info;
+  return 0;
+}
+
+static int tsdbWriteBlockInfo(SCommitH *pCommih) {
+  SDFile *    pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih);
+  SBlockIdx   blkIdx;
+  STable *    pTable = TSDB_COMMIT_TABLE(pCommih);
+  SBlock *    pBlock;
+  size_t      nSupBlocks;
+  size_t      nSubBlocks;
+  uint32_t    tlen;
+  SBlockInfo *pBlkInfo;
+  int64_t     offset;
+
+  nSupBlocks = taosArrayGetSize(pCommih->aSupBlk);
+  nSubBlocks = taosArrayGetSize(pCommih->aSubBlk);
+
+  if (nSupBlocks <= 0) {
+    // No data (data all deleted)
+    return 0;
+  }
+
+  tlen = (uint32_t)(sizeof(SBlockInfo) + sizeof(SBlock) * (nSupBlocks + nSubBlocks) + sizeof(TSCKSUM));
+
+  // Write SBlockInfo part
+  if (tsdbMakeRoom((void **)(&(TSDB_COMMIT_BUF(pCommih))), tlen) < 0) return -1;
+  pBlkInfo = TSDB_COMMIT_BUF(pCommih);
+
+  pBlkInfo->delimiter = TSDB_FILE_DELIMITER;
+  pBlkInfo->tid = TABLE_TID(pTable);
+  pBlkInfo->uid = TABLE_UID(pTable);
+
+  memcpy((void *)(pBlkInfo->blocks), taosArrayGet(pCommih->aSupBlk, 0), nSupBlocks * sizeof(SBlock));
+  if (nSubBlocks > 0) {
+    memcpy((void *)(pBlkInfo->blocks + nSupBlocks), taosArrayGet(pCommih->aSubBlk, 0), nSubBlocks * sizeof(SBlock));
+
+    for (int i = 0; i < nSupBlocks; i++) {
+      pBlock = pBlkInfo->blocks + i;
+
+      if (pBlock->numOfSubBlocks > 1) {
+        pBlock->offset += (sizeof(SBlockInfo) + sizeof(SBlock) * nSupBlocks);
+      }
+    }
+  }
+
+  taosCalcChecksumAppend(0, (uint8_t *)pBlkInfo, tlen);
+
+  if (tsdbAppendDFile(pHeadf, TSDB_COMMIT_BUF(pCommih), tlen, &offset) < 0) {
+    return -1;
+  }
+
+  tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(pBlkInfo, tlen - sizeof(TSCKSUM)));
+
+  // Set blkIdx
+  pBlock = taosArrayGet(pCommih->aSupBlk, nSupBlocks - 1);
+
+  blkIdx.tid = TABLE_TID(pTable);
+  blkIdx.uid = TABLE_UID(pTable);
+  blkIdx.hasLast = pBlock->last ? 1 : 0;
+  blkIdx.maxKey = pBlock->keyLast;
+  blkIdx.numOfBlocks = (uint32_t)nSupBlocks;
+  blkIdx.len = tlen;
+  blkIdx.offset = (uint32_t)offset;
+
+  ASSERT(blkIdx.numOfBlocks > 0);
+
+  if (taosArrayPush(pCommih->aBlkIdx, (void *)(&blkIdx)) == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  return 0;
+}
+
+static int tsdbWriteBlockIdx(SCommitH *pCommih) {
+  SBlockIdx *pBlkIdx;
+  SDFile *   pHeadf = TSDB_COMMIT_HEAD_FILE(pCommih);
+  size_t     nidx = taosArrayGetSize(pCommih->aBlkIdx);
+  int        tlen = 0, size;
+  int64_t    offset;
+
+  if (nidx <= 0) {
+    // All data are deleted
+    return 0;
+  }
+
+  for (size_t i = 0; i < nidx; i++) {
+    pBlkIdx = (SBlockIdx *)taosArrayGet(pCommih->aBlkIdx, i);
+
+    size = tsdbEncodeSBlockIdx(NULL, pBlkIdx);
+    if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommih)), tlen + size) < 0) return -1;
+
+    void *ptr = POINTER_SHIFT(TSDB_COMMIT_BUF(pCommih), tlen);
+    tsdbEncodeSBlockIdx(&ptr, pBlkIdx);
+
+    tlen += size;
+  }
+
+  tlen += sizeof(TSCKSUM);
+  if (tsdbMakeRoom((void **)(&TSDB_COMMIT_BUF(pCommih)), tlen) < 0) return -1;
+  taosCalcChecksumAppend(0, (uint8_t *)TSDB_COMMIT_BUF(pCommih), tlen);
+
+  if (tsdbAppendDFile(pHeadf, TSDB_COMMIT_BUF(pCommih), tlen, &offset) < tlen) {
+    tsdbError("vgId:%d failed to write block index part to file %s since %s", TSDB_COMMIT_REPO_ID(pCommih),
+              TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno));
+    return -1;
+  }
+
+  tsdbUpdateDFileMagic(pHeadf, POINTER_SHIFT(TSDB_COMMIT_BUF(pCommih), tlen - sizeof(TSCKSUM)));
+  pHeadf->info.offset = (uint32_t)offset;
+  pHeadf->info.len = tlen;
+
+  return 0;
+}
+
+static int tsdbCommitMemData(SCommitH *pCommith, SCommitIter *pIter, TSKEY keyLimit, bool toData) {
+  STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith);
+  STsdbCfg * pCfg = REPO_CFG(pRepo);
+  SMergeInfo mInfo;
+  int32_t    defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith);
+  SDFile *   pDFile;
+  bool       isLast;
+  SBlock     block;
+
+  while (true) {
+    tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, defaultRows, pCommith->pDataCols, NULL, 0,
+                          pCfg->update, &mInfo);
+
+    if (pCommith->pDataCols->numOfRows <= 0) break;
+
+    if (toData || pCommith->pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) {
+      pDFile = TSDB_COMMIT_DATA_FILE(pCommith);
+      isLast = false;
+    } else {
+      pDFile = TSDB_COMMIT_LAST_FILE(pCommith);
+      isLast = true;
+    }
+
+    if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1;
+
+    if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static int tsdbMergeMemData(SCommitH *pCommith, SCommitIter *pIter, int bidx) {
+  STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith);
+  STsdbCfg * pCfg = REPO_CFG(pRepo);
+  int        nBlocks = pCommith->readh.pBlkIdx->numOfBlocks;
+  SBlock *   pBlock = pCommith->readh.pBlkInfo->blocks + bidx;
+  TSKEY      keyLimit;
+  int16_t    colId = 0;
+  SMergeInfo mInfo;
+  SBlock     subBlocks[TSDB_MAX_SUBBLOCKS];
+  SBlock     block, supBlock;
+  SDFile *   pDFile;
+
+  if (bidx == nBlocks - 1) {
+    keyLimit = pCommith->maxKey;
   } else {
-    pGroup->files[TSDB_FILE_TYPE_LAST].info = helperLastF(pHelper)->info;
+    keyLimit = pBlock[1].keyFirst - 1;
   }
 
-  pGroup->files[TSDB_FILE_TYPE_DATA].info = helperDataF(pHelper)->info;
+  SSkipListIterator titer = *(pIter->pIter);
+  if (tsdbLoadBlockDataCols(&(pCommith->readh), pBlock, NULL, &colId, 1) < 0) return -1;
+
+  tsdbLoadDataFromCache(pIter->pTable, &titer, keyLimit, INT32_MAX, NULL, pCommith->readh.pDCols[0]->cols[0].pData,
+                        pCommith->readh.pDCols[0]->numOfRows, pCfg->update, &mInfo);
+
+  if (mInfo.nOperations == 0) {
+    // no new data to insert (all updates denied)
+    if (tsdbMoveBlock(pCommith, bidx) < 0) {
+      return -1;
+    }
+    *(pIter->pIter) = titer;
+  } else if (pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed == 0) {
+    // Ignore the block
+    ASSERT(0);
+    *(pIter->pIter) = titer;
+  } else if (tsdbCanAddSubBlock(pCommith, pBlock, &mInfo)) {
+    // Add a sub-block
+    tsdbLoadDataFromCache(pIter->pTable, pIter->pIter, keyLimit, INT32_MAX, pCommith->pDataCols,
+                          pCommith->readh.pDCols[0]->cols[0].pData, pCommith->readh.pDCols[0]->numOfRows, pCfg->update,
+                          &mInfo);
+    if (pBlock->last) {
+      pDFile = TSDB_COMMIT_LAST_FILE(pCommith);
+    } else {
+      pDFile = TSDB_COMMIT_DATA_FILE(pCommith);
+    }
+
+    if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, pBlock->last, false) < 0) return -1;
 
-  pthread_rwlock_unlock(&(pFileH->fhlock));
+    if (pBlock->numOfSubBlocks == 1) {
+      subBlocks[0] = *pBlock;
+      subBlocks[0].numOfSubBlocks = 0;
+    } else {
+      memcpy(subBlocks, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset),
+             sizeof(SBlock) * pBlock->numOfSubBlocks);
+    }
+    subBlocks[pBlock->numOfSubBlocks] = block;
+    supBlock = *pBlock;
+    supBlock.keyFirst = mInfo.keyFirst;
+    supBlock.keyLast = mInfo.keyLast;
+    supBlock.numOfSubBlocks++;
+    supBlock.numOfRows = pBlock->numOfRows + mInfo.rowsInserted - mInfo.rowsDeleteSucceed;
+    supBlock.offset = taosArrayGetSize(pCommith->aSubBlk) * sizeof(SBlock);
+
+    if (tsdbCommitAddBlock(pCommith, &supBlock, subBlocks, supBlock.numOfSubBlocks) < 0) return -1;
+  } else {
+    if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1;
+    if (tsdbMergeBlockData(pCommith, pIter, pCommith->readh.pDCols[0], keyLimit, bidx == (nBlocks - 1)) < 0) return -1;
+  }
 
   return 0;
+}
 
-_err:
-  tfree(dataDir);
-  tsdbCloseHelperFile(pHelper, 1, pGroup);
-  return -1;
+static int tsdbMoveBlock(SCommitH *pCommith, int bidx) {
+  SBlock *pBlock = pCommith->readh.pBlkInfo->blocks + bidx;
+  SDFile *pDFile;
+  SBlock  block;
+  bool    isSameFile;
+
+  ASSERT(pBlock->numOfSubBlocks > 0);
+
+  if (pBlock->last) {
+    pDFile = TSDB_COMMIT_LAST_FILE(pCommith);
+    isSameFile = pCommith->isLFileSame;
+  } else {
+    pDFile = TSDB_COMMIT_DATA_FILE(pCommith);
+    isSameFile = pCommith->isDFileSame;
+  }
+
+  if (isSameFile) {
+    if (pBlock->numOfSubBlocks == 1) {
+      if (tsdbCommitAddBlock(pCommith, pBlock, NULL, 0) < 0) {
+        return -1;
+      }
+    } else {
+      block = *pBlock;
+      block.offset = sizeof(SBlock) * taosArrayGetSize(pCommith->aSubBlk);
+
+      if (tsdbCommitAddBlock(pCommith, &block, POINTER_SHIFT(pCommith->readh.pBlkInfo, pBlock->offset),
+                             pBlock->numOfSubBlocks) < 0) {
+        return -1;
+      }
+    }
+  } else {
+    if (tsdbLoadBlockData(&(pCommith->readh), pBlock, NULL) < 0) return -1;
+    if (tsdbWriteBlock(pCommith, pDFile, pCommith->readh.pDCols[0], &block, pBlock->last, true) < 0) return -1;
+    if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1;
+  }
+
+  return 0;
 }
 
-static SCommitIter *tsdbCreateCommitIters(STsdbRepo *pRepo) {
-  SMemTable *pMem = pRepo->imem;
-  STsdbMeta *pMeta = pRepo->tsdbMeta;
+static int tsdbCommitAddBlock(SCommitH *pCommith, const SBlock *pSupBlock, const SBlock *pSubBlocks, int nSubBlocks) {
+  if (taosArrayPush(pCommith->aSupBlk, pSupBlock) < 0) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
 
-  SCommitIter *iters = (SCommitIter *)calloc(pMem->maxTables, sizeof(SCommitIter));
-  if (iters == NULL) {
+  if (pSubBlocks && taosArrayPushBatch(pCommith->aSubBlk, pSubBlocks, nSubBlocks) < 0) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return NULL;
+    return -1;
   }
 
-  if (tsdbRLockRepoMeta(pRepo) < 0) goto _err;
+  return 0;
+}
 
-  // reference all tables
-  for (int i = 0; i < pMem->maxTables; i++) {
-    if (pMeta->tables[i] != NULL) {
-      tsdbRefTable(pMeta->tables[i]);
-      iters[i].pTable = pMeta->tables[i];
+static int tsdbMergeBlockData(SCommitH *pCommith, SCommitIter *pIter, SDataCols *pDataCols, TSKEY keyLimit, bool isLastOneBlock) {
+  STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith);
+  STsdbCfg * pCfg = REPO_CFG(pRepo);
+  SBlock     block;
+  SDFile *   pDFile;
+  bool       isLast;
+  int32_t    defaultRows = TSDB_COMMIT_DEFAULT_ROWS(pCommith);
+
+  int biter = 0;
+  while (true) {
+    tsdbLoadAndMergeFromCache(pCommith->readh.pDCols[0], &biter, pIter, pCommith->pDataCols, keyLimit, defaultRows,
+                              pCfg->update);
+
+    if (pCommith->pDataCols->numOfRows == 0) break;
+
+    if (isLastOneBlock) {
+      if (pCommith->pDataCols->numOfRows < pCfg->minRowsPerFileBlock) {
+        pDFile = TSDB_COMMIT_LAST_FILE(pCommith);
+        isLast = true;
+      } else {
+        pDFile = TSDB_COMMIT_DATA_FILE(pCommith);
+        isLast = false;
+      }
+    } else {
+      pDFile = TSDB_COMMIT_DATA_FILE(pCommith);
+      isLast = false;
     }
+
+    if (tsdbWriteBlock(pCommith, pDFile, pCommith->pDataCols, &block, isLast, true) < 0) return -1;
+    if (tsdbCommitAddBlock(pCommith, &block, NULL, 0) < 0) return -1;
   }
 
-  if (tsdbUnlockRepoMeta(pRepo) < 0) goto _err;
+  return 0;
+}
 
-  for (int i = 0; i < pMem->maxTables; i++) {
-    if ((iters[i].pTable != NULL) && (pMem->tData[i] != NULL) && (TABLE_UID(iters[i].pTable) == pMem->tData[i]->uid)) {
-      if ((iters[i].pIter = tSkipListCreateIter(pMem->tData[i]->pData)) == NULL) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        goto _err;
+static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
+                                     TSKEY maxKey, int maxRows, int8_t update) {
+  TSKEY     key1 = INT64_MAX;
+  TSKEY     key2 = INT64_MAX;
+  STSchema *pSchema = NULL;
+
+  ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey);
+  tdResetDataCols(pTarget);
+
+  while (true) {
+    key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
+    bool isRowDel = false;
+    SDataRow row = tsdbNextIterRow(pCommitIter->pIter);
+    if (row == NULL || dataRowKey(row) > maxKey) {
+      key2 = INT64_MAX;
+    } else {
+      key2 = dataRowKey(row);
+      isRowDel = dataRowDeleted(row);
+    }
+
+    if (key1 == INT64_MAX && key2 == INT64_MAX) break;
+
+    if (key1 < key2) {
+      for (int i = 0; i < pDataCols->numOfCols; i++) {
+        dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
+                         pTarget->maxPoints);
+      }
+
+      pTarget->numOfRows++;
+      (*iter)++;
+    } else if (key1 > key2) {
+      if (!isRowDel) {
+        if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
+          pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
+          ASSERT(pSchema != NULL);
+        }
+
+        tdAppendDataRowToDataCol(row, pSchema, pTarget);
       }
 
-      tSkipListIterNext(iters[i].pIter);
+      tSkipListIterNext(pCommitIter->pIter);
+    } else {
+      if (update) {
+        if (!isRowDel) {
+          if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
+            pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
+            ASSERT(pSchema != NULL);
+          }
+
+          tdAppendDataRowToDataCol(row, pSchema, pTarget);
+        }
+      } else {
+        ASSERT(!isRowDel);
+
+        for (int i = 0; i < pDataCols->numOfCols; i++) {
+          dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
+                           pTarget->maxPoints);
+        }
+
+        pTarget->numOfRows++;
+      }
+      (*iter)++;
+      tSkipListIterNext(pCommitIter->pIter);
     }
+
+    if (pTarget->numOfRows >= maxRows) break;
   }
+}
 
-  return iters;
+static void tsdbResetCommitFile(SCommitH *pCommith) {
+  pCommith->isRFileSet = false;
+  pCommith->isDFileSame = false;
+  pCommith->isLFileSame = false;
+  taosArrayClear(pCommith->aBlkIdx);
+}
 
-_err:
-  tsdbDestroyCommitIters(iters, pMem->maxTables);
-  return NULL;
+static void tsdbResetCommitTable(SCommitH *pCommith) {
+  tdResetDataCols(pCommith->pDataCols);
+  taosArrayClear(pCommith->aSubBlk);
+  taosArrayClear(pCommith->aSupBlk);
+  pCommith->pTable = NULL;
 }
 
-static void tsdbDestroyCommitIters(SCommitIter *iters, int maxTables) {
-  if (iters == NULL) return;
+static int tsdbSetAndOpenCommitFile(SCommitH *pCommith, SDFileSet *pSet, int fid) {
+  SDiskID    did;
+  STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith);
+  SDFileSet *pWSet = TSDB_COMMIT_WRITE_FSET(pCommith);
+
+  tfsAllocDisk(tsdbGetFidLevel(fid, &(pCommith->rtn)), &(did.level), &(did.id));
+  if (did.level == TFS_UNDECIDED_LEVEL) {
+    terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
+    return -1;
+  }
 
-  for (int i = 1; i < maxTables; i++) {
-    if (iters[i].pTable != NULL) {
-      tsdbUnRefTable(iters[i].pTable);
-      tSkipListDestroyIter(iters[i].pIter);
+  // Open read FSET
+  if (pSet) {
+    if (tsdbSetAndOpenReadFSet(&(pCommith->readh), pSet) < 0) {
+      return -1;
+    }
+
+    pCommith->isRFileSet = true;
+
+    if (tsdbLoadBlockIdx(&(pCommith->readh)) < 0) {
+      tsdbCloseAndUnsetFSet(&(pCommith->readh));
+      return -1;
+    }
+  } else {
+    pCommith->isRFileSet = false;
+  }
+
+  // Set and open commit FSET
+  if (pSet == NULL || did.level > TSDB_FSET_LEVEL(pSet)) {
+    // Create a new FSET to write data
+    tsdbInitDFileSet(pWSet, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)));
+
+    if (tsdbCreateDFileSet(pWSet, true) < 0) {
+      if (pCommith->isRFileSet) {
+        tsdbCloseAndUnsetFSet(&(pCommith->readh));
+      }
+      return -1;
+    }
+
+    pCommith->isDFileSame = false;
+    pCommith->isLFileSame = false;
+  } else {
+    did.level = TSDB_FSET_LEVEL(pSet);
+    did.id = TSDB_FSET_ID(pSet);
+
+    pCommith->wSet.fid = fid;
+    pCommith->wSet.state = 0;
+
+    // TSDB_FILE_HEAD
+    SDFile *pWHeadf = TSDB_COMMIT_HEAD_FILE(pCommith);
+    tsdbInitDFile(pWHeadf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_HEAD);
+    if (tsdbCreateDFile(pWHeadf, true) < 0) {
+      if (pCommith->isRFileSet) {
+        tsdbCloseAndUnsetFSet(&(pCommith->readh));
+        return -1;
+      }
+    }
+
+    // TSDB_FILE_DATA
+    SDFile *pRDataf = TSDB_READ_DATA_FILE(&(pCommith->readh));
+    SDFile *pWDataf = TSDB_COMMIT_DATA_FILE(pCommith);
+    tsdbInitDFileEx(pWDataf, pRDataf);
+    if (tsdbOpenDFile(pWDataf, O_WRONLY) < 0) {
+      tsdbCloseDFile(pWHeadf);
+      tsdbRemoveDFile(pWHeadf);
+      if (pCommith->isRFileSet) {
+        tsdbCloseAndUnsetFSet(&(pCommith->readh));
+        return -1;
+      }
+    }
+    pCommith->isDFileSame = true;
+
+    // TSDB_FILE_LAST
+    SDFile *pRLastf = TSDB_READ_LAST_FILE(&(pCommith->readh));
+    SDFile *pWLastf = TSDB_COMMIT_LAST_FILE(pCommith);
+    if (pRLastf->info.size < 32 * 1024) {
+      tsdbInitDFileEx(pWLastf, pRLastf);
+      pCommith->isLFileSame = true;
+
+      if (tsdbOpenDFile(pWLastf, O_WRONLY) < 0) {
+        tsdbCloseDFileSet(pWSet);
+        tsdbRemoveDFile(pWHeadf);
+        if (pCommith->isRFileSet) {
+          tsdbCloseAndUnsetFSet(&(pCommith->readh));
+          return -1;
+        }
+      }
+    } else {
+      tsdbInitDFile(pWLastf, did, REPO_ID(pRepo), fid, FS_TXN_VERSION(REPO_FS(pRepo)), TSDB_FILE_LAST);
+      pCommith->isLFileSame = false;
+
+      if (tsdbCreateDFile(pWLastf, true) < 0) {
+        tsdbCloseDFileSet(pWSet);
+        tsdbRemoveDFile(pWHeadf);
+        if (pCommith->isRFileSet) {
+          tsdbCloseAndUnsetFSet(&(pCommith->readh));
+          return -1;
+        }
+      }
+    }
+  }
+
+  return 0;
+}
+
+static void tsdbCloseCommitFile(SCommitH *pCommith, bool hasError) {
+  if (pCommith->isRFileSet) {
+    tsdbCloseAndUnsetFSet(&(pCommith->readh));
+  }
+
+  if (!hasError) {
+    TSDB_FSET_FSYNC(TSDB_COMMIT_WRITE_FSET(pCommith));
+  }
+  tsdbCloseDFileSet(TSDB_COMMIT_WRITE_FSET(pCommith));
+}
+
+static bool tsdbCanAddSubBlock(SCommitH *pCommith, SBlock *pBlock, SMergeInfo *pInfo) {
+  STsdbRepo *pRepo = TSDB_COMMIT_REPO(pCommith);
+  STsdbCfg * pCfg = REPO_CFG(pRepo);
+  int        mergeRows = pBlock->numOfRows + pInfo->rowsInserted - pInfo->rowsDeleteSucceed;
+
+  ASSERT(mergeRows > 0);
+
+  if (pBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pInfo->nOperations <= pCfg->maxRowsPerFileBlock) {
+    if (pBlock->last) {
+      if (pCommith->isLFileSame && mergeRows < pCfg->minRowsPerFileBlock) return true;
+    } else {
+      if (mergeRows < pCfg->maxRowsPerFileBlock) return true;
     }
   }
 
-  free(iters);
+  return false;
 }
+
+static int tsdbApplyRtn(STsdbRepo *pRepo) {
+  SRtn       rtn;
+  SFSIter    fsiter;
+  STsdbFS *  pfs = REPO_FS(pRepo);
+  SDFileSet *pSet;
+
+  // Get retentioni snapshot
+  tsdbGetRtnSnap(pRepo, &rtn);
+
+  tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
+  while ((pSet = tsdbFSIterNext(&fsiter))) {
+    if (pSet->fid < rtn.minFid) continue;
+
+    if (tsdbApplyRtnOnFSet(pRepo, pSet, &rtn) < 0) {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static int tsdbApplyRtnOnFSet(STsdbRepo *pRepo, SDFileSet *pSet, SRtn *pRtn) {
+  SDiskID   did;
+  SDFileSet nSet;
+  STsdbFS * pfs = REPO_FS(pRepo);
+
+  ASSERT(pSet->fid >= pRtn->minFid);
+
+  tfsAllocDisk(tsdbGetFidLevel(pSet->fid, pRtn), &(did.level), &(did.id));
+  if (did.level == TFS_UNDECIDED_LEVEL) {
+    terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
+    return -1;
+  }
+
+  if (did.level > TSDB_FSET_LEVEL(pSet)) {
+    // Need to move the FSET to higher level
+    tsdbInitDFileSet(&nSet, did, REPO_ID(pRepo), pSet->fid, FS_TXN_VERSION(pfs));
+
+    if (tsdbCopyDFileSet(pSet, &nSet) < 0) {
+      return -1;
+    }
+
+    if (tsdbUpdateDFileSet(pfs, &nSet) < 0) {
+      return -1;
+    }
+  } else {
+    // On a correct level
+    if (tsdbUpdateDFileSet(pfs, pSet) < 0) {
+      return -1;
+    }
+  }
+
+  return 0;
+}
\ No newline at end of file

src/tsdb/src/tsdbCommitQueue.c

@@ -13,11 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
-#include "tglobal.h"
-#include "tlist.h"
-#include "tref.h"
-#include "tsdbMain.h"
+#include "tsdbint.h"
 
 typedef struct {
   bool            stop;
@@ -35,7 +31,7 @@ typedef struct {
 
 static void *tsdbLoopCommit(void *arg);
 
-SCommitQueue tsCommitQueue = {0};
+static SCommitQueue tsCommitQueue = {0};
 
 int tsdbInitCommitQueue() {
   int nthreads = tsNumOfCommitThreads;

src/tsdb/src/tsdbFS.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tsdbint.h"
+#include <regex.h>
+
+typedef enum { TSDB_TXN_TEMP_FILE = 0, TSDB_TXN_CURR_FILE } TSDB_TXN_FILE_T;
+static const char *tsdbTxnFname[] = {"current.t", "current"};
+#define TSDB_MAX_FSETS(keep, days) ((keep) / (days) + 3)
+
+static int  tsdbComparFidFSet(const void *arg1, const void *arg2);
+static void tsdbResetFSStatus(SFSStatus *pStatus);
+static int  tsdbSaveFSStatus(SFSStatus *pStatus, int vid);
+static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo);
+static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]);
+static int  tsdbOpenFSFromCurrent(STsdbRepo *pRepo);
+static int  tsdbScanAndTryFixFS(STsdbRepo *pRepo);
+static int  tsdbScanRootDir(STsdbRepo *pRepo);
+static int  tsdbScanDataDir(STsdbRepo *pRepo);
+static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf);
+static int  tsdbRestoreCurrent(STsdbRepo *pRepo);
+static int  tsdbComparTFILE(const void *arg1, const void *arg2);
+
+// ================== CURRENT file header info
+static int tsdbEncodeFSHeader(void **buf, SFSHeader *pHeader) {
+  int tlen = 0;
+
+  tlen += taosEncodeFixedU32(buf, pHeader->version);
+  tlen += taosEncodeFixedU32(buf, pHeader->len);
+
+  return tlen;
+}
+
+static void *tsdbDecodeFSHeader(void *buf, SFSHeader *pHeader) {
+  buf = taosDecodeFixedU32(buf, &(pHeader->version));
+  buf = taosDecodeFixedU32(buf, &(pHeader->len));
+
+  return buf;
+}
+
+// ================== STsdbFSMeta
+static int tsdbEncodeFSMeta(void **buf, STsdbFSMeta *pMeta) {
+  int tlen = 0;
+
+  tlen += taosEncodeFixedU32(buf, pMeta->version);
+  tlen += taosEncodeFixedI64(buf, pMeta->totalPoints);
+  tlen += taosEncodeFixedI64(buf, pMeta->totalStorage);
+
+  return tlen;
+}
+
+static void *tsdbDecodeFSMeta(void *buf, STsdbFSMeta *pMeta) {
+  buf = taosDecodeFixedU32(buf, &(pMeta->version));
+  buf = taosDecodeFixedI64(buf, &(pMeta->totalPoints));
+  buf = taosDecodeFixedI64(buf, &(pMeta->totalStorage));
+
+  return buf;
+}
+
+// ================== SFSStatus
+static int tsdbEncodeDFileSetArray(void **buf, SArray *pArray) {
+  int      tlen = 0;
+  uint64_t nset = taosArrayGetSize(pArray);
+
+  tlen += taosEncodeFixedU64(buf, nset);
+  for (size_t i = 0; i < nset; i++) {
+    SDFileSet *pSet = taosArrayGet(pArray, i);
+
+    tlen += tsdbEncodeDFileSet(buf, pSet);
+  }
+
+  return tlen;
+}
+
+static void *tsdbDecodeDFileSetArray(void *buf, SArray *pArray) {
+  uint64_t  nset;
+  SDFileSet dset;
+
+  taosArrayClear(pArray);
+
+  buf = taosDecodeFixedU64(buf, &nset);
+  for (size_t i = 0; i < nset; i++) {
+    buf = tsdbDecodeDFileSet(buf, &dset);
+    taosArrayPush(pArray, (void *)(&dset));
+  }
+  return buf;
+}
+
+static int tsdbEncodeFSStatus(void **buf, SFSStatus *pStatus) {
+  ASSERT(pStatus->pmf);
+
+  int tlen = 0;
+
+  tlen += tsdbEncodeSMFile(buf, pStatus->pmf);
+  tlen += tsdbEncodeDFileSetArray(buf, pStatus->df);
+
+  return tlen;
+}
+
+static void *tsdbDecodeFSStatus(void *buf, SFSStatus *pStatus) {
+  tsdbResetFSStatus(pStatus);
+
+  pStatus->pmf = &(pStatus->mf);
+
+  buf = tsdbDecodeSMFile(buf, pStatus->pmf);
+  buf = tsdbDecodeDFileSetArray(buf, pStatus->df);
+
+  return buf;
+}
+
+static SFSStatus *tsdbNewFSStatus(int maxFSet) {
+  SFSStatus *pStatus = (SFSStatus *)calloc(1, sizeof(*pStatus));
+  if (pStatus == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  TSDB_FILE_SET_CLOSED(&(pStatus->mf));
+
+  pStatus->df = taosArrayInit(maxFSet, sizeof(SDFileSet));
+  if (pStatus->df == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    free(pStatus);
+    return NULL;
+  }
+
+  return pStatus;
+}
+
+static SFSStatus *tsdbFreeFSStatus(SFSStatus *pStatus) {
+  if (pStatus) {
+    pStatus->df = taosArrayDestroy(pStatus->df);
+    free(pStatus);
+  }
+
+  return NULL;
+}
+
+static void tsdbResetFSStatus(SFSStatus *pStatus) {
+  if (pStatus == NULL) {
+    return;
+  }
+
+  TSDB_FILE_SET_CLOSED(&(pStatus->mf));
+
+  pStatus->pmf = NULL;
+  taosArrayClear(pStatus->df);
+}
+
+static void tsdbSetStatusMFile(SFSStatus *pStatus, const SMFile *pMFile) {
+  ASSERT(pStatus->pmf == NULL);
+
+  pStatus->pmf = &(pStatus->mf);
+  tsdbInitMFileEx(pStatus->pmf, (SMFile *)pMFile);
+}
+
+static int tsdbAddDFileSetToStatus(SFSStatus *pStatus, const SDFileSet *pSet) {
+  if (taosArrayPush(pStatus->df, (void *)pSet) == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  TSDB_FSET_SET_CLOSED(((SDFileSet *)taosArrayGetLast(pStatus->df)));
+
+  return 0;
+}
+
+// ================== STsdbFS
+STsdbFS *tsdbNewFS(STsdbCfg *pCfg) {
+  int      keep = pCfg->keep;
+  int      days = pCfg->daysPerFile;
+  int      maxFSet = TSDB_MAX_FSETS(keep, days);
+  STsdbFS *pfs;
+
+  pfs = (STsdbFS *)calloc(1, sizeof(*pfs));
+  if (pfs == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  int code = pthread_rwlock_init(&(pfs->lock), NULL);
+  if (code) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    free(pfs);
+    return NULL;
+  }
+
+  pfs->cstatus = tsdbNewFSStatus(maxFSet);
+  if (pfs->cstatus == NULL) {
+    tsdbFreeFS(pfs);
+    return NULL;
+  }
+
+  pfs->metaCache = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK);
+  if (pfs->metaCache == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbFreeFS(pfs);
+    return NULL;
+  }
+
+  pfs->nstatus = tsdbNewFSStatus(maxFSet);
+  if (pfs->nstatus == NULL) {
+    tsdbFreeFS(pfs);
+    return NULL;
+  }
+
+  return pfs;
+}
+
+void *tsdbFreeFS(STsdbFS *pfs) {
+  if (pfs) {
+    pfs->nstatus = tsdbFreeFSStatus(pfs->nstatus);
+    taosHashCleanup(pfs->metaCache);
+    pfs->metaCache = NULL;
+    pfs->cstatus = tsdbFreeFSStatus(pfs->cstatus);
+    pthread_rwlock_destroy(&(pfs->lock));
+  }
+
+  return NULL;
+}
+
+int tsdbOpenFS(STsdbRepo *pRepo) {
+  STsdbFS *pfs = REPO_FS(pRepo);
+  char     current[TSDB_FILENAME_LEN] = "\0";
+
+  ASSERT(pfs != NULL);
+
+  tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current);
+
+  if (access(current, F_OK) == 0) {
+    if (tsdbOpenFSFromCurrent(pRepo) < 0) {
+      tsdbError("vgId:%d failed to open FS since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+  } else {
+    if (tsdbRestoreCurrent(pRepo) < 0) {
+      tsdbError("vgId:%d failed to restore current file since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+  }
+
+  if (tsdbScanAndTryFixFS(pRepo) < 0) {
+    tsdbError("vgId:%d failed to scan and fix FS since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  // Load meta cache if has meta file
+  if ((!(pRepo->state & TSDB_STATE_BAD_META)) && tsdbLoadMetaCache(pRepo, true) < 0) {
+    tsdbError("vgId:%d failed to open FS while loading meta cache since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  return 0;
+}
+
+void tsdbCloseFS(STsdbRepo *pRepo) {
+  // Do nothing
+}
+
+// Start a new transaction to modify the file system
+void tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd) {
+  STsdbFS *pfs = REPO_FS(pRepo);
+  ASSERT(pfs->intxn == false);
+
+  pfs->intxn = true;
+  tsdbResetFSStatus(pfs->nstatus);
+  pfs->nstatus->meta = pfs->cstatus->meta;
+  if (pfs->cstatus->pmf == NULL) {
+    pfs->nstatus->meta.version = 0;
+  } else {
+    pfs->nstatus->meta.version = pfs->cstatus->meta.version + 1;
+  }
+  pfs->nstatus->meta.totalPoints = pfs->cstatus->meta.totalPoints + pointsAdd;
+  pfs->nstatus->meta.totalStorage = pfs->cstatus->meta.totalStorage += storageAdd;
+}
+
+void tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta) { pfs->nstatus->meta = *pMeta; }
+
+int tsdbEndFSTxn(STsdbRepo *pRepo) {
+  STsdbFS *pfs = REPO_FS(pRepo);
+  ASSERT(FS_IN_TXN(pfs));
+  SFSStatus *pStatus;
+
+  // Write current file system snapshot
+  if (tsdbSaveFSStatus(pfs->nstatus, REPO_ID(pRepo)) < 0) {
+    tsdbEndFSTxnWithError(pfs);
+    return -1;
+  }
+
+  // Make new 
+  tsdbWLockFS(pfs);
+  pStatus = pfs->cstatus;
+  pfs->cstatus = pfs->nstatus;
+  pfs->nstatus = pStatus;
+  tsdbUnLockFS(pfs);
+
+  // Apply actual change to each file and SDFileSet
+  tsdbApplyFSTxnOnDisk(pfs->nstatus, pfs->cstatus);
+
+  pfs->intxn = false;
+  return 0;
+}
+
+int tsdbEndFSTxnWithError(STsdbFS *pfs) {
+  tsdbApplyFSTxnOnDisk(pfs->nstatus, pfs->cstatus);
+  // TODO: if mf change, reload pfs->metaCache
+  pfs->intxn = false;
+  return 0;
+}
+
+void tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile) { tsdbSetStatusMFile(pfs->nstatus, pMFile); }
+
+int tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet) { return tsdbAddDFileSetToStatus(pfs->nstatus, pSet); }
+
+static int tsdbSaveFSStatus(SFSStatus *pStatus, int vid) {
+  SFSHeader fsheader;
+  void *    pBuf = NULL;
+  void *    ptr;
+  char      hbuf[TSDB_FILE_HEAD_SIZE] = "\0";
+  char      tfname[TSDB_FILENAME_LEN] = "\0";
+  char      cfname[TSDB_FILENAME_LEN] = "\0";
+
+  tsdbGetTxnFname(vid, TSDB_TXN_TEMP_FILE, tfname);
+  tsdbGetTxnFname(vid, TSDB_TXN_CURR_FILE, cfname);
+
+  int fd = open(tfname, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+  if (fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  fsheader.version = TSDB_FS_VERSION;
+  if (pStatus->pmf == NULL) {
+    ASSERT(taosArrayGetSize(pStatus->df) == 0);
+    fsheader.len = 0;
+  } else {
+    fsheader.len = tsdbEncodeFSStatus(NULL, pStatus) + sizeof(TSCKSUM);
+  }
+
+  // Encode header part and write
+  ptr = hbuf;
+  tsdbEncodeFSHeader(&ptr, &fsheader);
+  tsdbEncodeFSMeta(&ptr, &(pStatus->meta));
+
+  taosCalcChecksumAppend(0, (uint8_t *)hbuf, TSDB_FILE_HEAD_SIZE);
+
+  if (taosWrite(fd, hbuf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    close(fd);
+    remove(tfname);
+    return -1;
+  }
+
+  // Encode file status and write to file
+  if (fsheader.len > 0) {
+    if (tsdbMakeRoom(&(pBuf), fsheader.len) < 0) {
+      close(fd);
+      remove(tfname);
+      return -1;
+    }
+
+    ptr = pBuf;
+    tsdbEncodeFSStatus(&ptr, pStatus);
+    taosCalcChecksumAppend(0, (uint8_t *)pBuf, fsheader.len);
+
+    if (taosWrite(fd, pBuf, fsheader.len) < fsheader.len) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      close(fd);
+      remove(tfname);
+      taosTZfree(pBuf);
+      return -1;
+    }
+  }
+
+  // fsync, close and rename
+  if (fsync(fd) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    close(fd);
+    remove(tfname);
+    taosTZfree(pBuf);
+    return -1;
+  }
+
+  (void)close(fd);
+  (void)rename(tfname, cfname);
+  taosTZfree(pBuf);
+
+  return 0;
+}
+
+static void tsdbApplyFSTxnOnDisk(SFSStatus *pFrom, SFSStatus *pTo) {
+  int        ifrom = 0;
+  int        ito = 0;
+  size_t     sizeFrom, sizeTo;
+  SDFileSet *pSetFrom;
+  SDFileSet *pSetTo;
+
+  sizeFrom = taosArrayGetSize(pFrom->df);
+  sizeTo = taosArrayGetSize(pTo->df);
+
+  // Apply meta file change
+  tsdbApplyMFileChange(pFrom->pmf, pTo->pmf);
+
+  // Apply SDFileSet change
+  if (ifrom >= sizeFrom) {
+    pSetFrom = NULL;
+  } else {
+    pSetFrom = taosArrayGet(pFrom->df, ifrom);
+  }
+
+  if (ito >= sizeTo) {
+    pSetTo = NULL;
+  } else {
+    pSetTo = taosArrayGet(pTo->df, ito);
+  }
+
+  while (true) {
+    if ((pSetTo == NULL) && (pSetFrom == NULL)) break;
+
+    if (pSetTo == NULL || (pSetFrom && pSetFrom->fid < pSetTo->fid)) {
+      tsdbApplyDFileSetChange(pSetFrom, NULL);
+
+      ifrom++;
+      if (ifrom >= sizeFrom) {
+        pSetFrom = NULL;
+      } else {
+        pSetFrom = taosArrayGet(pFrom->df, ifrom);
+      }
+    } else if (pSetFrom == NULL || pSetFrom->fid > pSetTo->fid) {
+      // Do nothing
+      ito++;
+      if (ito >= sizeTo) {
+        pSetTo = NULL;
+      } else {
+        pSetTo = taosArrayGet(pTo->df, ito);
+      }
+    } else {
+      tsdbApplyDFileSetChange(pSetFrom, pSetTo);
+
+      ifrom++;
+      if (ifrom >= sizeFrom) {
+        pSetFrom = NULL;
+      } else {
+        pSetFrom = taosArrayGet(pFrom->df, ifrom);
+      }
+
+      ito++;
+      if (ito >= sizeTo) {
+        pSetTo = NULL;
+      } else {
+        pSetTo = taosArrayGet(pTo->df, ito);
+      }
+    }
+  }
+}
+
+// ================== SFSIter
+// ASSUMPTIONS: the FS Should be read locked when calling these functions
+void tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction) {
+  pIter->pfs = pfs;
+  pIter->direction = direction;
+
+  size_t size = taosArrayGetSize(pfs->cstatus->df);
+
+  pIter->version = pfs->cstatus->meta.version;
+
+  if (size == 0) {
+    pIter->index = -1;
+    pIter->fid = TSDB_IVLD_FID;
+  } else {
+    if (direction == TSDB_FS_ITER_FORWARD) {
+      pIter->index = 0;
+    } else {
+      pIter->index = (int)(size - 1);
+    }
+
+    pIter->fid = ((SDFileSet *)taosArrayGet(pfs->cstatus->df, pIter->index))->fid;
+  }
+}
+
+void tsdbFSIterSeek(SFSIter *pIter, int fid) {
+  STsdbFS *pfs = pIter->pfs;
+  size_t   size = taosArrayGetSize(pfs->cstatus->df);
+
+  int flags;
+  if (pIter->direction == TSDB_FS_ITER_FORWARD) {
+    flags = TD_GE;
+  } else {
+    flags = TD_LE;
+  }
+
+  void *ptr = taosbsearch(&fid, pfs->cstatus->df->pData, size, sizeof(SDFileSet), tsdbComparFidFSet, flags);
+  if (ptr == NULL) {
+    pIter->index = -1;
+    pIter->fid = TSDB_IVLD_FID;
+  } else {
+    pIter->index = (int)(TARRAY_ELEM_IDX(pfs->cstatus->df, ptr));
+    pIter->fid = ((SDFileSet *)ptr)->fid;
+  }
+}
+
+SDFileSet *tsdbFSIterNext(SFSIter *pIter) {
+  STsdbFS *  pfs = pIter->pfs;
+  SDFileSet *pSet;
+
+  if (pIter->index < 0) {
+    ASSERT(pIter->fid == TSDB_IVLD_FID);
+    return NULL;
+  }
+
+  ASSERT(pIter->fid != TSDB_IVLD_FID);
+
+  if (pIter->version != pfs->cstatus->meta.version) {
+    pIter->version = pfs->cstatus->meta.version;
+    tsdbFSIterSeek(pIter, pIter->fid);
+  }
+
+  if (pIter->index < 0) {
+    return NULL;
+  }
+
+  pSet = (SDFileSet *)taosArrayGet(pfs->cstatus->df, pIter->index);
+  ASSERT(pSet->fid == pIter->fid);
+
+  if (pIter->direction == TSDB_FS_ITER_FORWARD) {
+    pIter->index++;
+    if (pIter->index >= taosArrayGetSize(pfs->cstatus->df)) {
+      pIter->index = -1;
+    }
+  } else {
+    pIter->index--;
+  }
+
+  if (pIter->index >= 0) {
+    pIter->fid = ((SDFileSet *)taosArrayGet(pfs->cstatus->df, pIter->index))->fid;
+  } else {
+    pIter->fid = TSDB_IVLD_FID;
+  }
+
+  return pSet;
+}
+
+static int tsdbComparFidFSet(const void *arg1, const void *arg2) {
+  int        fid = *(int *)arg1;
+  SDFileSet *pSet = (SDFileSet *)arg2;
+
+  if (fid < pSet->fid) {
+    return -1;
+  } else if (fid == pSet->fid) {
+    return 0;
+  } else {
+    return 1;
+  }
+}
+
+static void tsdbGetTxnFname(int repoid, TSDB_TXN_FILE_T ftype, char fname[]) {
+  snprintf(fname, TSDB_FILENAME_LEN, "%s/vnode/vnode%d/tsdb/%s", TFS_PRIMARY_PATH(), repoid, tsdbTxnFname[ftype]);
+}
+
+static int tsdbOpenFSFromCurrent(STsdbRepo *pRepo) {
+  STsdbFS * pfs = REPO_FS(pRepo);
+  int       fd = -1;
+  void *    buffer = NULL;
+  SFSHeader fsheader;
+  char      current[TSDB_FILENAME_LEN] = "\0";
+  void *    ptr;
+
+  tsdbGetTxnFname(REPO_ID(pRepo), TSDB_TXN_CURR_FILE, current);
+
+  // current file exists, try to recover
+  fd = open(current, O_RDONLY | O_BINARY);
+  if (fd < 0) {
+    tsdbError("vgId:%d failed to open file %s since %s", REPO_ID(pRepo), current, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  if (tsdbMakeRoom(&buffer, TSDB_FILE_HEAD_SIZE) < 0) {
+    goto _err;
+  }
+
+  int nread = (int)taosRead(fd, buffer, TSDB_FILE_HEAD_SIZE);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILENAME_LEN, current,
+              strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    goto _err;
+  }
+
+  if (nread < TSDB_FILE_HEAD_SIZE) {
+    tsdbError("vgId:%d failed to read header of file %s, read bytes:%d", REPO_ID(pRepo), current, nread);
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    goto _err;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)buffer, TSDB_FILE_HEAD_SIZE)) {
+    tsdbError("vgId:%d header of file %s failed checksum check", REPO_ID(pRepo), current);
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    goto _err;
+  }
+
+  SFSStatus *pStatus = pfs->cstatus;
+  ptr = buffer;
+  ptr = tsdbDecodeFSHeader(ptr, &fsheader);
+  ptr = tsdbDecodeFSMeta(ptr, &(pStatus->meta));
+
+  if (fsheader.version != TSDB_FS_VERSION) {
+    // TODO: handle file version change
+  }
+
+  if (fsheader.len > 0) {
+    if (tsdbMakeRoom(&buffer, fsheader.len) < 0) {
+      goto _err;
+    }
+
+    nread = (int)taosRead(fd, buffer, fsheader.len);
+    if (nread < 0) {
+      tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), current, strerror(errno));
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      goto _err;
+    }
+
+    if (nread < fsheader.len) {
+      tsdbError("vgId:%d failed to read %d bytes from file %s", REPO_ID(pRepo), fsheader.len, current);
+      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+      goto _err;
+    }
+
+    if (!taosCheckChecksumWhole((uint8_t *)buffer, fsheader.len)) {
+      tsdbError("vgId:%d file %s is corrupted since wrong checksum", REPO_ID(pRepo), current);
+      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+      goto _err;
+    }
+
+    ptr = buffer;
+    ptr = tsdbDecodeFSStatus(ptr, pStatus);
+  } else {
+    tsdbResetFSStatus(pStatus);
+  }
+
+  taosTZfree(buffer);
+  close(fd);
+
+  return 0;
+
+_err:
+  if (fd >= 0) {
+    close(fd);
+  }
+  taosTZfree(buffer);
+  return -1;
+}
+
+// Scan and try to fix incorrect files
+static int tsdbScanAndTryFixFS(STsdbRepo *pRepo) {
+  STsdbFS *  pfs = REPO_FS(pRepo);
+  SFSStatus *pStatus = pfs->cstatus;
+
+  if (tsdbScanAndTryFixMFile(pRepo) < 0) {
+    tsdbError("vgId:%d failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  size_t size = taosArrayGetSize(pStatus->df);
+
+  for (size_t i = 0; i < size; i++) {
+    SDFileSet *pSet = (SDFileSet *)taosArrayGet(pStatus->df, i);
+
+    if (tsdbScanAndTryFixDFileSet(pRepo, pSet) < 0) {
+      tsdbError("vgId:%d failed to fix MFile since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+  }
+
+  // remove those unused files
+  tsdbScanRootDir(pRepo);
+  tsdbScanDataDir(pRepo);
+  return 0;
+}
+
+int tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta) {
+  char      tbuf[128];
+  STsdbFS * pfs = REPO_FS(pRepo);
+  SMFile    mf;
+  SMFile *  pMFile = &mf;
+  void *    pBuf = NULL;
+  SKVRecord rInfo;
+  int64_t   maxBufSize = 0;
+  SMFInfo   minfo;
+
+  // No meta file, just return
+  if (pfs->cstatus->pmf == NULL) return 0;
+
+  mf = pfs->cstatus->mf;
+  // Load cache first
+  if (tsdbOpenMFile(pMFile, O_RDONLY) < 0) {
+    return -1;
+  }
+
+  if (tsdbLoadMFileHeader(pMFile, &minfo) < 0) {
+    tsdbCloseMFile(pMFile);
+    return -1;
+  }
+
+  while (true) {
+    int64_t tsize = tsdbReadMFile(pMFile, tbuf, sizeof(SKVRecord));
+    if (tsize == 0) break;
+    if (tsize < sizeof(SKVRecord)) {
+      tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s", REPO_ID(pRepo), sizeof(SKVRecord),
+                TSDB_FILE_FULL_NAME(pMFile));
+      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+      tsdbCloseMFile(pMFile);
+      return -1;
+    }
+
+    void *ptr = tsdbDecodeKVRecord(tbuf, &rInfo);
+    ASSERT(POINTER_DISTANCE(ptr, tbuf) == sizeof(SKVRecord));
+    // ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true);
+
+    if (rInfo.offset < 0) {
+      taosHashRemove(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid));
+#if 0
+      pStore->info.size += sizeof(SKVRecord);
+      pStore->info.nRecords--;
+      pStore->info.nDels++;
+      pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
+#endif
+    } else {
+      ASSERT(rInfo.offset > 0 && rInfo.size > 0);
+      if (taosHashPut(pfs->metaCache, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) {
+        tsdbError("vgId:%d failed to load meta cache from file %s since OOM", REPO_ID(pRepo),
+                  TSDB_FILE_FULL_NAME(pMFile));
+        terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
+        tsdbCloseMFile(pMFile);
+        return -1;
+      }
+
+      maxBufSize = MAX(maxBufSize, rInfo.size);
+
+      if (tsdbSeekMFile(pMFile, rInfo.size, SEEK_CUR) < 0) {
+        tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
+                  tstrerror(terrno));
+        tsdbCloseMFile(pMFile);
+        return -1;
+      }
+
+#if 0
+      pStore->info.size += (sizeof(SKVRecord) + rInfo.size);
+      pStore->info.nRecords++;
+#endif
+    }
+  }
+
+  if (recoverMeta) {
+    pBuf = malloc((size_t)maxBufSize);
+    if (pBuf == NULL) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      tsdbCloseMFile(pMFile);
+      return -1;
+    }
+
+    SKVRecord *pRecord = taosHashIterate(pfs->metaCache, NULL);
+    while (pRecord) {
+      if (tsdbSeekMFile(pMFile, pRecord->offset + sizeof(SKVRecord), SEEK_SET) < 0) {
+        tsdbError("vgId:%d failed to seek file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
+                  tstrerror(terrno));
+        tfree(pBuf);
+        tsdbCloseMFile(pMFile);
+        return -1;
+      }
+
+      int nread = (int)tsdbReadMFile(pMFile, pBuf, pRecord->size);
+      if (nread < 0) {
+        tsdbError("vgId:%d failed to read file %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
+                  tstrerror(terrno));
+        tfree(pBuf);
+        tsdbCloseMFile(pMFile);
+        return -1;
+      }
+
+      if (nread < pRecord->size) {
+        tsdbError("vgId:%d failed to read file %s since file corrupted, expected read:%" PRId64 " actual read:%d",
+                  REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), pRecord->size, nread);
+        terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+        tfree(pBuf);
+        tsdbCloseMFile(pMFile);
+        return -1;
+      }
+
+      if (tsdbRestoreTable(pRepo, pBuf, (int)pRecord->size) < 0) {
+        tsdbError("vgId:%d failed to restore table, uid %" PRId64 ", since %s" PRIu64, REPO_ID(pRepo), pRecord->uid,
+                  tstrerror(terrno));
+        tfree(pBuf);
+        tsdbCloseMFile(pMFile);
+        return -1;
+      }
+
+      pRecord = taosHashIterate(pfs->metaCache, pRecord);
+    }
+
+    tsdbOrgMeta(pRepo);
+  }
+
+  tsdbCloseMFile(pMFile);
+  tfree(pBuf);
+  return 0;
+}
+
+static int tsdbScanRootDir(STsdbRepo *pRepo) {
+  char         rootDir[TSDB_FILENAME_LEN];
+  char         bname[TSDB_FILENAME_LEN];
+  STsdbFS *    pfs = REPO_FS(pRepo);
+  const TFILE *pf;
+
+  tsdbGetRootDir(REPO_ID(pRepo), rootDir);
+  TDIR *tdir = tfsOpendir(rootDir);
+  if (tdir == NULL) {
+    tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
+    return -1;
+  }
+
+  while ((pf = tfsReaddir(tdir))) {
+    tfsbasename(pf, bname);
+
+    if (strcmp(bname, tsdbTxnFname[TSDB_TXN_CURR_FILE]) == 0 || strcmp(bname, "data") == 0) {
+      // Skip current file and data directory
+      continue;
+    }
+
+    if (tfsIsSameFile(pf, &(pfs->cstatus->pmf->f))) {
+      continue;
+    }
+
+    tfsremove(pf);
+    tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf));
+  }
+
+  tfsClosedir(tdir);
+
+  return 0;
+}
+
+static int tsdbScanDataDir(STsdbRepo *pRepo) {
+  char         dataDir[TSDB_FILENAME_LEN];
+  char         bname[TSDB_FILENAME_LEN];
+  STsdbFS *    pfs = REPO_FS(pRepo);
+  const TFILE *pf;
+
+  tsdbGetDataDir(REPO_ID(pRepo), dataDir);
+  TDIR *tdir = tfsOpendir(dataDir);
+  if (tdir == NULL) {
+    tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), dataDir, tstrerror(terrno));
+    return -1;
+  }
+
+  while ((pf = tfsReaddir(tdir))) {
+    tfsbasename(pf, bname);
+
+    if (!tsdbIsTFileInFS(pfs, pf)) {
+      tfsremove(pf);
+      tsdbDebug("vgId:%d invalid file %s is removed", REPO_ID(pRepo), TFILE_NAME(pf));
+    }
+  }
+
+  tfsClosedir(tdir);
+
+  return 0;
+}
+
+static bool tsdbIsTFileInFS(STsdbFS *pfs, const TFILE *pf) {
+  SFSIter fsiter;
+  tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
+  SDFileSet *pSet;
+
+  while ((pSet = tsdbFSIterNext(&fsiter))) {
+    for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+      SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype);
+      if (tfsIsSameFile(pf, TSDB_FILE_F(pDFile))) {
+        return true;
+      }
+    }
+  }
+
+  return false;
+}
+
+static int tsdbRestoreMeta(STsdbRepo *pRepo) {
+  char         rootDir[TSDB_FILENAME_LEN];
+  char         bname[TSDB_FILENAME_LEN];
+  TDIR *       tdir = NULL;
+  const TFILE *pf = NULL;
+  const char * pattern = "^meta(-ver[0-9]+)?$";
+  regex_t      regex;
+  STsdbFS *    pfs = REPO_FS(pRepo);
+
+  regcomp(&regex, pattern, REG_EXTENDED);
+
+  tsdbInfo("vgId:%d try to restore meta", REPO_ID(pRepo));
+
+  tsdbGetRootDir(REPO_ID(pRepo), rootDir);
+
+  tdir = tfsOpendir(rootDir);
+  if (tdir == NULL) {
+    tsdbError("vgId:%d failed to open dir %s since %s", REPO_ID(pRepo), rootDir, tstrerror(terrno));
+    regfree(&regex);
+    return -1;
+  }
+
+  while ((pf = tfsReaddir(tdir))) {
+    tfsbasename(pf, bname);
+
+    if (strcmp(bname, "data") == 0) {
+      // Skip the data/ directory
+      continue;
+    }
+
+    if (strcmp(bname, tsdbTxnFname[TSDB_TXN_TEMP_FILE]) == 0) {
+      // Skip current.t file
+      tsdbInfo("vgId:%d file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf));
+      tfsremove(pf);
+      continue;
+    }
+
+    int code = regexec(&regex, bname, 0, NULL, 0);
+    if (code == 0) {
+      // Match
+      if (pfs->cstatus->pmf != NULL) {
+        tsdbError("vgId:%d failed to restore meta since two file exists, file1 %s and file2 %s", REPO_ID(pRepo),
+                  TSDB_FILE_FULL_NAME(pfs->cstatus->pmf), TFILE_NAME(pf));
+        terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+        tfsClosedir(tdir);
+        regfree(&regex);
+        return -1;
+      } else {
+        uint32_t version = 0;
+        if (strcmp(bname, "meta") != 0) {
+          sscanf(bname, "meta-ver%" PRIu32, &version);
+          pfs->cstatus->meta.version = version;
+        }
+
+        pfs->cstatus->pmf = &(pfs->cstatus->mf);
+        pfs->cstatus->pmf->f = *pf;
+        TSDB_FILE_SET_CLOSED(pfs->cstatus->pmf);
+
+        if (tsdbOpenMFile(pfs->cstatus->pmf, O_RDONLY) < 0) {
+          tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+          tfsClosedir(tdir);
+          regfree(&regex);
+          return -1;
+        }
+
+        if (tsdbLoadMFileHeader(pfs->cstatus->pmf, &(pfs->cstatus->pmf->info)) < 0) {
+          tsdbError("vgId:%d failed to restore meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+          tsdbCloseMFile(pfs->cstatus->pmf);
+          tfsClosedir(tdir);
+          regfree(&regex);
+          return -1;
+        }
+
+        tsdbCloseMFile(pfs->cstatus->pmf);
+      }
+    } else if (code == REG_NOMATCH) {
+      // Not match
+      tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf));
+      tfsremove(pf);
+      continue;
+    } else {
+      // Has other error
+      tsdbError("vgId:%d failed to restore meta file while run regexec since %s", REPO_ID(pRepo), strerror(code));
+      terrno = TAOS_SYSTEM_ERROR(code);
+      tfsClosedir(tdir);
+      regfree(&regex);
+      return -1;
+    }
+  }
+
+  if (pfs->cstatus->pmf) {
+    tsdbInfo("vgId:%d meta file %s is restored", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pfs->cstatus->pmf));
+  } else {
+    tsdbInfo("vgId:%d no meta file is restored", REPO_ID(pRepo));
+  }
+
+  tfsClosedir(tdir);
+  regfree(&regex);
+  return 0;
+}
+
+static int tsdbRestoreDFileSet(STsdbRepo *pRepo) {
+  char         dataDir[TSDB_FILENAME_LEN];
+  char         bname[TSDB_FILENAME_LEN];
+  TDIR *       tdir = NULL;
+  const TFILE *pf = NULL;
+  const char * pattern = "^v[0-9]+f[0-9]+\\.(head|data|last)(-ver[0-9]+)?$";
+  SArray *     fArray = NULL;
+  regex_t      regex;
+  STsdbFS *    pfs = REPO_FS(pRepo);
+
+  tsdbGetDataDir(REPO_ID(pRepo), dataDir);
+
+  // Resource allocation and init
+  regcomp(&regex, pattern, REG_EXTENDED);
+
+  fArray = taosArrayInit(1024, sizeof(TFILE));
+  if (fArray == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir,
+              tstrerror(terrno));
+    regfree(&regex);
+    return -1;
+  }
+
+  tdir = tfsOpendir(dataDir);
+  if (tdir == NULL) {
+    tsdbError("vgId:%d failed to restore DFileSet while open directory %s since %s", REPO_ID(pRepo), dataDir,
+              tstrerror(terrno));
+    taosArrayDestroy(fArray);
+    regfree(&regex);
+    return -1;
+  }
+
+  while ((pf = tfsReaddir(tdir))) {
+    tfsbasename(pf, bname);
+
+    int code = regexec(&regex, bname, 0, NULL, 0);
+    if (code == 0) {
+      if (taosArrayPush(fArray, (void *)pf) < 0) {
+        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+        tfsClosedir(tdir);
+        taosArrayDestroy(fArray);
+        regfree(&regex);
+        return -1;
+      }
+    } else if (code == REG_NOMATCH) {
+      // Not match
+      tsdbInfo("vgId:%d invalid file %s exists, remove it", REPO_ID(pRepo), TFILE_NAME(pf));
+      tfsremove(pf);
+      continue;
+    } else {
+      // Has other error
+      tsdbError("vgId:%d failed to restore DFileSet Array while run regexec since %s", REPO_ID(pRepo), strerror(code));
+      terrno = TAOS_SYSTEM_ERROR(code);
+      tfsClosedir(tdir);
+      taosArrayDestroy(fArray);
+      regfree(&regex);
+      return -1;
+    }
+  }
+
+  tfsClosedir(tdir);
+  regfree(&regex);
+
+  // Sort the array according to file name
+  taosArraySort(fArray, tsdbComparTFILE);
+
+  size_t index = 0;
+  // Loop to recover each file set
+  for (;;) {
+    if (index >= taosArrayGetSize(fArray)) {
+      break;
+    }
+
+    SDFileSet fset = {0};
+
+    TSDB_FSET_SET_CLOSED(&fset);
+
+    // Loop to recover ONE fset
+    for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+      SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype);
+
+      if (index >= taosArrayGetSize(fArray)) {
+        tsdbError("vgId:%d incomplete DFileSet, fid:%d", REPO_ID(pRepo), fset.fid);
+        taosArrayDestroy(fArray);
+        return -1;
+      }
+
+      pf = taosArrayGet(fArray, index);
+
+      int         tvid, tfid;
+      TSDB_FILE_T ttype;
+      uint32_t    tversion;
+      char        bname[TSDB_FILENAME_LEN];
+
+      tfsbasename(pf, bname);
+      tsdbParseDFilename(bname, &tvid, &tfid, &ttype, &tversion);
+
+      ASSERT(tvid == REPO_ID(pRepo));
+
+      if (ftype == 0) {
+        fset.fid = tfid;
+      } else {
+        if (tfid != fset.fid) {
+          tsdbError("vgId:%d incomplete dFileSet, fid:%d", REPO_ID(pRepo), fset.fid);
+          taosArrayDestroy(fArray);
+          return -1;
+        }
+      }
+
+      if (ttype != ftype) {
+        tsdbError("vgId:%d incomplete dFileSet, fid:%d", REPO_ID(pRepo), fset.fid);
+        taosArrayDestroy(fArray);
+        return -1;
+      }
+
+      pDFile->f = *pf;
+      
+      if (tsdbOpenDFile(pDFile, O_RDONLY) < 0) {
+        tsdbError("vgId:%d failed to open DFile %s since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno));
+        taosArrayDestroy(fArray);
+        return -1;
+      }
+
+      if (tsdbLoadDFileHeader(pDFile, &(pDFile->info)) < 0) {
+        tsdbError("vgId:%d failed to load DFile %s header since %s", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
+                  tstrerror(terrno));
+        taosArrayDestroy(fArray);
+        return -1;
+      }
+
+      tsdbCloseDFile(pDFile);
+      index++;
+    }
+
+    tsdbInfo("vgId:%d FSET %d is restored", REPO_ID(pRepo), fset.fid);
+    taosArrayPush(pfs->cstatus->df, &fset);
+  }
+
+  // Resource release
+  taosArrayDestroy(fArray);
+
+  return 0;
+}
+
+static int tsdbRestoreCurrent(STsdbRepo *pRepo) {
+  // Loop to recover mfile
+  if (tsdbRestoreMeta(pRepo) < 0) {
+    tsdbError("vgId:%d failed to restore current since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  // Loop to recover dfile set
+  if (tsdbRestoreDFileSet(pRepo) < 0) {
+    tsdbError("vgId:%d failed to restore DFileSet since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  if (tsdbSaveFSStatus(pRepo->fs->cstatus, REPO_ID(pRepo)) < 0) {
+    tsdbError("vgId:%d failed to restore corrent since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  return 0;
+}
+
+static int tsdbComparTFILE(const void *arg1, const void *arg2) {
+  TFILE *pf1 = (TFILE *)arg1;
+  TFILE *pf2 = (TFILE *)arg2;
+
+  int         vid1, fid1, vid2, fid2;
+  TSDB_FILE_T ftype1, ftype2;
+  uint32_t    version1, version2;
+  char        bname1[TSDB_FILENAME_LEN];
+  char        bname2[TSDB_FILENAME_LEN];
+
+  tfsbasename(pf1, bname1);
+  tfsbasename(pf2, bname2);
+  tsdbParseDFilename(bname1, &vid1, &fid1, &ftype1, &version1);
+  tsdbParseDFilename(bname2, &vid2, &fid2, &ftype2, &version2);
+
+  if (fid1 < fid2) {
+    return -1;
+  } else if (fid1 > fid2) {
+    return 1;
+  } else {
+    if (ftype1 < ftype2) {
+      return -1;
+    } else if (ftype1 > ftype2) {
+      return 1;
+    } else {
+      return 0;
+    }
+  }
+}
\ No newline at end of file

src/tsdb/src/tsdbFile.c

@@ -12,433 +12,476 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#define _DEFAULT_SOURCE
-#define TAOS_RANDOM_FILE_FAIL_TEST
-#include <regex.h>
-#include "os.h"
-#include "talgo.h"
-#include "tchecksum.h"
-#include "tsdbMain.h"
-#include "tutil.h"
 
+#include "tsdbint.h"
 
-const char *tsdbFileSuffix[] = {".head", ".data", ".last", ".stat", ".h", ".d", ".l", ".s"};
+static const char *TSDB_FNAME_SUFFIX[] = {
+    "head",  // TSDB_FILE_HEAD
+    "data",  // TSDB_FILE_DATA
+    "last",  // TSDB_FILE_LAST
+    "",      // TSDB_FILE_MAX
+    "meta"   // TSDB_FILE_META
+};
 
-static int   tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type);
-static void  tsdbDestroyFile(SFile *pFile);
-static int   compFGroup(const void *arg1, const void *arg2);
-static int   keyFGroupCompFunc(const void *key, const void *fgroup);
-static void  tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo);
-static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep);
-static int   tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days);
+static void  tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname);
+static int   tsdbRollBackMFile(SMFile *pMFile);
+static int   tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo);
+static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo);
+static int   tsdbRollBackDFile(SDFile *pDFile);
 
-// ---------------- INTERNAL FUNCTIONS ----------------
-STsdbFileH *tsdbNewFileH(STsdbCfg *pCfg) {
-  STsdbFileH *pFileH = (STsdbFileH *)calloc(1, sizeof(*pFileH));
-  if (pFileH == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
-  }
+// ============== SMFile
+void tsdbInitMFile(SMFile *pMFile, SDiskID did, int vid, uint32_t ver) {
+  char fname[TSDB_FILENAME_LEN];
 
-  int code = pthread_rwlock_init(&(pFileH->fhlock), NULL);
-  if (code != 0) {
-    tsdbError("vgId:%d failed to init file handle lock since %s", pCfg->tsdbId, strerror(code));
-    terrno = TAOS_SYSTEM_ERROR(code);
-    goto _err;
-  }
+  TSDB_FILE_SET_CLOSED(pMFile);
 
-  pFileH->maxFGroups = TSDB_MAX_FILE(pCfg->keep, pCfg->daysPerFile);
+  memset(&(pMFile->info), 0, sizeof(pMFile->info));
+  pMFile->info.magic = TSDB_FILE_INIT_MAGIC;
 
-  pFileH->pFGroup = (SFileGroup *)calloc(pFileH->maxFGroups, sizeof(SFileGroup));
-  if (pFileH->pFGroup == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
-  }
+  tsdbGetFilename(vid, 0, ver, TSDB_FILE_META, fname);
+  tfsInitFile(TSDB_FILE_F(pMFile), did.level, did.id, fname);
+}
+
+void tsdbInitMFileEx(SMFile *pMFile, SMFile *pOMFile) {
+  *pMFile = *pOMFile;
+  TSDB_FILE_SET_CLOSED(pMFile);
+}
+
+int tsdbEncodeSMFile(void **buf, SMFile *pMFile) {
+  int tlen = 0;
 
-  return pFileH;
+  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
+  tlen += tfsEncodeFile(buf, &(pMFile->f));
 
-_err:
-  tsdbFreeFileH(pFileH);
-  return NULL;
+  return tlen;
 }
 
-void tsdbFreeFileH(STsdbFileH *pFileH) {
-  if (pFileH) {
-    pthread_rwlock_destroy(&pFileH->fhlock);
-    tfree(pFileH->pFGroup);
-    free(pFileH);
-  }
+void *tsdbDecodeSMFile(void *buf, SMFile *pMFile) {
+  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
+  buf = tfsDecodeFile(buf, &(pMFile->f));
+  TSDB_FILE_SET_CLOSED(pMFile);
+
+  return buf;
+}
+
+int tsdbEncodeSMFileEx(void **buf, SMFile *pMFile) {
+  int tlen = 0;
+
+  tlen += tsdbEncodeMFInfo(buf, &(pMFile->info));
+  tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pMFile));
+
+  return tlen;
 }
 
-int tsdbOpenFileH(STsdbRepo *pRepo) {
-  ASSERT(pRepo != NULL && pRepo->tsdbFileH != NULL);
+void *tsdbDecodeSMFileEx(void *buf, SMFile *pMFile) {
+  char *aname;
+  buf = tsdbDecodeMFInfo(buf, &(pMFile->info));
+  buf = taosDecodeString(buf, &aname);
+  strncpy(TSDB_FILE_FULL_NAME(pMFile), aname, TSDB_FILENAME_LEN);
+  TSDB_FILE_SET_CLOSED(pMFile);
+
+  tfree(aname);
 
-  char *  tDataDir = NULL;
-  DIR *   dir = NULL;
-  int     fid = 0;
-  int     vid = 0;
-  regex_t regex1 = {0}, regex2 = {0};
-  int     code = 0;
-  char    fname[TSDB_FILENAME_LEN] = "\0";
+  return buf;
+}
 
-  SFileGroup  fileGroup = {0};
-  STsdbFileH *pFileH = pRepo->tsdbFileH;
-  STsdbCfg *  pCfg = &(pRepo->config);
+int tsdbApplyMFileChange(SMFile *from, SMFile *to) {
+  if (from == NULL && to == NULL) return 0;
 
-  tDataDir = tsdbGetDataDirName(pRepo->rootDir);
-  if (tDataDir == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
+  if (from != NULL) {
+    if (to == NULL) {
+      return tsdbRemoveMFile(from);
+    } else {
+      if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
+        if (from->info.size > to->info.size) {
+          tsdbRollBackMFile(to);
+        }
+      } else {
+        return tsdbRemoveMFile(from);
+      }
+    }
   }
 
-  dir = opendir(tDataDir);
-  if (dir == NULL) {
-    if (errno == ENOENT) {
-      tsdbError("vgId:%d directory %s not exist", REPO_ID(pRepo), tDataDir);
-      terrno = TAOS_SYSTEM_ERROR(errno);
+  return 0;
+}
 
-      if (taosMkDir(tDataDir, 0755) < 0) {
-        tsdbError("vgId:%d failed to create directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        goto _err;
+int tsdbCreateMFile(SMFile *pMFile, bool updateHeader) {
+  ASSERT(pMFile->info.size == 0 && pMFile->info.magic == TSDB_FILE_INIT_MAGIC);
+
+  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+  if (pMFile->fd < 0) {
+    if (errno == ENOENT) {
+      // Try to create directory recursively
+      char *s = strdup(TFILE_REL_NAME(&(pMFile->f)));
+      if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pMFile), TSDB_FILE_ID(pMFile)) < 0) {
+        tfree(s);
+        return -1;
       }
+      tfree(s);
 
-      dir = opendir(tDataDir);
-      if (dir == NULL) {
-        tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
+      pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+      if (pMFile->fd < 0) {
         terrno = TAOS_SYSTEM_ERROR(errno);
-        goto _err;
+        return -1;
       }
     } else {
-      tsdbError("vgId:%d failed to open directory %s since %s", REPO_ID(pRepo), tDataDir, strerror(errno));
       terrno = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
+      return -1;
     }
   }
 
-  code = regcomp(&regex1, "^v[0-9]+f[0-9]+\\.(head|data|last|stat)$", REG_EXTENDED);
-  if (code != 0) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
+  if (!updateHeader) {
+    return 0;
   }
 
-  code = regcomp(&regex2, "^v[0-9]+f[0-9]+\\.(h|d|l|s)$", REG_EXTENDED);
-  if (code != 0) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
+  if (tsdbUpdateMFileHeader(pMFile) < 0) {
+    tsdbCloseMFile(pMFile);
+    tsdbRemoveMFile(pMFile);
+    return -1;
   }
 
-  int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile);
+  pMFile->info.size += TSDB_FILE_HEAD_SIZE;
 
-  struct dirent *dp = NULL;
-  while ((dp = readdir(dir)) != NULL) {
-    if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue;
+  return 0;
+}
 
-    code = regexec(&regex1, dp->d_name, 0, NULL, 0);
-    if (code == 0) {
-      sscanf(dp->d_name, "v%df%d", &vid, &fid);
-      if (vid != REPO_ID(pRepo)) {
-        tsdbError("vgId:%d invalid file %s exists, ignore it", REPO_ID(pRepo), dp->d_name);
-        continue;
-      }
+int tsdbUpdateMFileHeader(SMFile *pMFile) {
+  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
 
-      if (fid < mfid) {
-        for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
-          tsdbGetDataFileName(pRepo->rootDir, pCfg->tsdbId, fid, type, fname);
-          (void)remove(fname);
-        }
-        continue;
-      }
+  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
+    return -1;
+  }
 
-      if (tsdbSearchFGroup(pFileH, fid, TD_EQ) != NULL) continue;
-      memset((void *)(&fileGroup), 0, sizeof(SFileGroup));
-      fileGroup.fileId = fid;
-
-      tsdbInitFileGroup(&fileGroup, pRepo);
-    } else if (code == REG_NOMATCH) {
-      code = regexec(&regex2, dp->d_name, 0, NULL, 0);
-      if (code == 0) {
-        size_t tsize = strlen(tDataDir) + strlen(dp->d_name) + 2;
-        char * fname1 = malloc(tsize);
-        if (fname1 == NULL) {
-          terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-          goto _err;
-        }
-        sprintf(fname1, "%s/%s", tDataDir, dp->d_name);
-
-        tsize = tsize + 64;
-        char *fname2 = malloc(tsize);
-        if (fname2 == NULL) {
-          free(fname1);
-          terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-          goto _err;
-        }
-        sprintf(fname2, "%s/%s_back_%" PRId64, tDataDir, dp->d_name, taosGetTimestamp(TSDB_TIME_PRECISION_MILLI));
+  void *ptr = buf;
+  tsdbEncodeMFInfo(&ptr, TSDB_FILE_INFO(pMFile));
 
-        (void)taosRename(fname1, fname2);
+  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
+  if (tsdbWriteMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
+    return -1;
+  }
 
-        tsdbDebug("vgId:%d file %s exists, backup it as %s", REPO_ID(pRepo), fname1, fname2);
+  return 0;
+}
 
-        free(fname1);
-        free(fname2);
-        continue;
-      } else if (code == REG_NOMATCH) {
-        tsdbError("vgId:%d invalid file %s exists, ignore it", REPO_ID(pRepo), dp->d_name);
-        continue;
-      } else {
-        goto _err;
-      }
-    } else {
-      goto _err;
-    }
+int tsdbLoadMFileHeader(SMFile *pMFile, SMFInfo *pInfo) {
+  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
 
-    pFileH->pFGroup[pFileH->nFGroups++] = fileGroup;
-    qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup);
-    tsdbDebug("vgId:%d file group %d is restored, nFGroups %d", REPO_ID(pRepo), fileGroup.fileId, pFileH->nFGroups);
-  }
+  ASSERT(TSDB_FILE_OPENED(pMFile));
 
-  regfree(&regex1);
-  regfree(&regex2);
-  tfree(tDataDir);
-  closedir(dir);
-  return 0;
+  if (tsdbSeekMFile(pMFile, 0, SEEK_SET) < 0) {
+    return -1;
+  }
 
-_err:
-  for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) tsdbDestroyFile(&fileGroup.files[type]);
+  if (tsdbReadMFile(pMFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
+    return -1;
+  }
 
-  regfree(&regex1);
-  regfree(&regex2);
+  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return -1;
+  }
 
-  tfree(tDataDir);
-  if (dir != NULL) closedir(dir);
-  tsdbCloseFileH(pRepo);
-  return -1;
+  tsdbDecodeMFInfo(buf, pInfo);
+  return 0;
 }
 
-void tsdbCloseFileH(STsdbRepo *pRepo) {
-  STsdbFileH *pFileH = pRepo->tsdbFileH;
+int tsdbScanAndTryFixMFile(STsdbRepo *pRepo) {
+  SMFile *    pMFile = pRepo->fs->cstatus->pmf;
+  struct stat mfstat;
+  SMFile      mf;
 
-  for (int i = 0; i < pFileH->nFGroups; i++) {
-    SFileGroup *pFGroup = pFileH->pFGroup + i;
-    for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
-      tsdbDestroyFile(&pFGroup->files[type]);
-    }
+  if (pMFile == NULL) {
+    // No meta file, no need to scan
+    return 0;
   }
-}
 
-SFileGroup *tsdbCreateFGroupIfNeed(STsdbRepo *pRepo, char *dataDir, int fid) {
-  STsdbFileH *pFileH = pRepo->tsdbFileH;
-  STsdbCfg *  pCfg = &(pRepo->config);
+  tsdbInitMFileEx(&mf, pMFile);
 
-  if (pFileH->nFGroups >= pFileH->maxFGroups) {
-    int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile);
-    if (pFileH->pFGroup[0].fileId < mfid) {
-      pthread_rwlock_wrlock(&pFileH->fhlock);
-      tsdbRemoveFileGroup(pRepo, &(pFileH->pFGroup[0]));
-      pthread_rwlock_unlock(&pFileH->fhlock);
-    }
+  if (access(TSDB_FILE_FULL_NAME(pMFile), F_OK) != 0) {
+    tsdbError("vgId:%d meta file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
+              TSDB_FILE_FULL_NAME(pMFile));
+    pRepo->state |= TSDB_STATE_BAD_META;
+    return 0;
   }
 
-  ASSERT(pFileH->nFGroups < pFileH->maxFGroups);
+  if (stat(TSDB_FILE_FULL_NAME(&mf), &mfstat) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
 
-  SFileGroup  fGroup;
-  SFileGroup *pFGroup = &fGroup;
+  if (pMFile->info.size < mfstat.st_size) {
+    if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
+      return -1;
+    }
 
-  SFileGroup *pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ);
-  if (pGroup == NULL) {  // if not exists, create one
-    pFGroup->fileId = fid;
-    for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
-      if (tsdbCreateFile(&pFGroup->files[type], pRepo, fid, type) < 0) {
-        for (int i = type; i >= 0; i--) {
-          remove(pFGroup->files[i].fname);
-        }
+    if (taosFtruncate(mf.fd, mf.info.size) < 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      tsdbCloseMFile(&mf);
+      return -1;
+    }
 
-        return NULL;
-      }
+    if (tsdbUpdateMFileHeader(&mf) < 0) {
+      tsdbCloseMFile(&mf);
+      return -1;
     }
 
-    pthread_rwlock_wrlock(&pFileH->fhlock);
-    pFileH->pFGroup[pFileH->nFGroups++] = fGroup;
-    qsort((void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), compFGroup);
-    pthread_rwlock_unlock(&pFileH->fhlock);
-    pGroup = tsdbSearchFGroup(pFileH, fid, TD_EQ);
-    ASSERT(pGroup != NULL);
+    tsdbCloseMFile(&mf);
+    tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile),
+             mfstat.st_size, pMFile->info.size);
+  } else if (pMFile->info.size > mfstat.st_size) {
+    tsdbError("vgId:%d meta file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
+              REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile), mfstat.st_size, pMFile->info.size);
+    pRepo->state |= TSDB_STATE_BAD_META;
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return 0;
+  } else {
+    tsdbDebug("vgId:%d meta file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pMFile));
   }
 
-  return pGroup;
+  return 0;
 }
 
-void tsdbInitFileGroupIter(STsdbFileH *pFileH, SFileGroupIter *pIter, int direction) {
-  pIter->pFileH = pFileH;
-  pIter->direction = direction;
+int tsdbEncodeMFInfo(void **buf, SMFInfo *pInfo) {
+  int tlen = 0;
 
-  if (pFileH->nFGroups == 0) {
-    pIter->index = -1;
-    pIter->fileId = -1;
-  } else {
-    if (direction == TSDB_FGROUP_ITER_FORWARD) {
-      pIter->index = 0;
-    } else {
-      pIter->index = pFileH->nFGroups - 1;
-    }
-    pIter->fileId = pFileH->pFGroup[pIter->index].fileId;
-  }
+  tlen += taosEncodeVariantI64(buf, pInfo->size);
+  tlen += taosEncodeVariantI64(buf, pInfo->tombSize);
+  tlen += taosEncodeVariantI64(buf, pInfo->nRecords);
+  tlen += taosEncodeVariantI64(buf, pInfo->nDels);
+  tlen += taosEncodeFixedU32(buf, pInfo->magic);
+
+  return tlen;
+}
+
+void *tsdbDecodeMFInfo(void *buf, SMFInfo *pInfo) {
+  buf = taosDecodeVariantI64(buf, &(pInfo->size));
+  buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
+  buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
+  buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
+  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
+
+  return buf;
 }
 
-void tsdbSeekFileGroupIter(SFileGroupIter *pIter, int fid) {
-  STsdbFileH *pFileH = pIter->pFileH;
+static int tsdbRollBackMFile(SMFile *pMFile) {
+  SMFile mf;
 
-  if (pFileH->nFGroups == 0) {
-    pIter->index = -1;
-    pIter->fileId = -1;
-    return;
+  tsdbInitMFileEx(&mf, pMFile);
+
+  if (tsdbOpenMFile(&mf, O_WRONLY) < 0) {
+    return -1;
   }
 
-  int   flags = (pIter->direction == TSDB_FGROUP_ITER_FORWARD) ? TD_GE : TD_LE;
-  void *ptr = taosbsearch(&fid, (void *)pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags);
-  if (ptr == NULL) {
-    pIter->index = -1;
-    pIter->fileId = -1;
-  } else {
-    pIter->index = (int)(POINTER_DISTANCE(ptr, pFileH->pFGroup) / sizeof(SFileGroup));
-    pIter->fileId = ((SFileGroup *)ptr)->fileId;
+  if (taosFtruncate(TSDB_FILE_FD(&mf), pMFile->info.size) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbCloseMFile(&mf);
+    return -1;
   }
+
+  if (tsdbUpdateMFileHeader(&mf) < 0) {
+    tsdbCloseMFile(&mf);
+    return -1;
+  }
+
+  TSDB_FILE_FSYNC(&mf);
+
+  tsdbCloseMFile(&mf);
+  return 0;
 }
 
-SFileGroup *tsdbGetFileGroupNext(SFileGroupIter *pIter) {
-  STsdbFileH *pFileH = pIter->pFileH;
-  SFileGroup *pFGroup = NULL;
+// ============== Operations on SDFile
+void tsdbInitDFile(SDFile *pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype) {
+  char fname[TSDB_FILENAME_LEN];
 
-  if (pIter->index < 0 || pIter->index >= pFileH->nFGroups || pIter->fileId < 0) return NULL;
+  TSDB_FILE_SET_CLOSED(pDFile);
 
-  pFGroup = &pFileH->pFGroup[pIter->index];
-  if (pFGroup->fileId != pIter->fileId) {
-    tsdbSeekFileGroupIter(pIter, pIter->fileId);
-  }
+  memset(&(pDFile->info), 0, sizeof(pDFile->info));
+  pDFile->info.magic = TSDB_FILE_INIT_MAGIC;
 
-  if (pIter->index < 0) return NULL;
+  tsdbGetFilename(vid, fid, ver, ftype, fname);
+  tfsInitFile(&(pDFile->f), did.level, did.id, fname);
+}
 
-  pFGroup = &pFileH->pFGroup[pIter->index];
-  ASSERT(pFGroup->fileId == pIter->fileId);
+void tsdbInitDFileEx(SDFile *pDFile, SDFile *pODFile) {
+  *pDFile = *pODFile;
+  TSDB_FILE_SET_CLOSED(pDFile);
+}
 
-  if (pIter->direction == TSDB_FGROUP_ITER_FORWARD) {
-    pIter->index++;
-  } else {
-    pIter->index--;
-  }
+int tsdbEncodeSDFile(void **buf, SDFile *pDFile) {
+  int tlen = 0;
 
-  if (pIter->index >= 0 && pIter->index < pFileH->nFGroups) {
-    pIter->fileId = pFileH->pFGroup[pIter->index].fileId;
-  } else {
-    pIter->fileId = -1;
-  }
+  tlen += tsdbEncodeDFInfo(buf, &(pDFile->info));
+  tlen += tfsEncodeFile(buf, &(pDFile->f));
 
-  return pFGroup;
+  return tlen;
 }
 
-int tsdbOpenFile(SFile *pFile, int oflag) {
-  ASSERT(!TSDB_IS_FILE_OPENED(pFile));
+void *tsdbDecodeSDFile(void *buf, SDFile *pDFile) {
+  buf = tsdbDecodeDFInfo(buf, &(pDFile->info));
+  buf = tfsDecodeFile(buf, &(pDFile->f));
+  TSDB_FILE_SET_CLOSED(pDFile);
 
-  pFile->fd = open(pFile->fname, oflag | O_BINARY, 0755);
-  if (pFile->fd < 0) {
-    tsdbError("failed to open file %s since %s", pFile->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
+  return buf;
+}
 
-  tsdbTrace("open file %s, fd %d", pFile->fname, pFile->fd);
+static int tsdbEncodeSDFileEx(void **buf, SDFile *pDFile) {
+  int tlen = 0;
 
-  return 0;
+  tlen += tsdbEncodeDFInfo(buf, &(pDFile->info));
+  tlen += taosEncodeString(buf, TSDB_FILE_FULL_NAME(pDFile));
+
+  return tlen;
 }
 
-void tsdbCloseFile(SFile *pFile) {
-  if (TSDB_IS_FILE_OPENED(pFile)) {
-    tsdbTrace("close file %s, fd %d", pFile->fname, pFile->fd);
-    close(pFile->fd);
-    pFile->fd = -1;
-  }
+static void *tsdbDecodeSDFileEx(void *buf, SDFile *pDFile) {
+  char *aname;
+
+  buf = tsdbDecodeDFInfo(buf, &(pDFile->info));
+  buf = taosDecodeString(buf, &aname);
+  strncpy(TSDB_FILE_FULL_NAME(pDFile), aname, TSDB_FILENAME_LEN);
+  TSDB_FILE_SET_CLOSED(pDFile);
+  tfree(aname);
+
+  return buf;
 }
 
-int tsdbCreateFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) {
-  memset((void *)pFile, 0, sizeof(SFile));
-  pFile->fd = -1;
+int tsdbCreateDFile(SDFile *pDFile, bool updateHeader) {
+  ASSERT(pDFile->info.size == 0 && pDFile->info.magic == TSDB_FILE_INIT_MAGIC);
 
-  tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), fid, type, pFile->fname);
+  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+  if (pDFile->fd < 0) {
+    if (errno == ENOENT) {
+      // Try to create directory recursively
+      char *s = strdup(TFILE_REL_NAME(&(pDFile->f)));
+      if (tfsMkdirRecurAt(dirname(s), TSDB_FILE_LEVEL(pDFile), TSDB_FILE_ID(pDFile)) < 0) {
+        tfree(s);
+        return -1;
+      }
+      tfree(s);
 
-  if (access(pFile->fname, F_OK) == 0) {
-    tsdbError("vgId:%d file %s already exists", REPO_ID(pRepo), pFile->fname);
-    terrno = TSDB_CODE_TDB_FILE_ALREADY_EXISTS;
-    goto _err;
+      pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0755);
+      if (pDFile->fd < 0) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        return -1;
+      }
+    } else {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      return -1;
+    }
   }
 
-  if (tsdbOpenFile(pFile, O_RDWR | O_CREAT) < 0) {
-    goto _err;
+  if (!updateHeader) {
+    return 0;
   }
 
-  pFile->info.size = TSDB_FILE_HEAD_SIZE;
-  pFile->info.magic = TSDB_FILE_INIT_MAGIC;
-
-  if (tsdbUpdateFileHeader(pFile) < 0) {
-    tsdbCloseFile(pFile);
+  if (tsdbUpdateDFileHeader(pDFile) < 0) {
+    tsdbCloseDFile(pDFile);
+    tsdbRemoveDFile(pDFile);
     return -1;
   }
 
-  tsdbCloseFile(pFile);
+  pDFile->info.size += TSDB_FILE_HEAD_SIZE;
 
   return 0;
-
-_err:
-  return -1;
 }
 
-SFileGroup *tsdbSearchFGroup(STsdbFileH *pFileH, int fid, int flags) {
-  void *ptr =
-      taosbsearch((void *)(&fid), (void *)(pFileH->pFGroup), pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, flags);
-  if (ptr == NULL) return NULL;
-  return (SFileGroup *)ptr;
+int tsdbUpdateDFileHeader(SDFile *pDFile) {
+  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
+
+  if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) {
+    return -1;
+  }
+
+  void *ptr = buf;
+  taosEncodeFixedU32(&ptr, TSDB_FS_VERSION);
+  tsdbEncodeDFInfo(&ptr, &(pDFile->info));
+
+  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
+  if (tsdbWriteDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
+    return -1;
+  }
+
+  return 0;
 }
 
-void tsdbFitRetention(STsdbRepo *pRepo) {
-  STsdbCfg *pCfg = &(pRepo->config);
-  STsdbFileH *pFileH = pRepo->tsdbFileH;
-  SFileGroup *pGroup = pFileH->pFGroup;
+int tsdbLoadDFileHeader(SDFile *pDFile, SDFInfo *pInfo) {
+  char     buf[TSDB_FILE_HEAD_SIZE] = "\0";
+  uint32_t version;
 
-  int mfid = tsdbGetCurrMinFid(pCfg->precision, pCfg->keep, pCfg->daysPerFile);
+  ASSERT(TSDB_FILE_OPENED(pDFile));
 
-  pthread_rwlock_wrlock(&(pFileH->fhlock));
+  if (tsdbSeekDFile(pDFile, 0, SEEK_SET) < 0) {
+    return -1;
+  }
 
-  while (pFileH->nFGroups > 0 && pGroup[0].fileId < mfid) {
-    tsdbRemoveFileGroup(pRepo, pGroup);
+  if (tsdbReadDFile(pDFile, buf, TSDB_FILE_HEAD_SIZE) < 0) {
+    return -1;
   }
 
-  pthread_rwlock_unlock(&(pFileH->fhlock));
+  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return -1;
+  }
+
+  void *pBuf = buf;
+  pBuf = taosDecodeFixedU32(pBuf, &version);
+  pBuf = tsdbDecodeDFInfo(pBuf, pInfo);
+  return 0;
 }
 
-int tsdbUpdateFileHeader(SFile *pFile) {
-  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
+static int tsdbScanAndTryFixDFile(STsdbRepo *pRepo, SDFile *pDFile) {
+  struct stat dfstat;
+  SDFile      df;
 
-  void *pBuf = (void *)buf;
-  taosEncodeFixedU32((void *)(&pBuf), TSDB_FILE_VERSION);
-  tsdbEncodeSFileInfo((void *)(&pBuf), &(pFile->info));
+  tsdbInitDFileEx(&df, pDFile);
 
-  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
+  if (access(TSDB_FILE_FULL_NAME(pDFile), F_OK) != 0) {
+    tsdbError("vgId:%d data file %s not exit, report to upper layer to fix it", REPO_ID(pRepo),
+              TSDB_FILE_FULL_NAME(pDFile));
+    pRepo->state |= TSDB_STATE_BAD_DATA;
+    return 0;
+  }
 
-  if (lseek(pFile->fd, 0, SEEK_SET) < 0) {
-    tsdbError("failed to lseek file %s since %s", pFile->fname, strerror(errno));
+  if (stat(TSDB_FILE_FULL_NAME(&df), &dfstat) < 0) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
-  if (taosWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
-    tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
+
+  if (pDFile->info.size < dfstat.st_size) {
+    if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
+      return -1;
+    }
+
+    if (taosFtruncate(df.fd, df.info.size) < 0) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      tsdbCloseDFile(&df);
+      return -1;
+    }
+
+    if (tsdbUpdateDFileHeader(&df) < 0) {
+      tsdbCloseDFile(&df);
+      return -1;
+    }
+
+    tsdbCloseDFile(&df);
+    tsdbInfo("vgId:%d file %s is truncated from %" PRId64 " to %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
+             dfstat.st_size, pDFile->info.size);
+  } else if (pDFile->info.size > dfstat.st_size) {
+    tsdbError("vgId:%d data file %s has wrong size %" PRId64 " expected %" PRId64 ", report to upper layer to fix it",
+              REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile), dfstat.st_size, pDFile->info.size);
+    pRepo->state |= TSDB_STATE_BAD_DATA;
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return 0;
+  } else {
+    tsdbDebug("vgId:%d file %s passes the scan", REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile));
   }
 
   return 0;
 }
 
-int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) {
+static int tsdbEncodeDFInfo(void **buf, SDFInfo *pInfo) {
   int tlen = 0;
+
   tlen += taosEncodeFixedU32(buf, pInfo->magic);
   tlen += taosEncodeFixedU32(buf, pInfo->len);
   tlen += taosEncodeFixedU32(buf, pInfo->totalBlocks);
@@ -450,7 +493,7 @@ int tsdbEncodeSFileInfo(void **buf, const STsdbFileInfo *pInfo) {
   return tlen;
 }
 
-void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) {
+static void *tsdbDecodeDFInfo(void *buf, SDFInfo *pInfo) {
   buf = taosDecodeFixedU32(buf, &(pInfo->magic));
   buf = taosDecodeFixedU32(buf, &(pInfo->len));
   buf = taosDecodeFixedU32(buf, &(pInfo->totalBlocks));
@@ -462,156 +505,186 @@ void *tsdbDecodeSFileInfo(void *buf, STsdbFileInfo *pInfo) {
   return buf;
 }
 
-void tsdbRemoveFileGroup(STsdbRepo *pRepo, SFileGroup *pFGroup) {
-  ASSERT(pFGroup != NULL);
-  STsdbFileH *pFileH = pRepo->tsdbFileH;
-
-  SFileGroup fileGroup = *pFGroup;
+static int tsdbApplyDFileChange(SDFile *from, SDFile *to) {
+  ASSERT(from != NULL || to != NULL);
 
-  int nFilesLeft = pFileH->nFGroups - (int)(POINTER_DISTANCE(pFGroup, pFileH->pFGroup) / sizeof(SFileGroup) + 1);
-  if (nFilesLeft > 0) {
-    memmove((void *)pFGroup, POINTER_SHIFT(pFGroup, sizeof(SFileGroup)), sizeof(SFileGroup) * nFilesLeft);
-  }
-
-  pFileH->nFGroups--;
-  ASSERT(pFileH->nFGroups >= 0);
-
-  for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
-    if (remove(fileGroup.files[type].fname) < 0) {
-      tsdbError("vgId:%d failed to remove file %s", REPO_ID(pRepo), fileGroup.files[type].fname);
+  if (from != NULL) {
+    if (to == NULL) {
+      tsdbRemoveDFile(from);
+    } else {
+      if (tfsIsSameFile(TSDB_FILE_F(from), TSDB_FILE_F(to))) {
+        if (from->info.size > to->info.size) {
+          tsdbRollBackDFile(to);
+        }
+      } else {
+        tsdbRemoveDFile(from);
+      }
     }
-    tsdbDestroyFile(&fileGroup.files[type]);
   }
+
+  return 0;
 }
 
-int tsdbLoadFileHeader(SFile *pFile, uint32_t *version) {
-  char buf[TSDB_FILE_HEAD_SIZE] = "\0";
+static int tsdbRollBackDFile(SDFile *pDFile) {
+  SDFile df = *pDFile;
 
-  if (lseek(pFile->fd, 0, SEEK_SET) < 0) {
-    tsdbError("failed to lseek file %s to start since %s", pFile->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
+  if (tsdbOpenDFile(&df, O_WRONLY) < 0) {
     return -1;
   }
 
-  if (taosRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
-    tsdbError("failed to read file %s header part with %d bytes, reason:%s", pFile->fname, TSDB_FILE_HEAD_SIZE,
-              strerror(errno));
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+  if (taosFtruncate(TSDB_FILE_FD(&df), pDFile->info.size) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbCloseDFile(&df);
     return -1;
   }
 
-  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
-    tsdbError("file %s header part is corrupted with failed checksum", pFile->fname);
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+  if (tsdbUpdateDFileHeader(&df) < 0) {
+    tsdbCloseDFile(&df);
     return -1;
   }
 
-  void *pBuf = (void *)buf;
-  pBuf = taosDecodeFixedU32(pBuf, version);
-  pBuf = tsdbDecodeSFileInfo(pBuf, &(pFile->info));
+  TSDB_FILE_FSYNC(&df);
 
+  tsdbCloseDFile(&df);
   return 0;
 }
 
-void tsdbGetFileInfoImpl(char *fname, uint32_t *magic, int64_t *size) {
-  uint32_t      version = 0;
-  SFile         file;
-  SFile *       pFile = &file;
+// ============== Operations on SDFileSet
+void tsdbInitDFileSet(SDFileSet *pSet, SDiskID did, int vid, int fid, uint32_t ver) {
+  pSet->fid = fid;
+  pSet->state = 0;
 
-  strncpy(pFile->fname, fname, TSDB_FILENAME_LEN - 1);
-  pFile->fd = -1;
-
-  if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err;
-  if (tsdbLoadFileHeader(pFile, &version) < 0) goto _err;
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    SDFile *pDFile = TSDB_DFILE_IN_SET(pSet, ftype);
+    tsdbInitDFile(pDFile, did, vid, fid, ver, ftype);
+  }
+}
 
-  off_t offset = lseek(pFile->fd, 0, SEEK_END);
-  if (offset < 0) goto _err;
-  tsdbCloseFile(pFile);
+void tsdbInitDFileSetEx(SDFileSet *pSet, SDFileSet *pOSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tsdbInitDFileEx(TSDB_DFILE_IN_SET(pSet, ftype), TSDB_DFILE_IN_SET(pOSet, ftype));
+  }
+}
 
-  *magic = pFile->info.magic;
-  *size = offset;
+int tsdbEncodeDFileSet(void **buf, SDFileSet *pSet) {
+  int tlen = 0;
 
-  return;
+  tlen += taosEncodeFixedI32(buf, pSet->fid);
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tlen += tsdbEncodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype));
+  }
 
-_err:
-  tsdbCloseFile(pFile);
-  *magic = TSDB_FILE_INIT_MAGIC;
-  *size = 0;
+  return tlen;
 }
 
-// ---------------- LOCAL FUNCTIONS ----------------
-static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) {
-  uint32_t version;
+void *tsdbDecodeDFileSet(void *buf, SDFileSet *pSet) {
+  int32_t fid;
 
-  tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), fid, type, pFile->fname);
+  buf = taosDecodeFixedI32(buf, &(fid));
+  pSet->state = 0;
+  pSet->fid = fid;
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    buf = tsdbDecodeSDFile(buf, TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+  return buf;
+}
 
-  pFile->fd = -1;
-  if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err;
+int tsdbEncodeDFileSetEx(void **buf, SDFileSet *pSet) {
+  int tlen = 0;
 
-  if (tsdbLoadFileHeader(pFile, &version) < 0) {
-    tsdbError("vgId:%d failed to load file %s header part since %s", REPO_ID(pRepo), pFile->fname, tstrerror(terrno));
-    goto _err;
+  tlen += taosEncodeFixedI32(buf, pSet->fid);
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tlen += tsdbEncodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype));
   }
 
-  if (pFile->info.size == TSDB_FILE_HEAD_SIZE) {
-    pFile->info.size = lseek(pFile->fd, 0, SEEK_END);
-  }
+  return tlen;
+}
+
+void *tsdbDecodeDFileSetEx(void *buf, SDFileSet *pSet) {
+  int32_t fid;
 
-  if (version != TSDB_FILE_VERSION) {
-    // TODO: deal with error
-    tsdbError("vgId:%d file %s version %u is not the same as program version %u which may cause problem",
-              REPO_ID(pRepo), pFile->fname, version, TSDB_FILE_VERSION);
+  buf = taosDecodeFixedI32(buf, &(fid));
+  pSet->fid = fid;
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    buf = tsdbDecodeSDFileEx(buf, TSDB_DFILE_IN_SET(pSet, ftype));
   }
+  return buf;
+}
 
-  tsdbCloseFile(pFile);
+int tsdbApplyDFileSetChange(SDFileSet *from, SDFileSet *to) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    SDFile *pDFileFrom = (from) ? TSDB_DFILE_IN_SET(from, ftype) : NULL;
+    SDFile *pDFileTo = (to) ? TSDB_DFILE_IN_SET(to, ftype) : NULL;
+    if (tsdbApplyDFileChange(pDFileFrom, pDFileTo) < 0) {
+      return -1;
+    }
+  }
 
   return 0;
-_err:
-  tsdbDestroyFile(pFile);
-  return -1;
 }
 
-static void tsdbDestroyFile(SFile *pFile) { tsdbCloseFile(pFile); }
+int tsdbCreateDFileSet(SDFileSet *pSet, bool updateHeader) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbCreateDFile(TSDB_DFILE_IN_SET(pSet, ftype), updateHeader) < 0) {
+      tsdbCloseDFileSet(pSet);
+      tsdbRemoveDFileSet(pSet);
+      return -1;
+    }
+  }
 
-static int compFGroup(const void *arg1, const void *arg2) {
-  int val1 = ((SFileGroup *)arg1)->fileId;
-  int val2 = ((SFileGroup *)arg2)->fileId;
+  return 0;
+}
 
-  if (val1 < val2) {
-    return -1;
-  } else if (val1 > val2) {
-    return 1;
-  } else {
-    return 0;
+int tsdbUpdateDFileSetHeader(SDFileSet *pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbUpdateDFileHeader(TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
+      return -1;
+    }
   }
+  return 0;
 }
 
-static int keyFGroupCompFunc(const void *key, const void *fgroup) {
-  int         fid = *(int *)key;
-  SFileGroup *pFGroup = (SFileGroup *)fgroup;
-  if (fid == pFGroup->fileId) {
-    return 0;
-  } else {
-    return fid > pFGroup->fileId ? 1 : -1;
+int tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet *pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbScanAndTryFixDFile(pRepo, TSDB_DFILE_IN_SET(pSet, ftype)) < 0) {
+      return -1;
+    }
   }
+  return 0;
 }
 
-static void tsdbInitFileGroup(SFileGroup *pFGroup, STsdbRepo *pRepo) {
-  for (int type = 0; type < TSDB_FILE_TYPE_MAX; type++) {
-    if (tsdbInitFile(&pFGroup->files[type], pRepo, pFGroup->fileId, type) < 0) {
-      memset(&pFGroup->files[type].info, 0, sizeof(STsdbFileInfo));
-      pFGroup->files[type].info.magic = TSDB_FILE_INIT_MAGIC;
-      pFGroup->state = 1;
-      pRepo->state = TSDB_STATE_BAD_FILE;
-      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+int tsdbParseDFilename(const char *fname, int *vid, int *fid, TSDB_FILE_T *ftype, uint32_t *version) {
+  char *p = NULL;
+  *version = 0;
+  *ftype = TSDB_FILE_MAX;
+
+  sscanf(fname, "v%df%d.%m[a-z]-ver%" PRIu32, vid, fid, &p, version);
+  for (TSDB_FILE_T i = 0; i < TSDB_FILE_MAX; i++) {
+    if (strcmp(p, TSDB_FNAME_SUFFIX[i]) == 0) {
+      *ftype = i;
+      break;
     }
   }
-}
 
-static TSKEY tsdbGetCurrMinKey(int8_t precision, int32_t keep) {
-  return (TSKEY)(taosGetTimestamp(precision) - keep * tsMsPerDay[precision]);
+  tfree(p);
+  return 0;
 }
 
-static int tsdbGetCurrMinFid(int8_t precision, int32_t keep, int32_t days) {
-  return (int)(TSDB_KEY_FILEID(tsdbGetCurrMinKey(precision, keep), days, precision));
+static void tsdbGetFilename(int vid, int fid, uint32_t ver, TSDB_FILE_T ftype, char *fname) {
+  ASSERT(ftype != TSDB_FILE_MAX);
+
+  if (ftype < TSDB_FILE_MAX) {
+    if (ver == 0) {
+      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s", vid, vid, fid, TSDB_FNAME_SUFFIX[ftype]);
+    } else {
+      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data/v%df%d.%s-ver%" PRIu32, vid, vid, fid,
+               TSDB_FNAME_SUFFIX[ftype], ver);
+    }
+  } else {
+    if (ver == 0) {
+      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s", vid, TSDB_FNAME_SUFFIX[ftype]);
+    } else {
+      snprintf(fname, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/%s-ver%" PRIu32, vid, TSDB_FNAME_SUFFIX[ftype], ver);
+    }
+  }
 }
\ No newline at end of file

src/tsdb/src/tsdbMain.c

@@ -14,130 +14,108 @@
  */
 
 // no test file errors here
-#include "tsdbMain.h"
-#include "os.h"
-#include "talgo.h"
-#include "taosdef.h"
-#include "tchecksum.h"
-#include "tscompression.h"
-#include "tsdb.h"
-#include "tulog.h"
-
-#define TSDB_CFG_FILE_NAME "config"
-#define TSDB_DATA_DIR_NAME "data"
-#define TSDB_META_FILE_NAME "meta"
-#define TSDB_META_FILE_INDEX 10000000
+#include "tsdbint.h"
+
 #define IS_VALID_PRECISION(precision) \
   (((precision) >= TSDB_TIME_PRECISION_MILLI) && ((precision) <= TSDB_TIME_PRECISION_NANO))
 #define TSDB_DEFAULT_COMPRESSION TWO_STAGE_COMP
 #define IS_VALID_COMPRESSION(compression) (((compression) >= NO_COMPRESSION) && ((compression) <= TWO_STAGE_COMP))
 
-static int32_t     tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
-static int32_t     tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg);
-static int32_t     tsdbUnsetRepoEnv(char *rootDir);
-static int32_t     tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg);
-static int         tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg);
-static char *      tsdbGetCfgFname(char *rootDir);
-static STsdbRepo * tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg);
-static void        tsdbFreeRepo(STsdbRepo *pRepo);
-static int         tsdbRestoreInfo(STsdbRepo *pRepo);
-static void        tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression);
-static int         tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep);
-static int         tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks);
-static int         keyFGroupCompFunc(const void *key, const void *fgroup);
-static int         tsdbEncodeCfg(void **buf, STsdbCfg *pCfg);
-static void *      tsdbDecodeCfg(void *buf, STsdbCfg *pCfg);
-static void        tsdbStartStream(STsdbRepo *pRepo);
-static void        tsdbStopStream(STsdbRepo *pRepo);
+static int32_t    tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg);
+static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH);
+static void       tsdbFreeRepo(STsdbRepo *pRepo);
+static void       tsdbStartStream(STsdbRepo *pRepo);
+static void       tsdbStopStream(STsdbRepo *pRepo);
 
 // Function declaration
-int32_t tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg) {
-  DIR *dir = opendir(rootDir);
-  if (dir) {
-    tsdbDebug("repository %s already exists", rootDir);
-    closedir(dir);
-    return 0;
-  } else {
-    if (ENOENT != errno) {
-      tsdbError("failed to open directory %s since %s", rootDir, strerror(errno));
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      return -1;
-    }
-  }
+int32_t tsdbCreateRepo(int repoid) {
+  char tsdbDir[TSDB_FILENAME_LEN] = "\0";
+  char dataDir[TSDB_FILENAME_LEN] = "\0";
 
-  if (mkdir(rootDir, 0755) < 0) {
-    tsdbError("vgId:%d failed to create rootDir %s since %s", pCfg->tsdbId, rootDir, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
+  tsdbGetRootDir(repoid, tsdbDir);
+  if (tfsMkdir(tsdbDir) < 0) {
+    goto _err;
   }
 
-  if (tsdbCheckAndSetDefaultCfg(pCfg) < 0) return -1;
+  tsdbGetDataDir(repoid, dataDir);
+  if (tfsMkdir(dataDir) < 0) {
+    goto _err;
+  }
 
-  if (tsdbSetRepoEnv(rootDir, pCfg) < 0) return -1;
+  // TODO: need to create current file with nothing in
 
-  tsdbDebug(
-      "vgId:%d tsdb env create succeed! cacheBlockSize %d totalBlocks %d daysPerFile %d keep "
-      "%d minRowsPerFileBlock %d maxRowsPerFileBlock %d precision %d compression %d update %d cacheLastRow %d",
-      pCfg->tsdbId, pCfg->cacheBlockSize, pCfg->totalBlocks, pCfg->daysPerFile, pCfg->keep, pCfg->minRowsPerFileBlock,
-      pCfg->maxRowsPerFileBlock, pCfg->precision, pCfg->compression, pCfg->update, pCfg->cacheLastRow);
   return 0;
+
+_err:
+  tsdbError("vgId:%d failed to create TSDB repository since %s", repoid, tstrerror(terrno));
+  return -1;
 }
 
-int32_t tsdbDropRepo(char *rootDir) { return tsdbUnsetRepoEnv(rootDir); }
+int32_t tsdbDropRepo(int repoid) {
+  char tsdbDir[TSDB_FILENAME_LEN] = "\0";
 
-TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH) {
-  STsdbCfg   config = {0};
-  STsdbRepo *pRepo = NULL;
+  tsdbGetRootDir(repoid, tsdbDir);
+  return tfsRmdir(tsdbDir);
+}
+
+STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) {
+  STsdbRepo *pRepo;
+  STsdbCfg   config = *pCfg;
 
   terrno = TSDB_CODE_SUCCESS;
 
-  if (tsdbLoadConfig(rootDir, &config) < 0) {
-    tsdbError("failed to open repo in rootDir %s since %s", rootDir, tstrerror(terrno));
+  // Check and set default configurations
+  if (tsdbCheckAndSetDefaultCfg(&config) < 0) {
+    tsdbError("vgId:%d failed to open TSDB repository since %s", config.tsdbId, tstrerror(terrno));
     return NULL;
   }
 
-  pRepo = tsdbNewRepo(rootDir, pAppH, &config);
-  if (pRepo == NULL) {
-    tsdbError("failed to open repo in rootDir %s since %s", rootDir, tstrerror(terrno));
+  // Create new TSDB object
+  if ((pRepo = tsdbNewRepo(&config, pAppH)) == NULL) {
+    tsdbError("vgId:%d failed to open TSDB repository while creating TSDB object since %s", config.tsdbId,
+              tstrerror(terrno));
     return NULL;
   }
 
+  // Open meta
   if (tsdbOpenMeta(pRepo) < 0) {
-    tsdbError("vgId:%d failed to open meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+    tsdbError("vgId:%d failed to open TSDB repository while opening Meta since %s", config.tsdbId, tstrerror(terrno));
+    tsdbCloseRepo(pRepo, false);
+    return NULL;
   }
 
   if (tsdbOpenBufPool(pRepo) < 0) {
-    tsdbError("vgId:%d failed to open buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+    tsdbError("vgId:%d failed to open TSDB repository while opening buffer pool since %s", config.tsdbId,
+              tstrerror(terrno));
+    tsdbCloseRepo(pRepo, false);
+    return NULL;
   }
 
-  if (tsdbOpenFileH(pRepo) < 0) {
-    tsdbError("vgId:%d failed to open file handle since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+  if (tsdbOpenFS(pRepo) < 0) {
+    tsdbError("vgId:%d failed to open TSDB repository while opening FS since %s", config.tsdbId, tstrerror(terrno));
+    tsdbCloseRepo(pRepo, false);
+    return NULL;
   }
 
-  if (tsdbRestoreInfo(pRepo) < 0) {
-    tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+  // TODO: Restore information from data
+  if ((!(pRepo->state & TSDB_STATE_BAD_DATA)) && tsdbRestoreInfo(pRepo) < 0) {
+    tsdbError("vgId:%d failed to open TSDB repository while restore info since %s", config.tsdbId, tstrerror(terrno));
+    tsdbCloseRepo(pRepo, false);
+    return NULL;
   }
 
   tsdbStartStream(pRepo);
 
-  tsdbDebug("vgId:%d open tsdb repository succeed!", REPO_ID(pRepo));
-
-  return (TSDB_REPO_T *)pRepo;
+  tsdbDebug("vgId:%d, TSDB repository opened", REPO_ID(pRepo));
 
-_err:
-  tsdbCloseRepo(pRepo, false);
-  return NULL;
+  return pRepo;
 }
 
 // Note: all working thread and query thread must stopped when calling this function
-int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
+int tsdbCloseRepo(STsdbRepo *repo, int toCommit) {
   if (repo == NULL) return 0;
 
-  STsdbRepo *pRepo = (STsdbRepo *)repo;
+  STsdbRepo *pRepo = repo;
   int        vgId = REPO_ID(pRepo);
 
   terrno = TSDB_CODE_SUCCESS;
@@ -145,16 +123,15 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
   tsdbStopStream(pRepo);
 
   if (toCommit) {
-    tsdbAsyncCommit(pRepo);
-    tsem_wait(&(pRepo->readyToCommit));
-    terrno = pRepo->code;
+    tsdbSyncCommit(repo);
   }
+
   tsdbUnRefMemTable(pRepo, pRepo->mem);
   tsdbUnRefMemTable(pRepo, pRepo->imem);
   pRepo->mem = NULL;
   pRepo->imem = NULL;
 
-  tsdbCloseFileH(pRepo);
+  tsdbCloseFS(pRepo);
   tsdbCloseBufPool(pRepo);
   tsdbCloseMeta(pRepo);
   tsdbFreeRepo(pRepo);
@@ -167,88 +144,67 @@ int tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit) {
   }
 }
 
-uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) {
-  STsdbRepo *pRepo = (STsdbRepo *)repo;
-  // STsdbMeta *pMeta = pRepo->tsdbMeta;
-  STsdbFileH *pFileH = pRepo->tsdbFileH;
-  uint32_t    magic = 0;
-  char *      fname = NULL;
-
-  struct stat fState;
+STsdbCfg *tsdbGetCfg(const STsdbRepo *repo) {
+  ASSERT(repo != NULL);
+  return &((STsdbRepo *)repo)->config;
+}
 
-  tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex);
-  ASSERT(*index <= eindex);
+int tsdbLockRepo(STsdbRepo *pRepo) {
+  int code = pthread_mutex_lock(&pRepo->mutex);
+  if (code != 0) {
+    tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  pRepo->repoLocked = true;
+  return 0;
+}
 
-  char *sdup = strdup(pRepo->rootDir);
-  char *prefix = dirname(sdup);
-  int   prefixLen = (int)strlen(prefix);
+int tsdbUnlockRepo(STsdbRepo *pRepo) {
+  ASSERT(IS_REPO_LOCKED(pRepo));
+  pRepo->repoLocked = false;
+  int code = pthread_mutex_unlock(&pRepo->mutex);
+  if (code != 0) {
+    tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
 
-  if (name[0] == 0) {  // get the file from index or after, but not larger than eindex
-    tfree(sdup);
-    int fid = (*index) / TSDB_FILE_TYPE_MAX;
+int tsdbCheckCommit(STsdbRepo *pRepo) {
+  ASSERT(pRepo->mem != NULL);
+  STsdbCfg *pCfg = &(pRepo->config);
 
-    if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) {
-      if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) {
-        fname = tsdbGetMetaFileName(pRepo->rootDir);
-        *index = TSDB_META_FILE_INDEX;
-        magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta);
-      } else {
-        return 0;
-      }
-    } else {
-      SFileGroup *pFGroup =
-          taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE);
-      if (pFGroup->fileId == fid) {
-        fname = strdup(pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].fname);
-        magic = pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX].info.magic;
-      } else {
-        if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) {
-          fname = strdup(pFGroup->files[0].fname);
-          *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX;
-          magic = pFGroup->files[0].info.magic;
-        } else {
-          return 0;
-        }
-      }
-    }
-    strcpy(name, fname + prefixLen);
-  } else {  // get the named file at the specified index. If not there, return 0
-    fname = malloc(prefixLen + strlen(name) + 2);
-    sprintf(fname, "%s/%s", prefix, name);
-    if (access(fname, F_OK) != 0) {
-      tfree(fname);
-      tfree(sdup);
-      return 0;
-    }
-    if (*index == TSDB_META_FILE_INDEX) {  // get meta file
-      tsdbGetStoreInfo(fname, &magic, size);
-    } else {
-      tsdbGetFileInfoImpl(fname, &magic, size);
-    }
-    tfree(fname);
-    tfree(sdup);
-    return magic;
+  STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
+  ASSERT(pBufBlock != NULL);
+  if ((pRepo->mem->extraBuffList != NULL) ||
+      ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) {
+    // trigger commit
+    if (tsdbAsyncCommit(pRepo) < 0) return -1;
   }
 
-  if (stat(fname, &fState) < 0) {
-    tfree(fname);
-    return 0;
-  }
+  return 0;
+}
 
-  *size = fState.st_size;
-  // magic = *size;
+STsdbMeta *tsdbGetMeta(STsdbRepo *pRepo) { return pRepo->tsdbMeta; }
 
-  tfree(fname);
-  return magic;
-}
+STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo) { return NULL; }
 
-STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo) {
+int tsdbGetState(STsdbRepo *repo) { return repo->state; }
+
+void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) {
   ASSERT(repo != NULL);
-  return &((STsdbRepo *)repo)->config;
+  STsdbRepo *pRepo = repo;
+  *totalPoints = pRepo->stat.pointsWritten;
+  *totalStorage = pRepo->stat.totalStorage;
+  *compStorage = pRepo->stat.compStorage;
 }
 
-int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) {
+int32_t tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg) {
   // TODO: think about multithread cases
+  return 0;
+#if 0
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   STsdbCfg   config = pRepo->config;
   STsdbCfg * pRCfg = &pRepo->config;
@@ -294,112 +250,115 @@ int32_t tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg) {
   }
 
   return 0;
+#endif
 }
 
-void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int64_t *compStorage) {
-  ASSERT(repo != NULL);
-  STsdbRepo *pRepo = repo;
-  *totalPoints = pRepo->stat.pointsWritten;
-  *totalStorage = pRepo->stat.totalStorage;
-  *compStorage = pRepo->stat.compStorage;
-}
-
-int tsdbGetState(TSDB_REPO_T *repo) {
-  return ((STsdbRepo *)repo)->state;
-}
+uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size) {
+  // TODO
+  return 0;
+#if 0
+  STsdbRepo *pRepo = (STsdbRepo *)repo;
+  // STsdbMeta *pMeta = pRepo->tsdbMeta;
+  STsdbFileH *pFileH = pRepo->tsdbFileH;
+  uint32_t    magic = 0;
+  char *      fname = NULL;
 
-// ----------------- INTERNAL FUNCTIONS -----------------
-char *tsdbGetMetaFileName(char *rootDir) {
-  int   tlen = (int)(strlen(rootDir) + strlen(TSDB_META_FILE_NAME) + 2);
-  char *fname = calloc(1, tlen);
-  if (fname == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return NULL;
-  }
+  struct stat fState;
 
-  snprintf(fname, tlen, "%s/%s", rootDir, TSDB_META_FILE_NAME);
-  return fname;
-}
+  tsdbDebug("vgId:%d name:%s index:%d eindex:%d", pRepo->config.tsdbId, name, *index, eindex);
+  ASSERT(*index <= eindex);
 
-void tsdbGetDataFileName(char *rootDir, int vid, int fid, int type, char *fname) {
-  snprintf(fname, TSDB_FILENAME_LEN, "%s/%s/v%df%d%s", rootDir, TSDB_DATA_DIR_NAME, vid, fid, tsdbFileSuffix[type]);
-}
+  if (name[0] == 0) {  // get the file from index or after, but not larger than eindex
+    int fid = (*index) / TSDB_FILE_TYPE_MAX;
 
-int tsdbLockRepo(STsdbRepo *pRepo) {
-  int code = pthread_mutex_lock(&pRepo->mutex);
-  if (code != 0) {
-    tsdbError("vgId:%d failed to lock tsdb since %s", REPO_ID(pRepo), strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
+    if (pFileH->nFGroups == 0 || fid > pFileH->pFGroup[pFileH->nFGroups - 1].fileId) {
+      if (*index <= TSDB_META_FILE_INDEX && TSDB_META_FILE_INDEX <= eindex) {
+        fname = tsdbGetMetaFileName(pRepo->rootDir);
+        *index = TSDB_META_FILE_INDEX;
+        magic = TSDB_META_FILE_MAGIC(pRepo->tsdbMeta);
+        sprintf(name, "tsdb/%s", TSDB_META_FILE_NAME);
+      } else {
+        return 0;
+      }
+    } else {
+      SFileGroup *pFGroup =
+          taosbsearch(&fid, pFileH->pFGroup, pFileH->nFGroups, sizeof(SFileGroup), keyFGroupCompFunc, TD_GE);
+      if (pFGroup->fileId == fid) {
+        SFile *pFile = &pFGroup->files[(*index) % TSDB_FILE_TYPE_MAX];
+        fname = strdup(TSDB_FILE_NAME(pFile));
+        magic = pFile->info.magic;
+        char *tfname = strdup(fname);
+        sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname));
+        tfree(tfname);
+      } else {
+        if ((pFGroup->fileId + 1) * TSDB_FILE_TYPE_MAX - 1 < (int)eindex) {
+          SFile *pFile = &pFGroup->files[0];
+          fname = strdup(TSDB_FILE_NAME(pFile));
+          *index = pFGroup->fileId * TSDB_FILE_TYPE_MAX;
+          magic = pFile->info.magic;
+          char *tfname = strdup(fname);
+          sprintf(name, "tsdb/%s/%s", TSDB_DATA_DIR_NAME, basename(tfname));
+          tfree(tfname);
+        } else {
+          return 0;
+        }
+      }
+    }
+  } else {  // get the named file at the specified index. If not there, return 0
+    fname = malloc(256);
+    sprintf(fname, "%s/vnode/vnode%d/%s", TFS_PRIMARY_PATH(), REPO_ID(pRepo), name);
+    if (access(fname, F_OK) != 0) {
+      tfree(fname);
+      return 0;
+    }
+    if (*index == TSDB_META_FILE_INDEX) {  // get meta file
+      tsdbGetStoreInfo(fname, &magic, size);
+    } else {
+      char tfname[TSDB_FILENAME_LEN] = "\0";
+      sprintf(tfname, "vnode/vnode%d/tsdb/%s/%s", REPO_ID(pRepo), TSDB_DATA_DIR_NAME, basename(name));
+      tsdbGetFileInfoImpl(tfname, &magic, size);
+    }
+    tfree(fname);
+    return magic;
   }
-  pRepo->repoLocked = true;
-  return 0;
-}
 
-int tsdbUnlockRepo(STsdbRepo *pRepo) {
-  ASSERT(IS_REPO_LOCKED(pRepo));
-  pRepo->repoLocked = false;
-  int code = pthread_mutex_unlock(&pRepo->mutex);
-  if (code != 0) {
-    tsdbError("vgId:%d failed to unlock tsdb since %s", REPO_ID(pRepo), strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(code);
-    return -1;
+  if (stat(fname, &fState) < 0) {
+    tfree(fname);
+    return 0;
   }
-  return 0;
-}
 
-char *tsdbGetDataDirName(char *rootDir) {
-  int   tlen = (int)(strlen(rootDir) + strlen(TSDB_DATA_DIR_NAME) + 2);
-  char *fname = calloc(1, tlen);
-  if (fname == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return NULL;
-  }
+  *size = fState.st_size;
+  // magic = *size;
 
-  snprintf(fname, tlen, "%s/%s", rootDir, TSDB_DATA_DIR_NAME);
-  return fname;
+  tfree(fname);
+  return magic;
+#endif
 }
 
-int tsdbGetNextMaxTables(int tid) {
-  ASSERT(tid >= 1 && tid <= TSDB_MAX_TABLES);
-  int maxTables = TSDB_INIT_NTABLES;
-  while (true) {
-    maxTables = MIN(maxTables, TSDB_MAX_TABLES);
-    if (tid <= maxTables) break;
-    maxTables *= 2;
-  }
-
-  return maxTables + 1;
+void tsdbGetRootDir(int repoid, char dirName[]) {
+  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb", repoid);
 }
 
-int tsdbCheckCommit(STsdbRepo *pRepo) {
-  ASSERT(pRepo->mem != NULL);
-  STsdbCfg *pCfg = &(pRepo->config);
-
-  STsdbBufBlock *pBufBlock = tsdbGetCurrBufBlock(pRepo);
-  ASSERT(pBufBlock != NULL);
-  if ((pRepo->mem->extraBuffList != NULL) ||
-      ((listNEles(pRepo->mem->bufBlockList) >= pCfg->totalBlocks / 3) && (pBufBlock->remain < TSDB_BUFFER_RESERVE))) {
-    // trigger commit
-    if (tsdbAsyncCommit(pRepo) < 0) return -1;
-  }
-
-  return 0;
+void tsdbGetDataDir(int repoid, char dirName[]) {
+  snprintf(dirName, TSDB_FILENAME_LEN, "vnode/vnode%d/tsdb/data", repoid);
 }
 
-STsdbMeta *    tsdbGetMeta(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbMeta; }
-STsdbFileH *   tsdbGetFile(TSDB_REPO_T *pRepo) { return ((STsdbRepo *)pRepo)->tsdbFileH; }
-STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo) { return NULL; }
-
-// ----------------- LOCAL FUNCTIONS -----------------
 static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
+  // Check tsdbId
+  if (pCfg->tsdbId < 0) {
+    tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId);
+    terrno = TSDB_CODE_TDB_INVALID_CONFIG;
+    return -1;
+  }
+
   // Check precision
   if (pCfg->precision == -1) {
     pCfg->precision = TSDB_DEFAULT_PRECISION;
   } else {
     if (!IS_VALID_PRECISION(pCfg->precision)) {
       tsdbError("vgId:%d invalid precision configuration %d", pCfg->tsdbId, pCfg->precision);
-      goto _err;
+      terrno = TSDB_CODE_TDB_INVALID_CONFIG;
+      return -1;
     }
   }
 
@@ -409,16 +368,11 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
   } else {
     if (!IS_VALID_COMPRESSION(pCfg->compression)) {
       tsdbError("vgId:%d invalid compression configuration %d", pCfg->tsdbId, pCfg->precision);
-      goto _err;
+      terrno = TSDB_CODE_TDB_INVALID_CONFIG;
+      return -1;
     }
   }
 
-  // Check tsdbId
-  if (pCfg->tsdbId < 0) {
-    tsdbError("vgId:%d invalid vgroup ID", pCfg->tsdbId);
-    goto _err;
-  }
-
   // Check daysPerFile
   if (pCfg->daysPerFile == -1) {
     pCfg->daysPerFile = TSDB_DEFAULT_DAYS_PER_FILE;
@@ -428,7 +382,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
           "vgId:%d invalid daysPerFile configuration! daysPerFile %d TSDB_MIN_DAYS_PER_FILE %d TSDB_MAX_DAYS_PER_FILE "
           "%d",
           pCfg->tsdbId, pCfg->daysPerFile, TSDB_MIN_DAYS_PER_FILE, TSDB_MAX_DAYS_PER_FILE);
-      goto _err;
+      terrno = TSDB_CODE_TDB_INVALID_CONFIG;
+      return -1;
     }
   }
 
@@ -441,7 +396,8 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
           "vgId:%d invalid minRowsPerFileBlock configuration! minRowsPerFileBlock %d TSDB_MIN_MIN_ROW_FBLOCK %d "
           "TSDB_MAX_MIN_ROW_FBLOCK %d",
           pCfg->tsdbId, pCfg->minRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
-      goto _err;
+      terrno = TSDB_CODE_TDB_INVALID_CONFIG;
+      return -1;
     }
   }
 
@@ -453,14 +409,16 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
           "vgId:%d invalid maxRowsPerFileBlock configuration! maxRowsPerFileBlock %d TSDB_MIN_MAX_ROW_FBLOCK %d "
           "TSDB_MAX_MAX_ROW_FBLOCK %d",
           pCfg->tsdbId, pCfg->maxRowsPerFileBlock, TSDB_MIN_MIN_ROW_FBLOCK, TSDB_MAX_MIN_ROW_FBLOCK);
-      goto _err;
+      terrno = TSDB_CODE_TDB_INVALID_CONFIG;
+      return -1;
     }
   }
 
   if (pCfg->minRowsPerFileBlock > pCfg->maxRowsPerFileBlock) {
     tsdbError("vgId:%d invalid configuration! minRowsPerFileBlock %d maxRowsPerFileBlock %d", pCfg->tsdbId,
               pCfg->minRowsPerFileBlock, pCfg->maxRowsPerFileBlock);
-    goto _err;
+    terrno = TSDB_CODE_TDB_INVALID_CONFIG;
+    return -1;
   }
 
   // Check keep
@@ -472,457 +430,193 @@ static int32_t tsdbCheckAndSetDefaultCfg(STsdbCfg *pCfg) {
           "vgId:%d invalid keep configuration! keep %d TSDB_MIN_KEEP %d "
           "TSDB_MAX_KEEP %d",
           pCfg->tsdbId, pCfg->keep, TSDB_MIN_KEEP, TSDB_MAX_KEEP);
-      goto _err;
+      terrno = TSDB_CODE_TDB_INVALID_CONFIG;
+      return -1;
     }
   }
 
-  // update check
-  if (pCfg->update != 0) pCfg->update = 1;
-
-  // update cacheLastRow
-  if (pCfg->cacheLastRow != 0) pCfg->cacheLastRow = 1;
-
-  return 0;
-
-_err:
-  terrno = TSDB_CODE_TDB_INVALID_CONFIG;
-  return -1;
-}
-
-static int32_t tsdbSetRepoEnv(char *rootDir, STsdbCfg *pCfg) {
-  if (tsdbSaveConfig(rootDir, pCfg) < 0) {
-    tsdbError("vgId:%d failed to set TSDB environment since %s", pCfg->tsdbId, tstrerror(terrno));
-    return -1;
-  }
-
-  char *dirName = tsdbGetDataDirName(rootDir);
-  if (dirName == NULL) return -1;
-
-  if (mkdir(dirName, 0755) < 0) {
-    tsdbError("vgId:%d failed to create directory %s since %s", pCfg->tsdbId, dirName, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    free(dirName);
-    return -1;
+  if (pCfg->keep1 == 0) {
+    pCfg->keep1 = pCfg->keep;
   }
 
-  free(dirName);
-
-  char *fname = tsdbGetMetaFileName(rootDir);
-  if (fname == NULL) return -1;
-  if (tdCreateKVStore(fname) < 0) {
-    tsdbError("vgId:%d failed to open KV store since %s", pCfg->tsdbId, tstrerror(terrno));
-    free(fname);
-    return -1;
-  }
-
-  free(fname);
-  return 0;
-}
-
-static int32_t tsdbUnsetRepoEnv(char *rootDir) {
-  taosRemoveDir(rootDir);
-  tsdbDebug("repository %s is removed", rootDir);
-  return 0;
-}
-
-static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
-  int   fd = -1;
-  char *fname = NULL;
-  char  buf[TSDB_FILE_HEAD_SIZE] = "\0";
-  char *pBuf = buf;
-
-  fname = tsdbGetCfgFname(rootDir);
-  if (fname == NULL) {
-    tsdbError("vgId:%d failed to save configuration since %s", pCfg->tsdbId, tstrerror(terrno));
-    goto _err;
-  }
-
-  fd = open(fname, O_WRONLY | O_CREAT | O_BINARY, 0755);
-  if (fd < 0) {
-    tsdbError("vgId:%d failed to open file %s since %s", pCfg->tsdbId, fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  int tlen = tsdbEncodeCfg((void *)(&pBuf), pCfg);
-  ASSERT((tlen + sizeof(TSCKSUM) <= TSDB_FILE_HEAD_SIZE) && (POINTER_DISTANCE(pBuf, buf) == tlen));
-
-  taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
-
-  if (taosWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
-    tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname,
-              strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
+  if (pCfg->keep2 == 0) {
+    pCfg->keep2 = pCfg->keep;
   }
 
-  if (fsync(fd) < 0) {
-    tsdbError("vgId:%d failed to fsync file %s since %s", pCfg->tsdbId, fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  free(fname);
-  close(fd);
-  return 0;
-
-_err:
-  tfree(fname);
-  if (fd >= 0) close(fd);
-  return -1;
-}
-
-static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
-  char *fname = NULL;
-  int   fd = -1;
-  char  buf[TSDB_FILE_HEAD_SIZE] = "\0";
-
-  fname = tsdbGetCfgFname(rootDir);
-  if (fname == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  fd = open(fname, O_RDONLY | O_BINARY);
-  if (fd < 0) {
-    tsdbError("failed to open file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (taosRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
-    tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (!taosCheckChecksumWhole((uint8_t *)buf, TSDB_FILE_HEAD_SIZE)) {
-    tsdbError("file %s is corrupted", fname);
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    goto _err;
-  }
-
-  tsdbDecodeCfg(buf, pCfg);
+  // update check
+  if (pCfg->update != 0) pCfg->update = 1;
 
-  tfree(fname);
-  close(fd);
+  // update cacheLastRow
+  if (pCfg->cacheLastRow != 0) pCfg->cacheLastRow = 1;
 
   return 0;
-
-_err:
-  tfree(fname);
-  if (fd >= 0) close(fd);
-  return -1;
-}
-
-static char *tsdbGetCfgFname(char *rootDir) {
-  int   tlen = (int)(strlen(rootDir) + strlen(TSDB_CFG_FILE_NAME) + 2);
-  char *fname = calloc(1, tlen);
-  if (fname == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return NULL;
-  }
-
-  snprintf(fname, tlen, "%s/%s", rootDir, TSDB_CFG_FILE_NAME);
-  return fname;
 }
 
-static STsdbRepo *tsdbNewRepo(char *rootDir, STsdbAppH *pAppH, STsdbCfg *pCfg) {
-  STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(STsdbRepo));
+static STsdbRepo *tsdbNewRepo(STsdbCfg *pCfg, STsdbAppH *pAppH) {
+  STsdbRepo *pRepo = (STsdbRepo *)calloc(1, sizeof(*pRepo));
   if (pRepo == NULL) {
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
+    return NULL;
   }
 
   pRepo->state = TSDB_STATE_OK;
   pRepo->code = TSDB_CODE_SUCCESS;
+  pRepo->config = *pCfg;
+  if (pAppH) {
+    pRepo->appH = *pAppH;
+  }
+  pRepo->repoLocked = false;
 
-  int code = pthread_mutex_init(&pRepo->mutex, NULL);
+  int code = pthread_mutex_init(&(pRepo->mutex), NULL);
   if (code != 0) {
     terrno = TAOS_SYSTEM_ERROR(code);
-    goto _err;
+    tsdbFreeRepo(pRepo);
+    return NULL;
   }
 
   code = tsem_init(&(pRepo->readyToCommit), 0, 1);
   if (code != 0) {
     code = errno;
     terrno = TAOS_SYSTEM_ERROR(code);
-    goto _err;
-  }
-
-  pRepo->repoLocked = false;
-
-  pRepo->rootDir = strdup(rootDir);
-  if (pRepo->rootDir == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
+    tsdbFreeRepo(pRepo);
+    return NULL;
   }
 
-  pRepo->config = *pCfg;
-  if (pAppH) pRepo->appH = *pAppH;
-
   pRepo->tsdbMeta = tsdbNewMeta(pCfg);
   if (pRepo->tsdbMeta == NULL) {
     tsdbError("vgId:%d failed to create meta since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+    tsdbFreeRepo(pRepo);
+    return NULL;
   }
 
   pRepo->pPool = tsdbNewBufPool(pCfg);
   if (pRepo->pPool == NULL) {
     tsdbError("vgId:%d failed to create buffer pool since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+    tsdbFreeRepo(pRepo);
+    return NULL;
   }
 
-  pRepo->tsdbFileH = tsdbNewFileH(pCfg);
-  if (pRepo->tsdbFileH == NULL) {
-    tsdbError("vgId:%d failed to create file handle since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
+  pRepo->fs = tsdbNewFS(pCfg);
+  if (pRepo->fs == NULL) {
+    tsdbError("vgId:%d failed to TSDB file system since %s", REPO_ID(pRepo), tstrerror(terrno));
+    tsdbFreeRepo(pRepo);
+    return NULL;
   }
 
   return pRepo;
-
-_err:
-  tsdbFreeRepo(pRepo);
-  return NULL;
 }
 
 static void tsdbFreeRepo(STsdbRepo *pRepo) {
   if (pRepo) {
-    tsdbFreeFileH(pRepo->tsdbFileH);
+    tsdbFreeFS(pRepo->fs);
     tsdbFreeBufPool(pRepo->pPool);
     tsdbFreeMeta(pRepo->tsdbMeta);
     // tsdbFreeMemTable(pRepo->mem);
     // tsdbFreeMemTable(pRepo->imem);
-    tfree(pRepo->rootDir);
     tsem_destroy(&(pRepo->readyToCommit));
     pthread_mutex_destroy(&pRepo->mutex);
     free(pRepo);
   }
 }
 
-static int tsdbRestoreInfo(STsdbRepo *pRepo) { // TODO
-  STsdbMeta * pMeta = pRepo->tsdbMeta;
-  STsdbFileH *pFileH = pRepo->tsdbFileH;
-  SFileGroup *pFGroup = NULL;
-  STsdbCfg *  pCfg = &(pRepo->config);
-  SCompBlock *pBlock = NULL;
+static void tsdbStartStream(STsdbRepo *pRepo) {
+  STsdbMeta *pMeta = pRepo->tsdbMeta;
+
+  for (int i = 0; i < pMeta->maxTables; i++) {
+    STable *pTable = pMeta->tables[i];
+    if (pTable && pTable->type == TSDB_STREAM_TABLE) {
+      pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql,
+                                                     tsdbGetTableSchemaImpl(pTable, false, false, -1));
+    }
+  }
+}
+
+static void tsdbStopStream(STsdbRepo *pRepo) {
+  STsdbMeta *pMeta = pRepo->tsdbMeta;
+
+  for (int i = 0; i < pMeta->maxTables; i++) {
+    STable *pTable = pMeta->tables[i];
+    if (pTable && pTable->type == TSDB_STREAM_TABLE) {
+      (*pRepo->appH.cqDropFunc)(pTable->cqhandle);
+    }
+  }
+}
+
+int tsdbRestoreInfo(STsdbRepo *pRepo) {
+  SFSIter    fsiter;
+  SReadH     readh;
+  SDFileSet *pSet;
+  STsdbMeta *pMeta = pRepo->tsdbMeta;
+  STsdbCfg * pCfg = REPO_CFG(pRepo);
+  SBlock *   pBlock;
 
-  SFileGroupIter iter;
-  SRWHelper      rhelper = {0};
+  if (tsdbInitReadH(&readh, pRepo) < 0) {
+    return -1;
+  }
 
-  if (tsdbInitReadHelper(&rhelper, pRepo) < 0) goto _err;
+  tsdbFSIterInit(&fsiter, REPO_FS(pRepo), TSDB_FS_ITER_BACKWARD);
+
+  while ((pSet = tsdbFSIterNext(&fsiter)) != NULL) {
+    if (tsdbSetAndOpenReadFSet(&readh, pSet) < 0) {
+      tsdbDestroyReadH(&readh);
+      return -1;
+    }
+
+    if (tsdbLoadBlockIdx(&readh) < 0) {
+      tsdbDestroyReadH(&readh);
+      return -1;
+    }
 
-  tsdbInitFileGroupIter(pFileH, &iter, TSDB_ORDER_DESC);
-  while ((pFGroup = tsdbGetFileGroupNext(&iter)) != NULL) {
-    if (pFGroup->state) continue;
-    if (tsdbSetAndOpenHelperFile(&rhelper, pFGroup) < 0) goto _err;
-    if (tsdbLoadCompIdx(&rhelper, NULL) < 0) goto _err;
     for (int i = 1; i < pMeta->maxTables; i++) {
       STable *pTable = pMeta->tables[i];
       if (pTable == NULL) continue;
-      if (tsdbSetHelperTable(&rhelper, pTable, pRepo) < 0) goto _err;
-      SCompIdx *pIdx = &(rhelper.curCompIdx);
 
-      TSKEY lastKey = tsdbGetTableLastKeyImpl(pTable);
-      if (pIdx->offset > 0 && lastKey < pIdx->maxKey) {
+      if (tsdbSetReadTable(&readh, pTable) < 0) {
+        tsdbDestroyReadH(&readh);
+        return -1;
+      }
+
+      TSKEY      lastKey = tsdbGetTableLastKeyImpl(pTable);
+      SBlockIdx *pIdx = readh.pBlkIdx;
+      if (pIdx && lastKey < pIdx->maxKey) {
         pTable->lastKey = pIdx->maxKey;
-        if (pCfg->cacheLastRow) { // load the block of data
-          if (tsdbLoadCompInfo(&rhelper, NULL) < 0) goto _err;
 
-          pBlock = rhelper.pCompInfo->blocks + pIdx->numOfBlocks - 1;
-          if (tsdbLoadBlockData(&rhelper, pBlock, NULL) < 0) goto _err;
+        if (pCfg->cacheLastRow) {
+          if (tsdbLoadBlockInfo(&readh, NULL) < 0) {
+            tsdbDestroyReadH(&readh);
+            return -1;
+          }
 
-          // construct the data row
+          pBlock = readh.pBlkInfo->blocks + pIdx->numOfBlocks - 1;
+
+          if (tsdbLoadBlockData(&readh, pBlock, NULL) < 0) {
+            tsdbDestroyReadH(&readh);
+            return -1;
+          }
+
+          // Get the data in row
           ASSERT(pTable->lastRow == NULL);
           STSchema *pSchema = tsdbGetTableSchema(pTable);
           pTable->lastRow = taosTMalloc(schemaTLen(pSchema));
           if (pTable->lastRow == NULL) {
-            goto _err;
+            terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+            tsdbDestroyReadH(&readh);
+            return -1;
           }
 
           tdInitDataRow(pTable->lastRow, pSchema);
           for (int icol = 0; icol < schemaNCols(pSchema); icol++) {
             STColumn *pCol = schemaColAt(pSchema, icol);
-            SDataCol *pDataCol = rhelper.pDataCols[0]->cols + icol;
+            SDataCol *pDataCol = readh.pDCols[0]->cols + icol;
             tdAppendColVal(pTable->lastRow, tdGetColDataOfRow(pDataCol, pBlock->numOfRows - 1), pCol->type, pCol->bytes,
                            pCol->offset);
           }
         }
       }
+      
     }
   }
 
-  tsdbDestroyHelper(&rhelper);
+  tsdbDestroyReadH(&readh);
   return 0;
-
-_err:
+}
\ No newline at end of file
-  tsdbDestroyHelper(&rhelper);
-  return -1;
-}
-
-static void tsdbAlterCompression(STsdbRepo *pRepo, int8_t compression) {
-  int8_t ocompression = pRepo->config.compression;
-  pRepo->config.compression = compression;
-  tsdbDebug("vgId:%d tsdb compression is changed from %d to %d", REPO_ID(pRepo), ocompression, compression);
-}
-
-static int tsdbAlterKeep(STsdbRepo *pRepo, int32_t keep) {
-  STsdbCfg *  pCfg = &pRepo->config;
-  STsdbFileH *pFileH = pRepo->tsdbFileH;
-  int         okeep = pCfg->keep;
-  SFileGroup *pFGroup = NULL;
-
-  ASSERT(pCfg->keep != keep);
-  int maxFiles = TSDB_MAX_FILE(keep, pCfg->daysPerFile);
-  
-  if (maxFiles != pFileH->maxFGroups) {
-    pthread_rwlock_wrlock(&(pFileH->fhlock));
-
-    pCfg->keep = keep;
-    pFGroup = (SFileGroup *)calloc(maxFiles, sizeof(SFileGroup));
-    if (pFGroup == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      pthread_rwlock_unlock(&(pFileH->fhlock));
-      return -1;
-    }
-
-    int mfid = (int)(TSDB_KEY_FILEID(taosGetTimestamp(pCfg->precision), pCfg->daysPerFile, pCfg->precision) -
-               TSDB_MAX_FILE(keep, pCfg->daysPerFile));
-
-    int i = 0;
-    for (; i < pFileH->nFGroups; i++) {
-      if (pFileH->pFGroup[i].fileId >= mfid) break;
-      tsdbRemoveFileGroup(pRepo, &(pFileH->pFGroup[i]));
-    }
-
-    for (int j = 0; i < pFileH->nFGroups; i++, j++) {
-      pFGroup[j] = pFileH->pFGroup[i];
-    }
-
-    free(pFileH->pFGroup);
-    pFileH->pFGroup = pFGroup;
-
-    pthread_rwlock_unlock(&(pFileH->fhlock));
-  }
-
-  tsdbDebug("vgId:%d keep is changed from %d to %d", REPO_ID(pRepo), okeep, keep);
-
-  return 0;
-}
-
-static int keyFGroupCompFunc(const void *key, const void *fgroup) {
-  int         fid = *(int *)key;
-  SFileGroup *pFGroup = (SFileGroup *)fgroup;
-  if (fid == pFGroup->fileId) {
-    return 0;
-  } else {
-    return fid > pFGroup->fileId ? 1 : -1;
-  }
-}
-
-static int tsdbEncodeCfg(void **buf, STsdbCfg *pCfg) {
-  int tlen = 0;
-
-  tlen += taosEncodeVariantI32(buf, pCfg->tsdbId);
-  tlen += taosEncodeFixedI32(buf, pCfg->cacheBlockSize);
-  tlen += taosEncodeVariantI32(buf, pCfg->totalBlocks);
-  tlen += taosEncodeVariantI32(buf, pCfg->daysPerFile);
-  tlen += taosEncodeVariantI32(buf, pCfg->keep);
-  tlen += taosEncodeVariantI32(buf, pCfg->keep1);
-  tlen += taosEncodeVariantI32(buf, pCfg->keep2);
-  tlen += taosEncodeVariantI32(buf, pCfg->minRowsPerFileBlock);
-  tlen += taosEncodeVariantI32(buf, pCfg->maxRowsPerFileBlock);
-  tlen += taosEncodeFixedI8(buf, pCfg->precision);
-  tlen += taosEncodeFixedI8(buf, pCfg->compression);
-  tlen += taosEncodeFixedI8(buf, pCfg->update);
-  tlen += taosEncodeFixedI8(buf, pCfg->cacheLastRow);
-
-  return tlen;
-}
-
-static void *tsdbDecodeCfg(void *buf, STsdbCfg *pCfg) {
-  buf = taosDecodeVariantI32(buf, &(pCfg->tsdbId));
-  buf = taosDecodeFixedI32(buf, &(pCfg->cacheBlockSize));
-  buf = taosDecodeVariantI32(buf, &(pCfg->totalBlocks));
-  buf = taosDecodeVariantI32(buf, &(pCfg->daysPerFile));
-  buf = taosDecodeVariantI32(buf, &(pCfg->keep));
-  buf = taosDecodeVariantI32(buf, &(pCfg->keep1));
-  buf = taosDecodeVariantI32(buf, &(pCfg->keep2));
-  buf = taosDecodeVariantI32(buf, &(pCfg->minRowsPerFileBlock));
-  buf = taosDecodeVariantI32(buf, &(pCfg->maxRowsPerFileBlock));
-  buf = taosDecodeFixedI8(buf, &(pCfg->precision));
-  buf = taosDecodeFixedI8(buf, &(pCfg->compression));
-  buf = taosDecodeFixedI8(buf, &(pCfg->update));
-  buf = taosDecodeFixedI8(buf, &(pCfg->cacheLastRow));
-
-  return buf;
-}
-
-static int tsdbAlterCacheTotalBlocks(STsdbRepo *pRepo, int totalBlocks) {
-  // TODO
-  // STsdbCache *pCache = pRepo->tsdbCache;
-  // int         oldNumOfBlocks = pCache->totalCacheBlocks;
-
-  // tsdbLockRepo((TsdbRepoT *)pRepo);
-
-  // ASSERT(pCache->totalCacheBlocks != totalBlocks);
-
-  // if (pCache->totalCacheBlocks < totalBlocks) {
-  //   ASSERT(pCache->totalCacheBlocks == pCache->pool.numOfCacheBlocks);
-  //   int blocksToAdd = pCache->totalCacheBlocks - totalBlocks;
-  //   pCache->totalCacheBlocks = totalBlocks;
-  //   for (int i = 0; i < blocksToAdd; i++) {
-  //     if (tsdbAddCacheBlockToPool(pCache) < 0) {
-  //       tsdbUnLockRepo((TsdbRepoT *)pRepo);
-  //       tsdbError("tsdbId:%d, failed to add cache block to cache pool", pRepo->config.tsdbId);
-  //       return -1;
-  //     }
-  //   }
-  // } else {
-  //   pCache->totalCacheBlocks = totalBlocks;
-  //   tsdbAdjustCacheBlocks(pCache);
-  // }
-  // pRepo->config.totalBlocks = totalBlocks;
-
-  // tsdbUnLockRepo((TsdbRepoT *)pRepo);
-  // tsdbDebug("vgId:%d, tsdb total cache blocks changed from %d to %d", pRepo->config.tsdbId, oldNumOfBlocks,
-  // totalBlocks);
-  return 0;
-}
-
-#if 0
-
-TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid) {
-  STsdbRepo *pRepo = (STsdbRepo *)repo;
-
-  STable *pTable = tsdbGetTableByUid(pRepo->tsdbMeta, uid);
-  if (pTable == NULL) return -1;
-
-  return TSDB_GET_TABLE_LAST_KEY(pTable);
-}
-
-#endif
-
-static void tsdbStartStream(STsdbRepo *pRepo) {
-  STsdbMeta *pMeta = pRepo->tsdbMeta;
-
-  for (int i = 0; i < pMeta->maxTables; i++) {
-    STable *pTable = pMeta->tables[i];
-    if (pTable && pTable->type == TSDB_STREAM_TABLE) {
-      pTable->cqhandle = (*pRepo->appH.cqCreateFunc)(pRepo->appH.cqH, TABLE_UID(pTable), TABLE_TID(pTable), TABLE_NAME(pTable)->data, pTable->sql,
-                                                     tsdbGetTableSchemaImpl(pTable, false, false, -1));
-    }
-  }
-}
-
-
-static void tsdbStopStream(STsdbRepo *pRepo) {
-  STsdbMeta *pMeta = pRepo->tsdbMeta;
-
-  for (int i = 0; i < pMeta->maxTables; i++) {
-    STable *pTable = pMeta->tables[i];
-    if (pTable && pTable->type == TSDB_STREAM_TABLE) {
-      (*pRepo->appH.cqDropFunc)(pTable->cqhandle);
-    }
-  }
-}

src/tsdb/src/tsdbMemTable.c

@@ -13,12 +13,23 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdb.h"
-#include "tsdbMain.h"
+#include "tsdbint.h"
 
 #define TSDB_DATA_SKIPLIST_LEVEL 5
 #define TSDB_MAX_INSERT_BATCH 512
 
+typedef struct {
+  int32_t  totalLen;
+  int32_t  len;
+  SDataRow row;
+} SSubmitBlkIter;
+
+typedef struct {
+  int32_t totalLen;
+  int32_t len;
+  void *  pMsg;
+} SSubmitMsgIter;
+
 static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo);
 static void        tsdbFreeMemTable(SMemTable *pMemTable);
 static STableData *tsdbNewTableData(STsdbCfg *pCfg, STable *pTable);
@@ -41,8 +52,8 @@ static int          tsdbUpdateTableLatestInfo(STsdbRepo *pRepo, STable *pTable,
 static FORCE_INLINE int tsdbCheckRowRange(STsdbRepo *pRepo, STable *pTable, SDataRow row, TSKEY minKey, TSKEY maxKey,
                                           TSKEY now);
 
-int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
-  STsdbRepo *    pRepo = (STsdbRepo *)repo;
+int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp) {
+  STsdbRepo *    pRepo = repo;
   SSubmitMsgIter msgIter = {0};
   SSubmitBlk *   pBlock = NULL;
   int32_t        affectedrows = 0;
@@ -225,8 +236,8 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
   return 0;
 }
 
-int tsdbSyncCommit(TSDB_REPO_T *repo) {
-  STsdbRepo *pRepo = (STsdbRepo *)repo;
+int tsdbSyncCommit(STsdbRepo *repo) {
+  STsdbRepo *pRepo = repo;
 
   tsdbAsyncCommit(pRepo);
   tsem_wait(&(pRepo->readyToCommit));
@@ -254,14 +265,17 @@ int tsdbSyncCommit(TSDB_REPO_T *repo) {
  */
 int tsdbLoadDataFromCache(STable *pTable, SSkipListIterator *pIter, TSKEY maxKey, int maxRowsToRead, SDataCols *pCols,
                           TKEY *filterKeys, int nFilterKeys, bool keepDup, SMergeInfo *pMergeInfo) {
-  ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0 && pMergeInfo != NULL);
+  ASSERT(maxRowsToRead > 0 && nFilterKeys >= 0);
   if (pIter == NULL) return 0;
-  STSchema *pSchema = NULL;
-  TSKEY     rowKey = 0;
-  TSKEY     fKey = 0;
-  bool      isRowDel = false;
-  int       filterIter = 0;
-  SDataRow  row = NULL;
+  STSchema * pSchema = NULL;
+  TSKEY      rowKey = 0;
+  TSKEY      fKey = 0;
+  bool       isRowDel = false;
+  int        filterIter = 0;
+  SDataRow   row = NULL;
+  SMergeInfo mInfo;
+
+  if (pMergeInfo == NULL) pMergeInfo = &mInfo;
 
   memset(pMergeInfo, 0, sizeof(*pMergeInfo));
   pMergeInfo->keyFirst = INT64_MAX;
@@ -452,11 +466,6 @@ static void tsdbFreeTableData(STableData *pTableData) {
 
 static char *tsdbGetTsTupleKey(const void *data) { return dataRowTuple((SDataRow)data); }
 
-void tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey) {
-  *minKey = fileId * daysPerFile * tsMsPerDay[precision];
-  *maxKey = *minKey + daysPerFile * tsMsPerDay[precision] - 1;
-}
-
 static int tsdbAdjustMemMaxTables(SMemTable *pMemTable, int maxTables) {
   ASSERT(pMemTable->maxTables < maxTables);
 

src/tsdb/src/tsdbMeta.c

@@ -12,20 +12,12 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include <stdlib.h>
-#include "hash.h"
-#include "taosdef.h"
-#include "tchecksum.h"
-#include "tsdb.h"
-#include "tsdbMain.h"
-#include "tskiplist.h"
+#include "tsdbint.h"
 
 #define TSDB_SUPER_TABLE_SL_LEVEL 5
 #define DEFAULT_TAG_INDEX_COLUMN 0
 
 static int     tsdbCompareSchemaVersion(const void *key1, const void *key2);
-static int     tsdbRestoreTable(void *pHandle, void *cont, int contLen);
-static void    tsdbOrgMeta(void *pHandle);
 static char *  getTagIndexKey(const void *pData);
 static STable *tsdbNewTable();
 static STable *tsdbCreateTableFromCfg(STableCfg *pCfg, bool isSuper);
@@ -53,7 +45,7 @@ static int     tsdbRmTableFromMeta(STsdbRepo *pRepo, STable *pTable);
 static int     tsdbAdjustMetaTables(STsdbRepo *pRepo, int tid);
 
 // ------------------ OUTER FUNCTIONS ------------------
-int tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg) {
+int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   STsdbMeta *pMeta = pRepo->tsdbMeta;
   STable *   super = NULL;
@@ -148,7 +140,7 @@ _err:
   return -1;
 }
 
-int tsdbDropTable(TSDB_REPO_T *repo, STableId tableId) {
+int tsdbDropTable(STsdbRepo *repo, STableId tableId) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   STsdbMeta *pMeta = pRepo->tsdbMeta;
   uint64_t   uid = tableId.uid;
@@ -301,7 +293,7 @@ static UNUSED_FUNC int32_t colIdCompar(const void* left, const void* right) {
   return (colId < p2->colId)? -1:1;
 }
 
-int tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg) {
+int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg) {
   STsdbRepo *pRepo = (STsdbRepo *)repo;
   STsdbMeta *pMeta = pRepo->tsdbMeta;
   STSchema * pNewSchema = NULL;
@@ -469,6 +461,8 @@ void tsdbFreeMeta(STsdbMeta *pMeta) {
 }
 
 int tsdbOpenMeta(STsdbRepo *pRepo) {
+  return 0;
+#if 0
   char *     fname = NULL;
   STsdbMeta *pMeta = pRepo->tsdbMeta;
   ASSERT(pMeta != NULL);
@@ -479,11 +473,11 @@ int tsdbOpenMeta(STsdbRepo *pRepo) {
     goto _err;
   }
 
-  pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo);
-  if (pMeta->pStore == NULL) {
-    tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno));
-    goto _err;
-  }
+  // pMeta->pStore = tdOpenKVStore(fname, tsdbRestoreTable, tsdbOrgMeta, (void *)pRepo);
+  // if (pMeta->pStore == NULL) {
+  //   tsdbError("vgId:%d failed to open TSDB meta while open the kv store since %s", REPO_ID(pRepo), tstrerror(terrno));
+  //   goto _err;
+  // }
 
   tsdbDebug("vgId:%d open TSDB meta succeed", REPO_ID(pRepo));
   tfree(fname);
@@ -492,6 +486,7 @@ int tsdbOpenMeta(STsdbRepo *pRepo) {
 _err:
   tfree(fname);
   return -1;
+#endif
 }
 
 int tsdbCloseMeta(STsdbRepo *pRepo) {
@@ -500,7 +495,7 @@ int tsdbCloseMeta(STsdbRepo *pRepo) {
   STable *   pTable = NULL;
 
   if (pMeta == NULL) return 0;
-  tdCloseKVStore(pMeta->pStore);
+  // tdCloseKVStore(pMeta->pStore);
   for (int i = 1; i < pMeta->maxTables; i++) {
     tsdbFreeTable(pMeta->tables[i]);
   }
@@ -609,10 +604,8 @@ void tsdbUpdateTableSchema(STsdbRepo *pRepo, STable *pTable, STSchema *pSchema,
   }
 }
 
-// ------------------ LOCAL FUNCTIONS ------------------
-static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
-  STsdbRepo *pRepo = (STsdbRepo *)pHandle;
-  STable *   pTable = NULL;
+int tsdbRestoreTable(STsdbRepo *pRepo, void *cont, int contLen) {
+  STable *pTable = NULL;
 
   if (!taosCheckChecksumWhole((uint8_t *)cont, contLen)) {
     terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
@@ -631,8 +624,7 @@ static int tsdbRestoreTable(void *pHandle, void *cont, int contLen) {
   return 0;
 }
 
-static void tsdbOrgMeta(void *pHandle) {
-  STsdbRepo *pRepo = (STsdbRepo *)pHandle;
+void tsdbOrgMeta(STsdbRepo *pRepo) {
   STsdbMeta *pMeta = pRepo->tsdbMeta;
 
   for (int i = 1; i < pMeta->maxTables; i++) {
@@ -643,6 +635,7 @@ static void tsdbOrgMeta(void *pHandle) {
   }
 }
 
+// ------------------ LOCAL FUNCTIONS ------------------
 static char *getTagIndexKey(const void *pData) {
   STable *pTable = (STable *)pData;
 

src/tsdb/src/tsdbRWHelper.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#define TAOS_RANDOM_FILE_FAIL_TEST
-#include "os.h"
-#include "talgo.h"
-#include "tchecksum.h"
-#include "tcoding.h"
-#include "tscompression.h"
-#include "tsdbMain.h"
-
-#define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM))
-#define TSDB_KEY_COL_OFFSET 0
-#define TSDB_GET_COMPBLOCK_IDX(h, b) (POINTER_DISTANCE(b, (h)->pCompInfo->blocks)/sizeof(SCompBlock))
-#define TSDB_IS_LAST_BLOCK(pb) ((pb)->last)
-
-static bool tsdbShouldCreateNewLast(SRWHelper *pHelper);
-static int  tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock,
-                                 bool isLast, bool isSuperBlock);
-static int  compareKeyBlock(const void *arg1, const void *arg2);
-static int  tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize);
-static int  tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
-static int  tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo);
-static int  tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx);
-static void tsdbResetHelperFileImpl(SRWHelper *pHelper);
-static int  tsdbInitHelperFile(SRWHelper *pHelper);
-static void tsdbDestroyHelperFile(SRWHelper *pHelper);
-static void tsdbResetHelperTableImpl(SRWHelper *pHelper);
-static void tsdbResetHelperTable(SRWHelper *pHelper);
-static void tsdbInitHelperTable(SRWHelper *pHelper);
-static void tsdbDestroyHelperTable(SRWHelper *pHelper);
-static void tsdbResetHelperBlockImpl(SRWHelper *pHelper);
-static void tsdbResetHelperBlock(SRWHelper *pHelper);
-static int  tsdbInitHelperBlock(SRWHelper *pHelper);
-static int  tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type);
-static int  tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows,
-                                         int maxPoints, char *buffer, int bufferSize);
-static int  tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols, int16_t *colIds,
-                                      int numOfColIds);
-static int  tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols);
-static int  tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx);
-static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx);
-static int   tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey);
-static void  tsdbDestroyHelperBlock(SRWHelper *pHelper);
-static int   tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBlock, SCompCol *pCompCol,
-                             SDataCol *pDataCol);
-static int   tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock);
-static int   tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
-                                    int *blkIdx);
-static void  tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
-                                       TSKEY maxKey, int maxRows, int8_t update);
-static bool  tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps);
-static int   tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx);
-
-// ---------------------- INTERNAL FUNCTIONS ----------------------
-int tsdbInitReadHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
-  return tsdbInitHelper(pHelper, pRepo, TSDB_READ_HELPER);
-}
-
-int tsdbInitWriteHelper(SRWHelper *pHelper, STsdbRepo *pRepo) {
-  return tsdbInitHelper(pHelper, pRepo, TSDB_WRITE_HELPER);
-}
-
-void tsdbDestroyHelper(SRWHelper *pHelper) {
-  if (pHelper) {
-    taosTZfree(pHelper->pBuffer);
-    taosTZfree(pHelper->compBuffer);
-    tsdbDestroyHelperFile(pHelper);
-    tsdbDestroyHelperTable(pHelper);
-    tsdbDestroyHelperBlock(pHelper);
-    memset((void *)pHelper, 0, sizeof(*pHelper));
-  }
-}
-
-void tsdbResetHelper(SRWHelper *pHelper) {
-  if (pHelper) {
-    // Reset the block part
-    tsdbResetHelperBlockImpl(pHelper);
-
-    // Reset the table part
-    tsdbResetHelperTableImpl(pHelper);
-
-    // Reset the file part
-    tsdbCloseHelperFile(pHelper, false, NULL);
-    tsdbResetHelperFileImpl(pHelper);
-
-    pHelper->state = TSDB_HELPER_CLEAR_STATE;
-  }
-}
-
-int tsdbSetAndOpenHelperFile(SRWHelper *pHelper, SFileGroup *pGroup) {
-  ASSERT(pHelper != NULL && pGroup != NULL);
-  SFile *    pFile = NULL;
-  STsdbRepo *pRepo = pHelper->pRepo;
-
-  // Clear the helper object
-  tsdbResetHelper(pHelper);
-
-  ASSERT(pHelper->state == TSDB_HELPER_CLEAR_STATE);
-
-  // Set the files
-  pHelper->files.fGroup = *pGroup;
-  if (helperType(pHelper) == TSDB_WRITE_HELPER) {
-    tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), pGroup->fileId, TSDB_FILE_TYPE_NHEAD,
-                        helperNewHeadF(pHelper)->fname);
-    tsdbGetDataFileName(pRepo->rootDir, REPO_ID(pRepo), pGroup->fileId, TSDB_FILE_TYPE_NLAST,
-                        helperNewLastF(pHelper)->fname);
-  }
-
-  // Open the files
-  if (tsdbOpenFile(helperHeadF(pHelper), O_RDONLY) < 0) return -1;
-  if (helperType(pHelper) == TSDB_WRITE_HELPER) {
-    if (tsdbOpenFile(helperDataF(pHelper), O_RDWR) < 0) return -1;
-    if (tsdbOpenFile(helperLastF(pHelper), O_RDWR) < 0) return -1;
-
-    // Create and open .h
-    pFile = helperNewHeadF(pHelper);
-    if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1;
-    pFile->info.size = TSDB_FILE_HEAD_SIZE;
-    pFile->info.magic = TSDB_FILE_INIT_MAGIC;
-    if (tsdbUpdateFileHeader(pFile) < 0) return -1;
-
-    // Create and open .l file if should
-    if (tsdbShouldCreateNewLast(pHelper)) {
-      pFile = helperNewLastF(pHelper);
-      if (tsdbOpenFile(pFile, O_WRONLY | O_CREAT) < 0) return -1;
-      pFile->info.size = TSDB_FILE_HEAD_SIZE;
-      pFile->info.magic = TSDB_FILE_INIT_MAGIC;
-      pFile->info.len = 0;
-      if (tsdbUpdateFileHeader(pFile) < 0) return -1;
-    }
-  } else {
-    if (tsdbOpenFile(helperDataF(pHelper), O_RDONLY) < 0) return -1;
-    if (tsdbOpenFile(helperLastF(pHelper), O_RDONLY) < 0) return -1;
-  }
-
-  helperSetState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN);
-
-  return 0;
-}
-
-int tsdbCloseHelperFile(SRWHelper *pHelper, bool hasError, SFileGroup *pGroup) {
-  SFile *pFile = NULL;
-
-  pFile = helperHeadF(pHelper);
-  tsdbCloseFile(pFile);
-
-  pFile = helperDataF(pHelper);
-  if (pFile->fd > 0) {
-    if (helperType(pHelper) == TSDB_WRITE_HELPER) {
-      if (!hasError) {
-        tsdbUpdateFileHeader(pFile);
-      } else {
-        ASSERT(pGroup != NULL);
-        taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_DATA].info.size);
-      }
-      fsync(pFile->fd);
-    }
-    tsdbCloseFile(pFile);
-  }
-
-  pFile = helperLastF(pHelper);
-  if (pFile->fd > 0) {
-    if (helperType(pHelper) == TSDB_WRITE_HELPER && !TSDB_NLAST_FILE_OPENED(pHelper)) {
-      if (!hasError) {
-        tsdbUpdateFileHeader(pFile);
-      } else {
-        ASSERT(pGroup != NULL);
-        taosFtruncate(pFile->fd, pGroup->files[TSDB_FILE_TYPE_LAST].info.size);
-      }
-      fsync(pFile->fd);
-    }
-    tsdbCloseFile(pFile);
-  }
-
-  if (helperType(pHelper) == TSDB_WRITE_HELPER) {
-    pFile = helperNewHeadF(pHelper);
-    if (pFile->fd > 0) {
-      if (!hasError) {
-        tsdbUpdateFileHeader(pFile);
-        fsync(pFile->fd);
-      }
-      tsdbCloseFile(pFile);
-      if (hasError) (void)remove(pFile->fname);
-    }
-
-    pFile = helperNewLastF(pHelper);
-    if (pFile->fd > 0) {
-      if (!hasError) {
-        tsdbUpdateFileHeader(pFile);
-        fsync(pFile->fd);
-      }
-      tsdbCloseFile(pFile);
-      if (hasError) (void)remove(pFile->fname);
-    }
-  }
-  return 0;
-}
-
-int tsdbSetHelperTable(SRWHelper *pHelper, STable *pTable, STsdbRepo *pRepo) {
-  ASSERT(helperHasState(pHelper, TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD));
-
-  // Clear members and state used by previous table
-  tsdbResetHelperTable(pHelper);
-  ASSERT(helperHasState(pHelper, (TSDB_HELPER_FILE_SET_AND_OPEN | TSDB_HELPER_IDX_LOAD)));
-
-  pHelper->tableInfo.tid = pTable->tableId.tid;
-  pHelper->tableInfo.uid = pTable->tableId.uid;
-  STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
-
-  if (tdInitDataCols(pHelper->pDataCols[0], pSchema) < 0) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  if (tdInitDataCols(pHelper->pDataCols[1], pSchema) < 0) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  if (pHelper->idxH.numOfIdx > 0) {
-    while (true) {
-      if (pHelper->idxH.curIdx >= pHelper->idxH.numOfIdx) {
-        memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx));
-        break;
-      }
-
-      SCompIdx *pIdx = &(pHelper->idxH.pIdxArray[pHelper->idxH.curIdx]);
-      if (pIdx->tid == TABLE_TID(pTable)) {
-        if (pIdx->uid == TABLE_UID(pTable)) {
-          pHelper->curCompIdx = *pIdx;
-        } else {
-          memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx));
-        }
-        pHelper->idxH.curIdx++;
-        break;
-      } else if (pIdx->tid > TABLE_TID(pTable)) {
-        memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx));
-        break;
-      } else {
-        pHelper->idxH.curIdx++;
-      }
-    }
-  } else {
-    memset(&(pHelper->curCompIdx), 0, sizeof(SCompIdx));
-  }
-
-  if (helperType(pHelper) == TSDB_WRITE_HELPER && pHelper->curCompIdx.hasLast) {
-    pHelper->hasOldLastBlock = true;
-  }
-
-  helperSetState(pHelper, TSDB_HELPER_TABLE_SET);
-  ASSERT(pHelper->state == ((TSDB_HELPER_TABLE_SET << 1) - 1));
-
-  return 0;
-}
-
-int tsdbCommitTableData(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) {
-  ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER);
-
-  SCompIdx *pIdx = &(pHelper->curCompIdx);
-  int       blkIdx = 0;
-
-  ASSERT(pIdx->offset == 0 || pIdx->uid == TABLE_UID(pCommitIter->pTable));
-  if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1;
-
-  while (true) {
-    ASSERT(blkIdx <= (int)pIdx->numOfBlocks);
-    TSKEY keyFirst = tsdbNextIterKey(pCommitIter->pIter);
-    if (keyFirst == TSDB_DATA_TIMESTAMP_NULL || keyFirst > maxKey) break;  // iter over
-
-    if (pIdx->len <= 0 || keyFirst > pIdx->maxKey) {
-      if (tsdbProcessAppendCommit(pHelper, pCommitIter, pDataCols, maxKey) < 0) return -1;
-      blkIdx = pIdx->numOfBlocks;
-    } else {
-      if (tsdbProcessMergeCommit(pHelper, pCommitIter, pDataCols, maxKey, &blkIdx) < 0) return -1;
-    }
-  }
-
-  return 0;
-}
-
-int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
-  STsdbCfg *pCfg = &pHelper->pRepo->config;
-
-  ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER);
-  SCompIdx * pIdx = &(pHelper->curCompIdx);
-  SCompBlock compBlock = {0};
-  if (TSDB_NLAST_FILE_OPENED(pHelper) && (pHelper->hasOldLastBlock)) {
-    if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1;
-
-    SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1);
-    ASSERT(pCompBlock->last);
-    if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
-    ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows &&
-           pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock);
-    if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0)
-      return -1;
-
-    if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1;
-
-#if 0
-    if (pCompBlock->numOfSubBlocks > 1) {
-      if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
-      ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows &&
-             pHelper->pDataCols[0]->numOfRows < pCfg->minRowsPerFileBlock);
-      if (tsdbWriteBlockToFile(pHelper, helperNewLastF(pHelper), pHelper->pDataCols[0], &compBlock, true, true) < 0)
-        return -1;
-
-      if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1;
-    } else {
-      if (lseek(helperLastF(pHelper)->fd, pCompBlock->offset, SEEK_SET) < 0) {
-        tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperLastF(pHelper)->fname,
-                  strerror(errno));
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        return -1;
-      }
-      pCompBlock->offset = lseek(helperNewLastF(pHelper)->fd, 0, SEEK_END);
-      if (pCompBlock->offset < 0) {
-        tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), helperNewLastF(pHelper)->fname,
-                  strerror(errno));
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        return -1;
-      }
-
-      if (taosSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
-        tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo),
-                  helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno));
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        return -1;
-      }
-    }
-#endif
-
-    pHelper->hasOldLastBlock = false;
-  }
-
-  return 0;
-}
-
-int tsdbWriteCompInfo(SRWHelper *pHelper) {
-  SCompIdx *pIdx = &(pHelper->curCompIdx);
-  off_t     offset = 0;
-  SFile *   pFile = helperNewHeadF(pHelper);
-
-  if (pIdx->len > 0) {
-    if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) {
-      if (tsdbLoadCompInfo(pHelper, NULL) < 0) return -1;
-    } else {
-      pHelper->pCompInfo->delimiter = TSDB_FILE_DELIMITER;
-      pHelper->pCompInfo->uid = pHelper->tableInfo.uid;
-      pHelper->pCompInfo->tid = pHelper->tableInfo.tid;
-      ASSERT(pIdx->len > sizeof(SCompInfo) + sizeof(TSCKSUM) &&
-             (pIdx->len - sizeof(SCompInfo) - sizeof(TSCKSUM)) % sizeof(SCompBlock) == 0);
-      taosCalcChecksumAppend(0, (uint8_t *)pHelper->pCompInfo, pIdx->len);
-    }
-
-    pFile->info.magic = taosCalcChecksum(
-        pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM)), sizeof(TSCKSUM));
-    offset = lseek(pFile->fd, 0, SEEK_END);
-    if (offset < 0) {
-      tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno));
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      return -1;
-    }
-    pIdx->offset = offset;
-    pIdx->uid = pHelper->tableInfo.uid;
-    pIdx->tid = pHelper->tableInfo.tid;
-    ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
-
-    if (taosWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < (int)pIdx->len) {
-      tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
-                pFile->fname, strerror(errno));
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      return -1;
-    }
-
-    if (taosTSizeof(pHelper->pWIdx) < pFile->info.len + sizeof(SCompIdx) + 12) {
-      pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, taosTSizeof(pHelper->pWIdx) == 0 ? 1024 : taosTSizeof(pHelper->pWIdx) * 2);
-      if (pHelper->pWIdx == NULL) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        return -1;
-      }
-    }
-
-    void *pBuf = POINTER_SHIFT(pHelper->pWIdx, pFile->info.len);
-    pFile->info.len += tsdbEncodeSCompIdx(&pBuf, &(pHelper->curCompIdx));
-
-    pFile->info.size += pIdx->len;
-    // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR));
-  }
-
-  return 0;
-}
-
-int tsdbWriteCompIdx(SRWHelper *pHelper) {
-  ASSERT(helperType(pHelper) == TSDB_WRITE_HELPER);
-  off_t offset = 0;
-
-  SFile *pFile = helperNewHeadF(pHelper);
-
-  pFile->info.len += sizeof(TSCKSUM);
-  if (taosTSizeof(pHelper->pWIdx) < pFile->info.len) {
-    pHelper->pWIdx = taosTRealloc(pHelper->pWIdx, pFile->info.len);
-    if (pHelper->pWIdx == NULL) {
-      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-      return -1;
-    }
-  }
-  taosCalcChecksumAppend(0, (uint8_t *)pHelper->pWIdx, pFile->info.len);
-  pFile->info.magic = taosCalcChecksum(
-      pFile->info.magic, (uint8_t *)POINTER_SHIFT(pHelper->pWIdx, pFile->info.len - sizeof(TSCKSUM)), sizeof(TSCKSUM));
-
-  offset = lseek(pFile->fd, 0, SEEK_END);
-  if (offset < 0) {
-    tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  ASSERT(offset == pFile->info.size);
-
-  if (taosWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < (int)pFile->info.len) {
-    tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
-              pFile->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  pFile->info.offset = offset;
-  pFile->info.size += pFile->info.len;
-  // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR));
-
-  return 0;
-}
-
-int tsdbLoadCompIdxImpl(SFile *pFile, uint32_t offset, uint32_t len, void *buffer) {
-  const char *prefixMsg = "failed to load SCompIdx part";
-  if (lseek(pFile->fd, offset, SEEK_SET) < 0) {
-    tsdbError("%s: seek to file %s offset %u failed since %s", prefixMsg, pFile->fname, offset, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosRead(pFile->fd, buffer, len) < len) {
-    tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, offset, len,
-              strerror(errno));
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    return -1;
-  }
-
-  if (!taosCheckChecksumWhole((uint8_t *)buffer, len)) {
-    tsdbError("%s: file %s corrupted, offset %u len %u", prefixMsg, pFile->fname, offset, len);
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    return -1;
-  }
-
-  return 0;
-}
-
-int tsdbDecodeSCompIdxImpl(void *buffer, uint32_t len, SCompIdx **ppCompIdx, int *numOfIdx) {
-  int   nIdx = 0;
-  void *pPtr = buffer;
-
-  while (POINTER_DISTANCE(pPtr, buffer) < (int)(len - sizeof(TSCKSUM))) {
-    size_t tlen = taosTSizeof(*ppCompIdx);
-    if (tlen < sizeof(SCompIdx) * (nIdx + 1)) {
-      *ppCompIdx = (SCompIdx *)taosTRealloc(*ppCompIdx, (tlen == 0) ? 1024 : tlen * 2);
-      if (*ppCompIdx == NULL) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        return -1;
-      }
-    }
-
-    pPtr = tsdbDecodeSCompIdx(pPtr, &((*ppCompIdx)[nIdx]));
-    if (pPtr == NULL) {
-      tsdbError("failed to decode SCompIdx part, idx:%d", nIdx);
-      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-      return -1;
-    }
-
-    nIdx++;
-
-    ASSERT(nIdx == 1 || (*ppCompIdx)[nIdx - 1].tid > (*ppCompIdx)[nIdx - 2].tid);
-    ASSERT(POINTER_DISTANCE(pPtr, buffer) <= (int)(len - sizeof(TSCKSUM)));
-  }
-
-  *numOfIdx = nIdx;
-  return 0;
-}
-
-int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
-  ASSERT(pHelper->state == TSDB_HELPER_FILE_SET_AND_OPEN);
-  SFile *pFile = helperHeadF(pHelper);
-
-  if (!helperHasState(pHelper, TSDB_HELPER_IDX_LOAD)) {
-    // If not load from file, just load it in object
-    if (pFile->info.len > 0) {
-      if ((pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pFile->info.len)) == NULL) {
-        terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-        return -1;
-      }
-
-      // Load SCompIdx binary from file
-      if (tsdbLoadCompIdxImpl(pFile, pFile->info.offset, pFile->info.len, (void *)(pHelper->pBuffer)) < 0) {
-        return -1;
-      }
-
-      // Decode the SCompIdx part
-      if (tsdbDecodeSCompIdxImpl(pHelper->pBuffer, pFile->info.len, &(pHelper->idxH.pIdxArray),
-                                 &(pHelper->idxH.numOfIdx)) < 0) {
-        tsdbError("vgId:%d failed to decode SCompIdx part from file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname,
-                  tstrerror(errno));
-        return -1;
-      }
-    }
-  }
-  helperSetState(pHelper, TSDB_HELPER_IDX_LOAD);
-
-  // Copy the memory for outside usage
-  if (target && pHelper->idxH.numOfIdx > 0)
-    memcpy(target, pHelper->idxH.pIdxArray, sizeof(SCompIdx) * pHelper->idxH.numOfIdx);
-
-  return 0;
-}
-
-int tsdbLoadCompInfoImpl(SFile *pFile, SCompIdx *pIdx, SCompInfo **ppCompInfo) {
-  const char *prefixMsg = "failed to load SCompInfo/SCompBlock part";
-
-  if (lseek(pFile->fd, pIdx->offset, SEEK_SET) < 0) {
-    tsdbError("%s: seek to file %s offset %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  *ppCompInfo = taosTRealloc((void *)(*ppCompInfo), pIdx->len);
-  if (*ppCompInfo == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  if (taosRead(pFile->fd, (void *)(*ppCompInfo), pIdx->len) < (int)pIdx->len) {
-    tsdbError("%s: read file %s offset %u len %u failed since %s", prefixMsg, pFile->fname, pIdx->offset, pIdx->len,
-              strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (!taosCheckChecksumWhole((uint8_t *)(*ppCompInfo), pIdx->len)) {
-    tsdbError("%s: file %s corrupted, offset %u len %u", prefixMsg, pFile->fname, pIdx->offset, pIdx->len);
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    return -1;
-  }
-
-  return 0;
-}
-
-int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) {
-  ASSERT(helperHasState(pHelper, TSDB_HELPER_TABLE_SET));
-
-  SCompIdx *pIdx = &(pHelper->curCompIdx);
-
-  SFile *pFile = helperHeadF(pHelper);
-
-  if (!helperHasState(pHelper, TSDB_HELPER_INFO_LOAD)) {
-    if (pIdx->offset > 0) {
-      ASSERT(pIdx->uid == pHelper->tableInfo.uid);
-
-      if (tsdbLoadCompInfoImpl(pFile, pIdx, &(pHelper->pCompInfo)) < 0) return -1;
-
-      ASSERT(pIdx->uid == pHelper->pCompInfo->uid && pIdx->tid == pHelper->pCompInfo->tid);
-    }
-
-    helperSetState(pHelper, TSDB_HELPER_INFO_LOAD);
-  }
-
-  if (target) memcpy(target, (void *)(pHelper->pCompInfo), pIdx->len);
-
-  return 0;
-}
-
-int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) {
-  ASSERT(pCompBlock->numOfSubBlocks <= 1);
-  SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper);
-
-  if (lseek(pFile->fd, (off_t)pCompBlock->offset, SEEK_SET) < 0) {
-    tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  size_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols);
-  pHelper->pCompData = taosTRealloc((void *)pHelper->pCompData, tsize);
-  if (pHelper->pCompData == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  if (taosRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
-    tsdbError("vgId:%d failed to read %" PRIzu " bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname,
-              strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (!taosCheckChecksumWhole((uint8_t *)pHelper->pCompData, (uint32_t)tsize)) {
-    tsdbError("vgId:%d file %s is broken, offset %" PRId64 " size %" PRIzu "", REPO_ID(pHelper->pRepo), pFile->fname,
-              (int64_t)pCompBlock->offset, tsize);
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    return -1;
-  }
-
-  ASSERT(pCompBlock->numOfCols == pHelper->pCompData->numOfCols);
-
-  if (target) memcpy(target, pHelper->pCompData, tsize);
-
-  return 0;
-}
-
-void tsdbGetDataStatis(SRWHelper *pHelper, SDataStatis *pStatis, int numOfCols) {
-  SCompData *pCompData = pHelper->pCompData;
-
-  for (int i = 0, j = 0; i < numOfCols;) {
-    if (j >= pCompData->numOfCols) {
-      pStatis[i].numOfNull = -1;
-      i++;
-      continue;
-    }
-
-    if (pStatis[i].colId == pCompData->cols[j].colId) {
-      pStatis[i].sum = pCompData->cols[j].sum;
-      pStatis[i].max = pCompData->cols[j].max;
-      pStatis[i].min = pCompData->cols[j].min;
-      pStatis[i].maxIndex = pCompData->cols[j].maxIndex;
-      pStatis[i].minIndex = pCompData->cols[j].minIndex;
-      pStatis[i].numOfNull = pCompData->cols[j].numOfNull;
-      i++;
-      j++;
-    } else if (pStatis[i].colId < pCompData->cols[j].colId) {
-      pStatis[i].numOfNull = -1;
-      i++;
-    } else {
-      j++;
-    }
-  }
-}
-
-int tsdbLoadBlockDataCols(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo, int16_t *colIds, int numOfColIds) {
-  ASSERT(pCompBlock->numOfSubBlocks >= 1);  // Must be super block
-  SCompBlock *pTCompBlock = pCompBlock;
-
-  int numOfSubBlocks = pCompBlock->numOfSubBlocks;
-  if (numOfSubBlocks > 1)
-    pTCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset);
-
-  tdResetDataCols(pHelper->pDataCols[0]);
-  if (tsdbLoadBlockDataColsImpl(pHelper, pTCompBlock, pHelper->pDataCols[0], colIds, numOfColIds) < 0) goto _err;
-  for (int i = 1; i < numOfSubBlocks; i++) {
-    tdResetDataCols(pHelper->pDataCols[1]);
-    pTCompBlock++;
-    if (tsdbLoadBlockDataColsImpl(pHelper, pTCompBlock, pHelper->pDataCols[1], colIds, numOfColIds) < 0) goto _err;
-    if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err;
-  }
-
-  ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows &&
-         dataColsKeyFirst(pHelper->pDataCols[0]) == pCompBlock->keyFirst &&
-         dataColsKeyLast(pHelper->pDataCols[0]) == pCompBlock->keyLast);
-
-  return 0;
-
-_err:
-  return -1;
-}
-
-int tsdbLoadBlockData(SRWHelper *pHelper, SCompBlock *pCompBlock, SCompInfo *pCompInfo) {
-  SCompBlock *pTCompBlock = pCompBlock;
-
-  int numOfSubBlock = pCompBlock->numOfSubBlocks;
-  if (numOfSubBlock > 1)
-    pTCompBlock = (SCompBlock *)POINTER_SHIFT((pCompInfo == NULL) ? pHelper->pCompInfo : pCompInfo, pCompBlock->offset);
-
-  tdResetDataCols(pHelper->pDataCols[0]);
-  if (tsdbLoadBlockDataImpl(pHelper, pTCompBlock, pHelper->pDataCols[0]) < 0) goto _err;
-  for (int i = 1; i < numOfSubBlock; i++) {
-    tdResetDataCols(pHelper->pDataCols[1]);
-    pTCompBlock++;
-    if (tsdbLoadBlockDataImpl(pHelper, pTCompBlock, pHelper->pDataCols[1]) < 0) goto _err;
-    if (tdMergeDataCols(pHelper->pDataCols[0], pHelper->pDataCols[1], pHelper->pDataCols[1]->numOfRows) < 0) goto _err;
-  }
-
-  ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows &&
-         dataColsKeyFirst(pHelper->pDataCols[0]) == pCompBlock->keyFirst &&
-         dataColsKeyLast(pHelper->pDataCols[0]) == pCompBlock->keyLast);
-
-  return 0;
-
-_err:
-  return -1;
-}
-
-// ---------------------- INTERNAL FUNCTIONS ----------------------
-static bool tsdbShouldCreateNewLast(SRWHelper *pHelper) {
-  ASSERT(helperLastF(pHelper)->fd > 0);
-  struct stat st;
-  if (fstat(helperLastF(pHelper)->fd, &st) < 0) return true;
-  if (st.st_size > 32 * 1024 + TSDB_FILE_HEAD_SIZE) return true;
-  return false;
-}
-
-static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDataCols, SCompBlock *pCompBlock,
-                                bool isLast, bool isSuperBlock) {
-  STsdbCfg * pCfg = &(pHelper->pRepo->config);
-  SCompData *pCompData = (SCompData *)(pHelper->pBuffer);
-  int64_t    offset = 0;
-  int        rowsToWrite = pDataCols->numOfRows;
-
-  ASSERT(rowsToWrite > 0 && rowsToWrite <= pCfg->maxRowsPerFileBlock);
-  ASSERT(isLast ? rowsToWrite < pCfg->minRowsPerFileBlock : true);
-
-  offset = lseek(pFile->fd, 0, SEEK_END);
-  if (offset < 0) {
-    tsdbError("vgId:%d failed to write block to file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname,
-              strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  int nColsNotAllNull = 0;
-  for (int ncol = 1; ncol < pDataCols->numOfCols; ncol++) {  // ncol from 1, we skip the timestamp column
-    SDataCol *pDataCol = pDataCols->cols + ncol;
-    SCompCol *pCompCol = pCompData->cols + nColsNotAllNull;
-
-    if (isNEleNull(pDataCol, rowsToWrite)) {  // all data to commit are NULL, just ignore it
-      continue;
-    }
-
-    memset(pCompCol, 0, sizeof(*pCompCol));
-
-    pCompCol->colId = pDataCol->colId;
-    pCompCol->type = pDataCol->type;
-    if (tDataTypes[pDataCol->type].statisFunc) {
-      (*tDataTypes[pDataCol->type].statisFunc)(
-          pDataCol->pData, rowsToWrite, &(pCompCol->min), &(pCompCol->max), &(pCompCol->sum), &(pCompCol->minIndex),
-          &(pCompCol->maxIndex), &(pCompCol->numOfNull));
-    }
-    nColsNotAllNull++;
-  }
-
-  ASSERT(nColsNotAllNull >= 0 && nColsNotAllNull <= pDataCols->numOfCols);
-
-  // Compress the data if neccessary
-  int     tcol = 0;
-  int32_t toffset = 0;
-  int32_t tsize = TSDB_GET_COMPCOL_LEN(nColsNotAllNull);
-  int32_t lsize = tsize;
-  int32_t keyLen = 0;
-  for (int ncol = 0; ncol < pDataCols->numOfCols; ncol++) {
-    if (ncol != 0 && tcol >= nColsNotAllNull) break;
-
-    SDataCol *pDataCol = pDataCols->cols + ncol;
-    SCompCol *pCompCol = pCompData->cols + tcol;
-
-    if (ncol != 0 && (pDataCol->colId != pCompCol->colId)) continue;
-    void *tptr = POINTER_SHIFT(pCompData, lsize);
-
-    int32_t flen = 0;  // final length
-    int32_t tlen = dataColGetNEleLen(pDataCol, rowsToWrite);
-
-    if (pCfg->compression) {
-      if (pCfg->compression == TWO_STAGE_COMP) {
-        pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, tlen + COMP_OVERFLOW_BYTES);
-        if (pHelper->compBuffer == NULL) {
-          terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-          goto _err;
-        }
-      }
-
-      flen = (*(tDataTypes[pDataCol->type].compFunc))((char *)pDataCol->pData, tlen, rowsToWrite, tptr,
-                                                         (int32_t)taosTSizeof(pHelper->pBuffer) - lsize, pCfg->compression,
-                                                         pHelper->compBuffer, (int32_t)taosTSizeof(pHelper->compBuffer));
-    } else {
-      flen = tlen;
-      memcpy(tptr, pDataCol->pData, flen);
-    }
-
-    // Add checksum
-    ASSERT(flen > 0);
-    flen += sizeof(TSCKSUM);
-    taosCalcChecksumAppend(0, (uint8_t *)tptr, flen);
-    pFile->info.magic =
-        taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(tptr, flen - sizeof(TSCKSUM)), sizeof(TSCKSUM));
-
-    if (ncol != 0) {
-      pCompCol->offset = toffset;
-      pCompCol->len = flen;
-      tcol++;
-    } else {
-      keyLen = flen;
-    }
-
-    toffset += flen;
-    lsize += flen;
-  }
-
-  pCompData->delimiter = TSDB_FILE_DELIMITER;
-  pCompData->uid = pHelper->tableInfo.uid;
-  pCompData->numOfCols = nColsNotAllNull;
-
-  taosCalcChecksumAppend(0, (uint8_t *)pCompData, tsize);
-  pFile->info.magic = taosCalcChecksum(pFile->info.magic, (uint8_t *)POINTER_SHIFT(pCompData, tsize - sizeof(TSCKSUM)),
-                                       sizeof(TSCKSUM));
-
-  // Write the whole block to file
-  if (taosWrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
-    tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname,
-              strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  // Update pCompBlock membership vairables
-  pCompBlock->last = isLast;
-  pCompBlock->offset = offset;
-  pCompBlock->algorithm = pCfg->compression;
-  pCompBlock->numOfRows = rowsToWrite;
-  pCompBlock->len = lsize;
-  pCompBlock->keyLen = keyLen;
-  pCompBlock->numOfSubBlocks = isSuperBlock ? 1 : 0;
-  pCompBlock->numOfCols = nColsNotAllNull;
-  pCompBlock->keyFirst = dataColsKeyFirst(pDataCols);
-  pCompBlock->keyLast = dataColsKeyAt(pDataCols, rowsToWrite - 1);
-
-  tsdbDebug("vgId:%d tid:%d a block of data is written to file %s, offset %" PRId64
-            " numOfRows %d len %d numOfCols %" PRId16 " keyFirst %" PRId64 " keyLast %" PRId64,
-            REPO_ID(helperRepo(pHelper)), pHelper->tableInfo.tid, pFile->fname, (int64_t)(pCompBlock->offset),
-            (int)(pCompBlock->numOfRows), pCompBlock->len, pCompBlock->numOfCols, pCompBlock->keyFirst,
-            pCompBlock->keyLast);
-
-  pFile->info.size += pCompBlock->len;
-  // ASSERT(pFile->info.size == lseek(pFile->fd, 0, SEEK_CUR));
-
-  return 0;
-
-_err:
-  return -1;
-}
-
-static int compareKeyBlock(const void *arg1, const void *arg2) {
-  TSKEY       key = *(TSKEY *)arg1;
-  SCompBlock *pBlock = (SCompBlock *)arg2;
-
-  if (key < pBlock->keyFirst) {
-    return -1;
-  } else if (key > pBlock->keyLast) {
-    return 1;
-  }
-
-  return 0;
-}
-
-static int tsdbAdjustInfoSizeIfNeeded(SRWHelper *pHelper, size_t esize) {
-  if (taosTSizeof((void *)pHelper->pCompInfo) <= esize) {
-    size_t tsize = esize + sizeof(SCompBlock) * 16;
-    pHelper->pCompInfo = (SCompInfo *)taosTRealloc(pHelper->pCompInfo, tsize);
-    if (pHelper->pCompInfo == NULL) return -1;
-  }
-
-  return 0;
-}
-
-static int tsdbInsertSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) {
-  SCompIdx *pIdx = &(pHelper->curCompIdx);
-
-  ASSERT(blkIdx >= 0 && blkIdx <= (int)pIdx->numOfBlocks);
-  ASSERT(pCompBlock->numOfSubBlocks == 1);
-
-  // Adjust memory if no more room
-  if (pIdx->len == 0) pIdx->len = sizeof(SCompInfo) + sizeof(TSCKSUM);
-  if (tsdbAdjustInfoSizeIfNeeded(pHelper, pIdx->len + sizeof(SCompInfo)) < 0) goto _err;
-
-  // Change the offset
-  for (uint32_t i = 0; i < pIdx->numOfBlocks; i++) {
-    SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i];
-    if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock);
-  }
-
-  // Memmove if needed
-  int tsize = pIdx->len - (sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx);
-  if (tsize > 0) {
-    ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) < taosTSizeof(pHelper->pCompInfo));
-    ASSERT(sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1) + tsize <= taosTSizeof(pHelper->pCompInfo));
-    memmove(POINTER_SHIFT(pHelper->pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * (blkIdx + 1)),
-            POINTER_SHIFT(pHelper->pCompInfo, sizeof(SCompInfo) + sizeof(SCompBlock) * blkIdx), tsize);
-  }
-  pHelper->pCompInfo->blocks[blkIdx] = *pCompBlock;
-
-  pIdx->numOfBlocks++;
-  pIdx->len += sizeof(SCompBlock);
-  ASSERT(pIdx->len <= taosTSizeof(pHelper->pCompInfo));
-  pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast;
-  pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last;
-
-  if (pIdx->numOfBlocks > 1) {
-    ASSERT(pHelper->pCompInfo->blocks[0].keyLast < pHelper->pCompInfo->blocks[1].keyFirst);
-  }
-
-  ASSERT((blkIdx == pIdx->numOfBlocks -1) || (!pCompBlock->last));
-
-  tsdbDebug("vgId:%d tid:%d a super block is inserted at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,
-            blkIdx);
-
-  return 0;
-
-_err:
-  return -1;
-}
-
-static int tsdbAddSubBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx, SMergeInfo *pMergeInfo) {
-  ASSERT(pCompBlock->numOfSubBlocks == 0);
-
-  SCompIdx *pIdx = &(pHelper->curCompIdx);
-  ASSERT(blkIdx >= 0 && blkIdx < (int)pIdx->numOfBlocks);
-
-  SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
-  ASSERT(pSCompBlock->numOfSubBlocks >= 1 && pSCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS);
-
-  size_t spaceNeeded =
-      (pSCompBlock->numOfSubBlocks == 1) ? pIdx->len + sizeof(SCompBlock) * 2 : pIdx->len + sizeof(SCompBlock);
-  if (tsdbAdjustInfoSizeIfNeeded(pHelper, spaceNeeded) < 0) goto _err;
-
-  pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
-
-  // Add the sub-block
-  if (pSCompBlock->numOfSubBlocks > 1) {
-    size_t tsize = (size_t)(pIdx->len - (pSCompBlock->offset + pSCompBlock->len));
-    if (tsize > 0) {
-      memmove((void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len + sizeof(SCompBlock)),
-              (void *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len), tsize);
-
-      for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) {
-        SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i];
-        if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += sizeof(SCompBlock);
-      }
-    }
-
-    *(SCompBlock *)((char *)(pHelper->pCompInfo) + pSCompBlock->offset + pSCompBlock->len) = *pCompBlock;
-
-    pSCompBlock->numOfSubBlocks++;
-    ASSERT(pSCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
-    pSCompBlock->len += sizeof(SCompBlock);
-    pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
-    pSCompBlock->keyFirst = pMergeInfo->keyFirst;
-    pSCompBlock->keyLast = pMergeInfo->keyLast;
-    pIdx->len += sizeof(SCompBlock);
-  } else {  // Need to create two sub-blocks
-    void *ptr = NULL;
-    for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) {
-      SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i;
-      if (pTCompBlock->numOfSubBlocks > 1) {
-        ptr = POINTER_SHIFT(pHelper->pCompInfo, pTCompBlock->offset);
-        break;
-      }
-    }
-
-    if (ptr == NULL) ptr = POINTER_SHIFT(pHelper->pCompInfo, pIdx->len - sizeof(TSCKSUM));
-
-    size_t tsize = pIdx->len - ((char *)ptr - (char *)(pHelper->pCompInfo));
-    if (tsize > 0) {
-      memmove(POINTER_SHIFT(ptr, sizeof(SCompBlock) * 2), ptr, tsize);
-      for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) {
-        SCompBlock *pTCompBlock = pHelper->pCompInfo->blocks + i;
-        if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset += (sizeof(SCompBlock) * 2);
-      }
-    }
-
-    ((SCompBlock *)ptr)[0] = *pSCompBlock;
-    ((SCompBlock *)ptr)[0].numOfSubBlocks = 0;
-
-    ((SCompBlock *)ptr)[1] = *pCompBlock;
-
-    pSCompBlock->numOfSubBlocks = 2;
-    pSCompBlock->numOfRows = pSCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
-    pSCompBlock->offset = ((char *)ptr) - ((char *)pHelper->pCompInfo);
-    pSCompBlock->len = sizeof(SCompBlock) * 2;
-    pSCompBlock->keyFirst = pMergeInfo->keyFirst;
-    pSCompBlock->keyLast = pMergeInfo->keyLast;
-
-    pIdx->len += (sizeof(SCompBlock) * 2);
-  }
-
-  pIdx->maxKey = pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].keyLast;
-  pIdx->hasLast = (uint32_t)pHelper->pCompInfo->blocks[pIdx->numOfBlocks - 1].last;
-
-  tsdbDebug("vgId:%d tid:%d a subblock is added at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid, blkIdx);
-
-  return 0;
-
-_err:
-  return -1;
-}
-
-static int tsdbUpdateSuperBlock(SRWHelper *pHelper, SCompBlock *pCompBlock, int blkIdx) {
-  ASSERT(pCompBlock->numOfSubBlocks == 1);
-
-  SCompIdx *pIdx = &(pHelper->curCompIdx);
-
-  ASSERT(blkIdx >= 0 && blkIdx < (int)pIdx->numOfBlocks);
-
-  SCompBlock *pSCompBlock = pHelper->pCompInfo->blocks + blkIdx;
-
-  ASSERT(pSCompBlock->numOfSubBlocks >= 1);
-
-  // Delete the sub blocks it has
-  if (pSCompBlock->numOfSubBlocks > 1) {
-    size_t tsize = (size_t)(pIdx->len - (pSCompBlock->offset + pSCompBlock->len));
-    if (tsize > 0) {
-      memmove(POINTER_SHIFT(pHelper->pCompInfo, pSCompBlock->offset),
-              POINTER_SHIFT(pHelper->pCompInfo, pSCompBlock->offset + pSCompBlock->len), tsize);
-    }
-
-    for (uint32_t i = blkIdx + 1; i < pIdx->numOfBlocks; i++) {
-      SCompBlock *pTCompBlock = &pHelper->pCompInfo->blocks[i];
-      if (pTCompBlock->numOfSubBlocks > 1) pTCompBlock->offset -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks);
-    }
-
-    pIdx->len -= (sizeof(SCompBlock) * pSCompBlock->numOfSubBlocks);
-  }
-
-  *pSCompBlock = *pCompBlock;
-
-  pIdx->maxKey = blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->keyLast;
-  pIdx->hasLast = (uint32_t)blockAtIdx(pHelper, pIdx->numOfBlocks - 1)->last;
-
-  ASSERT((blkIdx == pIdx->numOfBlocks-1) || (!pCompBlock->last));
-
-  tsdbDebug("vgId:%d tid:%d a super block is updated at index %d", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,
-            blkIdx);
-
-  return 0;
-}
-
-static int tsdbDeleteSuperBlock(SRWHelper *pHelper, int blkIdx) {
-  SCompIdx *pCompIdx = &(pHelper->curCompIdx);
-
-  ASSERT(pCompIdx->numOfBlocks > 0 && blkIdx < pCompIdx->numOfBlocks);
-
-  SCompBlock *pCompBlock= blockAtIdx(pHelper, blkIdx);
-  SCompBlock  compBlock = *pCompBlock;
-  ASSERT(pCompBlock->numOfSubBlocks > 0 && pCompBlock->numOfSubBlocks <= TSDB_MAX_SUBBLOCKS);
-
-  if (pCompIdx->numOfBlocks == 1) {
-    memset(pCompIdx, 0, sizeof(*pCompIdx));
-  } else {
-    int tsize = 0;
-
-    if (compBlock.numOfSubBlocks > 1) {
-      tsize = (int)(pCompIdx->len - (compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks));
-
-      ASSERT(tsize > 0);
-      memmove(POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset),
-              POINTER_SHIFT(pHelper->pCompInfo, compBlock.offset + sizeof(SCompBlock) * compBlock.numOfSubBlocks),
-              tsize);
-
-      pCompIdx->len = pCompIdx->len - sizeof(SCompBlock) * compBlock.numOfSubBlocks;
-    }
-
-    tsize = (int)(pCompIdx->len - POINTER_DISTANCE(blockAtIdx(pHelper, blkIdx + 1), pHelper->pCompInfo));
-    ASSERT(tsize > 0);
-    memmove((void *)blockAtIdx(pHelper, blkIdx), (void *)blockAtIdx(pHelper, blkIdx + 1), tsize);
-
-    pCompIdx->len -= sizeof(SCompBlock);
-
-    pCompIdx->numOfBlocks--;
-    pCompIdx->hasLast = (uint32_t)(blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->last);
-    pCompIdx->maxKey = blockAtIdx(pHelper, pCompIdx->numOfBlocks - 1)->keyLast;
-  }
-
-  return 0;
-}
-
-static void tsdbResetHelperFileImpl(SRWHelper *pHelper) {
-  pHelper->idxH.numOfIdx = 0;
-  pHelper->idxH.curIdx = 0;
-  memset((void *)&pHelper->files, 0, sizeof(pHelper->files));
-  helperHeadF(pHelper)->fd = -1;
-  helperDataF(pHelper)->fd = -1;
-  helperLastF(pHelper)->fd = -1;
-  helperNewHeadF(pHelper)->fd = -1;
-  helperNewLastF(pHelper)->fd = -1;
-}
-
-static int tsdbInitHelperFile(SRWHelper *pHelper) {
-  tsdbResetHelperFileImpl(pHelper);
-  return 0;
-}
-
-static void tsdbDestroyHelperFile(SRWHelper *pHelper) {
-  tsdbCloseHelperFile(pHelper, false, NULL);
-  tsdbResetHelperFileImpl(pHelper);
-  taosTZfree(pHelper->idxH.pIdxArray);
-  taosTZfree(pHelper->pWIdx);
-}
-
-// ---------- Operations on Helper Table part
-static void tsdbResetHelperTableImpl(SRWHelper *pHelper) {
-  memset((void *)&pHelper->tableInfo, 0, sizeof(SHelperTable));
-  pHelper->hasOldLastBlock = false;
-}
-
-static void tsdbResetHelperTable(SRWHelper *pHelper) {
-  tsdbResetHelperBlock(pHelper);
-  tsdbResetHelperTableImpl(pHelper);
-  helperClearState(pHelper, (TSDB_HELPER_TABLE_SET | TSDB_HELPER_INFO_LOAD));
-}
-
-static void tsdbInitHelperTable(SRWHelper *pHelper) { tsdbResetHelperTableImpl(pHelper); }
-
-static void tsdbDestroyHelperTable(SRWHelper *pHelper) { taosTZfree((void *)pHelper->pCompInfo); }
-
-// ---------- Operations on Helper Block part
-static void tsdbResetHelperBlockImpl(SRWHelper *pHelper) {
-  tdResetDataCols(pHelper->pDataCols[0]);
-  tdResetDataCols(pHelper->pDataCols[1]);
-}
-
-static void tsdbResetHelperBlock(SRWHelper *pHelper) {
-  tsdbResetHelperBlockImpl(pHelper);
-  // helperClearState(pHelper, TSDB_HELPER_)
-}
-
-static int tsdbInitHelperBlock(SRWHelper *pHelper) {
-  STsdbRepo *pRepo = helperRepo(pHelper);
-  STsdbMeta *pMeta = pHelper->pRepo->tsdbMeta;
-
-  pHelper->pDataCols[0] = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
-  pHelper->pDataCols[1] = tdNewDataCols(pMeta->maxRowBytes, pMeta->maxCols, pRepo->config.maxRowsPerFileBlock);
-  if (pHelper->pDataCols[0] == NULL || pHelper->pDataCols[1] == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  tsdbResetHelperBlockImpl(pHelper);
-
-  return 0;
-}
-
-static void tsdbDestroyHelperBlock(SRWHelper *pHelper) {
-  taosTZfree(pHelper->pCompData);
-  tdFreeDataCols(pHelper->pDataCols[0]);
-  tdFreeDataCols(pHelper->pDataCols[1]);
-}
-
-static int tsdbInitHelper(SRWHelper *pHelper, STsdbRepo *pRepo, tsdb_rw_helper_t type) {
-  STsdbCfg *pCfg = &pRepo->config;
-  memset((void *)pHelper, 0, sizeof(*pHelper));
-  STsdbMeta *pMeta = pRepo->tsdbMeta;
-
-  helperType(pHelper) = type;
-  helperRepo(pHelper) = pRepo;
-  helperState(pHelper) = TSDB_HELPER_CLEAR_STATE;
-
-  // Init file part
-  if (tsdbInitHelperFile(pHelper) < 0) goto _err;
-
-  // Init table part
-  tsdbInitHelperTable(pHelper);
-
-  // Init block part
-  if (tsdbInitHelperBlock(pHelper) < 0) goto _err;
-
-  // TODO: pMeta->maxRowBytes and pMeta->maxCols may change here causing invalid write
-  pHelper->pBuffer =
-      taosTMalloc(sizeof(SCompData) + (sizeof(SCompCol) + sizeof(TSCKSUM) + COMP_OVERFLOW_BYTES) * pMeta->maxCols +
-              pMeta->maxRowBytes * pCfg->maxRowsPerFileBlock + sizeof(TSCKSUM));
-  if (pHelper->pBuffer == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  return 0;
-
-_err:
-  tsdbDestroyHelper(pHelper);
-  return -1;
-}
-
-static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, char *content, int32_t len, int8_t comp, int numOfRows,
-                                        int maxPoints, char *buffer, int bufferSize) {
-  // Verify by checksum
-  if (!taosCheckChecksumWhole((uint8_t *)content, len)) {
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    return -1;
-  }
-
-  // Decode the data
-  if (comp) {
-    // // Need to decompress
-    int tlen = (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData,
-                                                             pDataCol->spaceSize, comp, buffer, bufferSize);
-    if (tlen <= 0) {
-      tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d",
-                len, comp, numOfRows, maxPoints, bufferSize);
-      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-      return -1;
-    }
-    pDataCol->len = tlen;
-    if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
-      dataColSetOffset(pDataCol, numOfRows);
-    }
-  } else {
-    // No need to decompress, just memcpy it
-    pDataCol->len = len - sizeof(TSCKSUM);
-    memcpy(pDataCol->pData, content, pDataCol->len);
-    if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
-      dataColSetOffset(pDataCol, numOfRows);
-    }
-  }
-  return 0;
-}
-
-static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBlock, SCompCol *pCompCol,
-                           SDataCol *pDataCol) {
-  ASSERT(pDataCol->colId == pCompCol->colId);
-  int tsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES;
-  pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pCompCol->len);
-  if (pHelper->pBuffer == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, tsize);
-  if (pHelper->compBuffer == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return -1;
-  }
-
-  int64_t offset = pCompBlock->offset + TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols) + pCompCol->offset;
-  if (lseek(pFile->fd, (off_t)offset, SEEK_SET) < 0) {
-    tsdbError("vgId:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pFile->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
-    tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname,
-              strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (tsdbCheckAndDecodeColumnData(pDataCol, pHelper->pBuffer, pCompCol->len, pCompBlock->algorithm,
-                                   pCompBlock->numOfRows, pHelper->pRepo->config.maxRowsPerFileBlock,
-                                   pHelper->compBuffer, (int32_t)taosTSizeof(pHelper->compBuffer)) < 0) {
-    tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pHelper->pRepo), pFile->fname,
-              pCompCol->colId, offset);
-    return -1;
-  }
-
-  return 0;
-}
-
-static int tsdbLoadBlockDataColsImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols, int16_t *colIds, int numOfColIds) {
-  ASSERT(pCompBlock->numOfSubBlocks <= 1);
-  ASSERT(colIds[0] == 0);
-
-  SFile *  pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper);
-  SCompCol compCol = {0};
-
-  // If only load timestamp column, no need to load SCompData part
-  if (numOfColIds > 1 && tsdbLoadCompData(pHelper, pCompBlock, NULL) < 0) goto _err;
-
-  pDataCols->numOfRows = pCompBlock->numOfRows;
-
-  int dcol = 0;
-  int ccol = 0;
-  for (int i = 0; i < numOfColIds; i++) {
-    int16_t   colId = colIds[i];
-    SDataCol *pDataCol = NULL;
-    SCompCol *pCompCol = NULL;
-
-    while (true) {
-      if (dcol >= pDataCols->numOfCols) {
-        pDataCol = NULL;
-        break;
-      }
-      pDataCol = &pDataCols->cols[dcol];
-      if (pDataCol->colId > colId) {
-        pDataCol = NULL;
-        break;
-      } else {
-        dcol++;
-        if (pDataCol->colId == colId) break;
-      }
-    }
-
-    if (pDataCol == NULL) continue;
-    ASSERT(pDataCol->colId == colId);
-
-    if (colId == 0) {  // load the key row
-      compCol.colId = colId;
-      compCol.len = pCompBlock->keyLen;
-      compCol.type = pDataCol->type;
-      compCol.offset = TSDB_KEY_COL_OFFSET;
-      pCompCol = &compCol;
-    } else {  // load non-key rows
-      while (true) {
-        if (ccol >= pCompBlock->numOfCols) {
-          pCompCol = NULL;
-          break;
-        }
-
-        pCompCol = &(pHelper->pCompData->cols[ccol]);
-        if (pCompCol->colId > colId) {
-          pCompCol = NULL;
-          break;
-        } else {
-          ccol++;
-          if (pCompCol->colId == colId) break;
-        }
-      }
-
-      if (pCompCol == NULL) {
-        dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
-        continue;
-      }
-
-      ASSERT(pCompCol->colId == pDataCol->colId);
-    }
-
-    if (tsdbLoadColData(pHelper, pFile, pCompBlock, pCompCol, pDataCol) < 0) goto _err;
-  }
-
-  return 0;
-
-_err:
-  return -1;
-}
-
-static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDataCols *pDataCols) {
-  ASSERT(pCompBlock->numOfSubBlocks <= 1);
-
-  SFile *pFile = (pCompBlock->last) ? helperLastF(pHelper) : helperDataF(pHelper);
-
-  pHelper->pBuffer = taosTRealloc(pHelper->pBuffer, pCompBlock->len);
-  if (pHelper->pBuffer == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  SCompData *pCompData = (SCompData *)pHelper->pBuffer;
-
-  int fd = pFile->fd;
-  if (lseek(fd, (off_t)pCompBlock->offset, SEEK_SET) < 0) {
-    tsdbError("vgId:%d tid:%d failed to lseek file %s since %s", REPO_ID(pHelper->pRepo), pHelper->tableInfo.tid,
-              pFile->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  if (taosRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
-    tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len,
-              pFile->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  int32_t tsize = TSDB_GET_COMPCOL_LEN(pCompBlock->numOfCols);
-  if (!taosCheckChecksumWhole((uint8_t *)pCompData, tsize)) {
-    tsdbError("vgId:%d file %s block data is corrupted offset %" PRId64 " len %d", REPO_ID(pHelper->pRepo),
-              pFile->fname, (int64_t)(pCompBlock->offset), pCompBlock->len);
-    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
-    goto _err;
-  }
-  ASSERT(pCompData->numOfCols == pCompBlock->numOfCols);
-
-  pDataCols->numOfRows = pCompBlock->numOfRows;
-
-  // Recover the data
-  int ccol = 0;  // loop iter for SCompCol object
-  int dcol = 0;  // loop iter for SDataCols object
-  while (dcol < pDataCols->numOfCols) {
-    SDataCol *pDataCol = &(pDataCols->cols[dcol]);
-    if (dcol != 0 && ccol >= pCompData->numOfCols) {
-      // Set current column as NULL and forward
-      dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
-      dcol++;
-      continue;
-    }
-
-    int16_t tcolId = 0;
-    int32_t toffset = TSDB_KEY_COL_OFFSET;
-    int32_t tlen = pCompBlock->keyLen;
-
-    if (dcol != 0) {
-      SCompCol *pCompCol = &(pCompData->cols[ccol]);
-      tcolId = pCompCol->colId;
-      toffset = pCompCol->offset;
-      tlen = pCompCol->len;
-    } else {
-      ASSERT(pDataCol->colId == tcolId);
-    }
-
-    if (tcolId == pDataCol->colId) {
-      if (pCompBlock->algorithm == TWO_STAGE_COMP) {
-        int zsize = pDataCol->bytes * pCompBlock->numOfRows + COMP_OVERFLOW_BYTES;
-        if (pDataCol->type == TSDB_DATA_TYPE_BINARY || pDataCol->type == TSDB_DATA_TYPE_NCHAR) {
-          zsize += (sizeof(VarDataLenT) * pCompBlock->numOfRows);
-        }
-        pHelper->compBuffer = taosTRealloc(pHelper->compBuffer, zsize);
-        if (pHelper->compBuffer == NULL) {
-          terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-          goto _err;
-        }
-      }
-      if (tsdbCheckAndDecodeColumnData(pDataCol, (char *)pCompData + tsize + toffset, tlen, pCompBlock->algorithm,
-                                       pCompBlock->numOfRows, pDataCols->maxPoints, pHelper->compBuffer,
-                                       (int32_t)taosTSizeof(pHelper->compBuffer)) < 0) {
-        tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d",
-                  REPO_ID(pHelper->pRepo), pFile->fname, tcolId, (int64_t)pCompBlock->offset, toffset);
-        goto _err;
-      }
-      if (dcol != 0) ccol++;
-      dcol++;
-    } else if (tcolId < pDataCol->colId) {
-      ccol++;
-    } else {
-      // Set current column as NULL and forward
-      dataColSetNEleNull(pDataCol, pCompBlock->numOfRows, pDataCols->maxPoints);
-      dcol++;
-    }
-  }
-
-  return 0;
-
-_err:
-  return -1;
-}
-
-static int tsdbEncodeSCompIdx(void **buf, SCompIdx *pIdx) {
-  int tlen = 0;
-
-  tlen += taosEncodeVariantI32(buf, pIdx->tid);
-  tlen += taosEncodeVariantU32(buf, pIdx->len);
-  tlen += taosEncodeVariantU32(buf, pIdx->offset);
-  tlen += taosEncodeFixedU8(buf, pIdx->hasLast);
-  tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks);
-  tlen += taosEncodeFixedU64(buf, pIdx->uid);
-  tlen += taosEncodeFixedU64(buf, pIdx->maxKey);
-
-  return tlen;
-}
-
-static void *tsdbDecodeSCompIdx(void *buf, SCompIdx *pIdx) {
-  uint8_t  hasLast = 0;
-  uint32_t numOfBlocks = 0;
-  uint64_t value = 0;
-
-  if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL;
-  if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL;
-  if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL;
-  if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL;
-  pIdx->hasLast = hasLast;
-  if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL;
-  pIdx->numOfBlocks = numOfBlocks;
-  if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
-  pIdx->uid = (int64_t)value;
-  if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
-  pIdx->maxKey = (TSKEY)value;
-
-  return buf;
-}
-
-static int tsdbProcessAppendCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey) {
-  STsdbCfg *  pCfg = &(pHelper->pRepo->config);
-  STable *    pTable = pCommitIter->pTable;
-  SCompIdx *  pIdx = &(pHelper->curCompIdx);
-  TSKEY       keyFirst = tsdbNextIterKey(pCommitIter->pIter);
-  int         defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
-  SCompBlock  compBlock = {0};
-  SMergeInfo  mergeInfo = {0};
-  SMergeInfo *pMergeInfo = &mergeInfo;
-
-  ASSERT(pIdx->len <= 0 || keyFirst > pIdx->maxKey);
-  if (pIdx->hasLast) {  // append to with last block
-    ASSERT(pIdx->len > 0);
-    SCompBlock *pCompBlock = blockAtIdx(pHelper, pIdx->numOfBlocks - 1);
-    ASSERT(pCompBlock->last && pCompBlock->numOfRows < pCfg->minRowsPerFileBlock);
-    tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock - pCompBlock->numOfRows, pDataCols,
-                          NULL, 0, pCfg->update, pMergeInfo);
-
-    ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
-
-    if (pDataCols->numOfRows > 0) {
-      ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
-
-      if (pDataCols->numOfRows + pCompBlock->numOfRows < pCfg->minRowsPerFileBlock &&
-          pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && !TSDB_NLAST_FILE_OPENED(pHelper)) {
-        if (tsdbWriteBlockToFile(pHelper, helperLastF(pHelper), pDataCols, &compBlock, true, false) < 0) return -1;
-        pMergeInfo->keyFirst = MIN(pMergeInfo->keyFirst, pCompBlock->keyFirst);
-        pMergeInfo->keyLast = MAX(pMergeInfo->keyLast, pCompBlock->keyLast);
-        if (tsdbAddSubBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1, pMergeInfo) < 0) return -1;
-      } else {
-        if (tsdbLoadBlockData(pHelper, pCompBlock, NULL) < 0) return -1;
-        ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows);
-
-        if (tdMergeDataCols(pHelper->pDataCols[0], pDataCols, pDataCols->numOfRows) < 0) return -1;
-        ASSERT(pHelper->pDataCols[0]->numOfRows == pCompBlock->numOfRows + pDataCols->numOfRows);
-
-        if (tsdbWriteBlockToProperFile(pHelper, pHelper->pDataCols[0], &compBlock) < 0) return -1;
-        if (tsdbUpdateSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks - 1) < 0) return -1;
-      }
-
-      if (pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
-    }
-  } else {
-    ASSERT(!pHelper->hasOldLastBlock);
-    tsdbLoadDataFromCache(pTable, pCommitIter->pIter, maxKey, defaultRowsInBlock, pDataCols, NULL, 0, pCfg->update, pMergeInfo);
-    ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
-
-    if (pDataCols->numOfRows > 0) {
-      ASSERT((pMergeInfo->keyFirst == dataColsKeyFirst(pDataCols)) && (pMergeInfo->keyLast == dataColsKeyLast(pDataCols)));
-      if (tsdbWriteBlockToProperFile(pHelper, pDataCols, &compBlock) < 0) return -1;
-      if (tsdbInsertSuperBlock(pHelper, &compBlock, pIdx->numOfBlocks) < 0) return -1;
-    }
-  }
-
-#ifndef NDEBUG
-  TSKEY keyNext = tsdbNextIterKey(pCommitIter->pIter);
-  ASSERT(keyNext == TSDB_DATA_TIMESTAMP_NULL || keyNext > pIdx->maxKey);
-#endif
-
-  return 0;
-}
-
-static int tsdbProcessMergeCommit(SRWHelper *pHelper, SCommitIter *pCommitIter, SDataCols *pDataCols, TSKEY maxKey,
-                                  int *blkIdx) {
-  STsdbCfg *  pCfg = &(pHelper->pRepo->config);
-  STable *    pTable = pCommitIter->pTable;
-  SCompIdx *  pIdx = &(pHelper->curCompIdx);
-  SCompBlock  compBlock = {0};
-  TSKEY       keyFirst = tsdbNextIterKey(pCommitIter->pIter);
-  int         defaultRowsInBlock = pCfg->maxRowsPerFileBlock * 4 / 5;
-  SDataCols * pDataCols0 = pHelper->pDataCols[0];
-  SMergeInfo  mergeInfo = {0};
-  SMergeInfo *pMergeInfo = &mergeInfo;
-  SCompBlock  oBlock = {0};
-
-  SSkipListIterator slIter = {0};
-
-  ASSERT(keyFirst <= pIdx->maxKey);
-
-  SCompBlock *pCompBlock = taosbsearch((void *)(&keyFirst), (void *)blockAtIdx(pHelper, *blkIdx),
-                                       pIdx->numOfBlocks - *blkIdx, sizeof(SCompBlock), compareKeyBlock, TD_GE);
-  ASSERT(pCompBlock != NULL);
-  int tblkIdx = (int32_t)(TSDB_GET_COMPBLOCK_IDX(pHelper, pCompBlock));
-  oBlock = *pCompBlock;
-
-  ASSERT((!TSDB_IS_LAST_BLOCK(&oBlock)) || (tblkIdx == pIdx->numOfBlocks - 1));
-
-  if ((!TSDB_IS_LAST_BLOCK(&oBlock)) && keyFirst < pCompBlock->keyFirst) {
-    while (true) {
-      tsdbLoadDataFromCache(pTable, pCommitIter->pIter, oBlock.keyFirst-1, defaultRowsInBlock, pDataCols, NULL, 0,
-                            pCfg->update, pMergeInfo);
-      ASSERT(pMergeInfo->rowsInserted == pMergeInfo->nOperations && pMergeInfo->nOperations == pDataCols->numOfRows);
-      if (pDataCols->numOfRows == 0) break;
-
-      if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
-      if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
-      tblkIdx++;
-    }
-    ASSERT(tblkIdx == 0 || (tsdbNextIterKey(pCommitIter->pIter) == TSDB_DATA_TIMESTAMP_NULL ||
-                            tsdbNextIterKey(pCommitIter->pIter) > blockAtIdx(pHelper, tblkIdx - 1)->keyLast));
-  } else {
-    int16_t colId = 0;
-    if (tsdbLoadBlockDataCols(pHelper, &oBlock, NULL, &colId, 1) < 0) return -1;
-
-    TSKEY keyLimit = (tblkIdx == pIdx->numOfBlocks - 1) ? maxKey : (blockAtIdx(pHelper, tblkIdx + 1)->keyFirst - 1);
-
-    slIter = *(pCommitIter->pIter);
-    tsdbLoadDataFromCache(pTable, &slIter, keyLimit, INT_MAX, NULL, pDataCols0->cols[0].pData, pDataCols0->numOfRows,
-                          pCfg->update, pMergeInfo);
-
-    if (pMergeInfo->nOperations == 0) {
-      // Do nothing
-      ASSERT(pMergeInfo->rowsDeleteFailed >= 0);
-      *(pCommitIter->pIter) = slIter;
-      tblkIdx++;
-    } else if (oBlock.numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed == 0) {
-      // Delete the block and do some stuff
-      // ASSERT(pMergeInfo->keyFirst == INT64_MAX && pMergeInfo->keyFirst == INT64_MIN);
-      if (tsdbDeleteSuperBlock(pHelper, tblkIdx) < 0) return -1;
-      *pCommitIter->pIter = slIter;
-      if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
-    } else if (tsdbCheckAddSubBlockCond(pHelper, &oBlock, pMergeInfo, pDataCols->maxPoints)) {
-      // Append as a sub-block of the searched block
-      tsdbLoadDataFromCache(pTable, pCommitIter->pIter, keyLimit, INT_MAX, pDataCols, pDataCols0->cols[0].pData,
-                            pDataCols0->numOfRows, pCfg->update, pMergeInfo);
-      ASSERT(memcmp(pCommitIter->pIter, &slIter, sizeof(slIter)) == 0);
-      if (tsdbWriteBlockToFile(pHelper, oBlock.last ? helperLastF(pHelper) : helperDataF(pHelper), pDataCols,
-                               &compBlock, oBlock.last, false) < 0) {
-        return -1;
-      }
-      if (tsdbAddSubBlock(pHelper, &compBlock, tblkIdx, pMergeInfo) < 0) {
-        return -1;
-      }
-      tblkIdx++;
-    } else {
-      // load the block data, merge with the memory data
-      if (tsdbLoadBlockData(pHelper, &oBlock, NULL) < 0) return -1;
-      int round = 0;
-      int dIter = 0;
-      while (true) {
-        tsdbLoadAndMergeFromCache(pDataCols0, &dIter, pCommitIter, pDataCols, keyLimit, defaultRowsInBlock,
-                                  pCfg->update);
-
-        if (pDataCols->numOfRows == 0) break;
-        if (tsdbWriteBlockToFile(pHelper, helperDataF(pHelper), pDataCols, &compBlock, false, true) < 0) return -1;
-
-        if (round == 0) {
-          if (oBlock.last && pHelper->hasOldLastBlock) pHelper->hasOldLastBlock = false;
-          if (tsdbUpdateSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
-        } else {
-          if (tsdbInsertSuperBlock(pHelper, &compBlock, tblkIdx) < 0) return -1;
-        }
-
-        round++;
-        tblkIdx++;
-      }
-    }
-  }
-
-  *blkIdx = tblkIdx;
-  return 0;
-}
-
-static void tsdbLoadAndMergeFromCache(SDataCols *pDataCols, int *iter, SCommitIter *pCommitIter, SDataCols *pTarget,
-                                     TSKEY maxKey, int maxRows, int8_t update) {
-  TSKEY     key1 = INT64_MAX;
-  TSKEY     key2 = INT64_MAX;
-  STSchema *pSchema = NULL;
-
-  ASSERT(maxRows > 0 && dataColsKeyLast(pDataCols) <= maxKey);
-  tdResetDataCols(pTarget);
-
-  while (true) {
-    key1 = (*iter >= pDataCols->numOfRows) ? INT64_MAX : dataColsKeyAt(pDataCols, *iter);
-    bool isRowDel = false;
-    SDataRow row = tsdbNextIterRow(pCommitIter->pIter);
-    if (row == NULL || dataRowKey(row) > maxKey) {
-      key2 = INT64_MAX;
-    } else {
-      key2 = dataRowKey(row);
-      isRowDel = dataRowDeleted(row);
-    }
-
-    if (key1 == INT64_MAX && key2 == INT64_MAX) break;
-
-    if (key1 < key2) {
-      for (int i = 0; i < pDataCols->numOfCols; i++) {
-        dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
-                         pTarget->maxPoints);
-      }
-
-      pTarget->numOfRows++;
-      (*iter)++;
-    } else if (key1 > key2) {
-      if (!isRowDel) {
-        if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
-          pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
-          ASSERT(pSchema != NULL);
-        }
-
-        tdAppendDataRowToDataCol(row, pSchema, pTarget);
-      }
-
-      tSkipListIterNext(pCommitIter->pIter);
-    } else {
-      if (update) {
-        if (!isRowDel) {
-          if (pSchema == NULL || schemaVersion(pSchema) != dataRowVersion(row)) {
-            pSchema = tsdbGetTableSchemaImpl(pCommitIter->pTable, false, false, dataRowVersion(row));
-            ASSERT(pSchema != NULL);
-          }
-
-          tdAppendDataRowToDataCol(row, pSchema, pTarget);
-        }
-      } else {
-        ASSERT(!isRowDel);
-
-        for (int i = 0; i < pDataCols->numOfCols; i++) {
-          dataColAppendVal(pTarget->cols + i, tdGetColDataOfRow(pDataCols->cols + i, *iter), pTarget->numOfRows,
-                           pTarget->maxPoints);
-        }
-
-        pTarget->numOfRows++;
-      }
-      (*iter)++;
-      tSkipListIterNext(pCommitIter->pIter);
-    }
-
-    if (pTarget->numOfRows >= maxRows) break;
-  }
-}
-
-static int tsdbWriteBlockToProperFile(SRWHelper *pHelper, SDataCols *pDataCols, SCompBlock *pCompBlock) {
-  STsdbCfg *pCfg = &(pHelper->pRepo->config);
-  SFile *   pFile = NULL;
-  bool      isLast = false;
-
-  ASSERT(pDataCols->numOfRows > 0);
-
-  if (pDataCols->numOfRows >= pCfg->minRowsPerFileBlock) {
-    pFile = helperDataF(pHelper);
-  } else {
-    isLast = true;
-    pFile = TSDB_NLAST_FILE_OPENED(pHelper) ? helperNewLastF(pHelper) : helperLastF(pHelper);
-  }
-
-  ASSERT(pFile->fd > 0);
-
-  if (tsdbWriteBlockToFile(pHelper, pFile, pDataCols, pCompBlock, isLast, true) < 0) return -1;
-
-  return 0;
-}
-
-static bool tsdbCheckAddSubBlockCond(SRWHelper *pHelper, SCompBlock *pCompBlock, SMergeInfo *pMergeInfo, int maxOps) {
-  STsdbCfg *pCfg = &(pHelper->pRepo->config);
-  int       mergeRows = pCompBlock->numOfRows + pMergeInfo->rowsInserted - pMergeInfo->rowsDeleteSucceed;
-
-  ASSERT(mergeRows > 0);
-
-  if (pCompBlock->numOfSubBlocks < TSDB_MAX_SUBBLOCKS && pMergeInfo->nOperations <= maxOps) {
-    if (pCompBlock->last) {
-      if (!TSDB_NLAST_FILE_OPENED(pHelper) && mergeRows < pCfg->minRowsPerFileBlock) return true;
-    } else {
-      if (mergeRows < pCfg->maxRowsPerFileBlock) return true;
-    }
-  }
-
-  return false;
-}
\ No newline at end of file

src/tsdb/src/tsdbRead.c

@@ -20,8 +20,7 @@
 #include "exception.h"
 
 #include "tlosertree.h"
-#include "tsdb.h"
-#include "tsdbMain.h"
+#include "tsdbint.h"
 #include "texpr.h"
 
 #define EXTRA_BYTES 2
@@ -54,7 +53,7 @@ typedef struct SQueryFilePos {
 } SQueryFilePos;
 
 typedef struct SDataBlockLoadInfo {
-  SFileGroup* fileGroup;
+  SDFileSet*  fileGroup;
   int32_t     slot;
   int32_t     tid;
   SArray*     pLoadedCols;
@@ -69,7 +68,7 @@ typedef struct STableCheckInfo {
   STableId      tableId;
   TSKEY         lastKey;
   STable*       pTableObj;
-  SCompInfo*    pCompInfo;
+  SBlockInfo*    pCompInfo;
   int32_t       compSize;
   int32_t       numOfBlocks:29; // number of qualified data blocks not the original blocks
   int8_t        chosen:2;       // indicate which iterator should move forward
@@ -79,7 +78,7 @@ typedef struct STableCheckInfo {
 } STableCheckInfo;
 
 typedef struct STableBlockInfo {
-  SCompBlock*        compBlock;
+  SBlock*        compBlock;
   STableCheckInfo*   pTableCheckInfo;
 } STableBlockInfo;
 
@@ -114,9 +113,9 @@ typedef struct STsdbQueryHandle {
   bool           loadExternalRow;  // load time window external data rows
   void*          qinfo;            // query info handle, for debug purpose
   int32_t        type;             // query type: retrieve all data blocks, 2. retrieve only last row, 3. retrieve direct prev|next rows
-  SFileGroup*    pFileGroup;
-  SFileGroupIter fileIter;
-  SRWHelper      rhelper;
+  SDFileSet*     pFileGroup;
+  SFSIter        fileIter;
+  SReadH         rhelper;
   STableBlockInfo* pDataBlockInfo;
 
   SDataCols     *pDataCols;        // in order to hold current file data block
@@ -142,7 +141,7 @@ static int32_t checkForCachedLastRow(STsdbQueryHandle* pQueryHandle, STableGroup
 static int32_t tsdbGetCachedLastRow(STable* pTable, SDataRow* pRes, TSKEY* lastKey);
 
 static void    changeQueryHandleForInterpQuery(TsdbQueryHandleT pHandle);
-static void    doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock);
+static void    doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock);
 static int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order);
 static int32_t tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int maxRowsToRead, STimeWindow* win, STsdbQueryHandle* pQueryHandle);
 static int32_t tsdbCheckInfoCompar(const void* key1, const void* key2);
@@ -308,7 +307,7 @@ static SArray* createCheckInfoFromCheckInfo(SArray* pTableCheckInfo, TSKEY skey)
   return pNew;
 }
 
-static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) {
+static STsdbQueryHandle* tsdbQueryTablesImpl(STsdbRepo* tsdb, STsdbQueryCond* pCond, void* qinfo, SMemRef* pMemRef) {
   STsdbQueryHandle* pQueryHandle = calloc(1, sizeof(STsdbQueryHandle));
   if (pQueryHandle == NULL) {
     goto out_of_memory;
@@ -329,7 +328,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
   pQueryHandle->pMemRef     = pMemRef;
   pQueryHandle->loadExternalRow = pCond->loadExternalRows;
 
-  if (tsdbInitReadHelper(&pQueryHandle->rhelper, (STsdbRepo*) tsdb) != 0) {
+  if (tsdbInitReadH(&pQueryHandle->rhelper, (STsdbRepo*)tsdb) != 0) {
     goto out_of_memory;
   }
 
@@ -388,7 +387,7 @@ static STsdbQueryHandle* tsdbQueryTablesImpl(TSDB_REPO_T* tsdb, STsdbQueryCond*
   return NULL;
 }
 
-TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) {
+TsdbQueryHandleT* tsdbQueryTables(STsdbRepo* tsdb, STsdbQueryCond* pCond, STableGroupInfo* groupList, void* qinfo, SMemRef* pRef) {
   STsdbQueryHandle* pQueryHandle = tsdbQueryTablesImpl(tsdb, pCond, qinfo, pRef);
 
   STsdbMeta* pMeta = tsdbGetMeta(tsdb);
@@ -406,7 +405,7 @@ TsdbQueryHandleT* tsdbQueryTables(TSDB_REPO_T* tsdb, STsdbQueryCond* pCond, STab
   return (TsdbQueryHandleT) pQueryHandle;
 }
 
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) {
+TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pMemRef) {
   pCond->twindow = updateLastrowForEachGroup(groupList);
 
   // no qualified table
@@ -442,7 +441,7 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle) {
   return res;
 }
 
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) {
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond* pCond, STableGroupInfo *groupList, void* qinfo, SMemRef* pRef) {
   STsdbQueryHandle *pQueryHandle = (STsdbQueryHandle*) tsdbQueryTables(tsdb, pCond, groupList, qinfo, pRef);
   pQueryHandle->loadExternalRow = true;
   if (pQueryHandle != NULL) {
@@ -713,7 +712,7 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio
   return (int32_t)fid;
 }
 
-static int32_t binarySearchForBlock(SCompBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) {
+static int32_t binarySearchForBlock(SBlock* pBlock, int32_t numOfBlocks, TSKEY skey, int32_t order) {
   int32_t firstSlot = 0;
   int32_t lastSlot = numOfBlocks - 1;
 
@@ -751,16 +750,16 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
     STableCheckInfo* pCheckInfo = taosArrayGet(pQueryHandle->pTableCheckInfo, i);
     pCheckInfo->numOfBlocks = 0;
 
-    if (tsdbSetHelperTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj, pQueryHandle->pTsdb) != TSDB_CODE_SUCCESS) {
+    if (tsdbSetReadTable(&pQueryHandle->rhelper, pCheckInfo->pTableObj) != TSDB_CODE_SUCCESS) {
       code = terrno;
       break;
     }
 
-    SCompIdx* compIndex = &pQueryHandle->rhelper.curCompIdx;
+    SBlockIdx* compIndex = pQueryHandle->rhelper.pBlkIdx;
 
     // no data block in this file, try next file
-    if (compIndex->len == 0 || compIndex->numOfBlocks == 0 || compIndex->uid != pCheckInfo->tableId.uid) {
-      continue; // no data blocks in the file belongs to pCheckInfo->pTable
+    if (compIndex == NULL || compIndex->uid != pCheckInfo->tableId.uid) {
+      continue;  // no data blocks in the file belongs to pCheckInfo->pTable
     }
 
     if (pCheckInfo->compSize < (int32_t)compIndex->len) {
@@ -773,12 +772,12 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
         break;
       }
 
-      pCheckInfo->pCompInfo = (SCompInfo*) t;
+      pCheckInfo->pCompInfo = (SBlockInfo*) t;
       pCheckInfo->compSize = compIndex->len;
     }
 
-    tsdbLoadCompInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo));
-    SCompInfo* pCompInfo = pCheckInfo->pCompInfo;
+    tsdbLoadBlockInfo(&(pQueryHandle->rhelper), (void *)(pCheckInfo->pCompInfo));
+    SBlockInfo* pCompInfo = pCheckInfo->pCompInfo;
 
     TSKEY s = TSKEY_INITIAL_VAL, e = TSKEY_INITIAL_VAL;
 
@@ -807,7 +806,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
     pCheckInfo->numOfBlocks = (end - start);
 
     if (start > 0) {
-      memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SCompBlock));
+      memmove(pCompInfo->blocks, &pCompInfo->blocks[start], pCheckInfo->numOfBlocks * sizeof(SBlock));
     }
 
     (*numOfBlocks) += pCheckInfo->numOfBlocks;
@@ -816,7 +815,7 @@ static int32_t getFileCompInfo(STsdbQueryHandle* pQueryHandle, int32_t* numOfBlo
   return code;
 }
 
-static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) {
+static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, int32_t slotIndex) {
   int64_t st = taosGetTimestampUs();
 
   STSchema *pSchema = tsdbGetTableSchema(pCheckInfo->pTableObj);
@@ -827,14 +826,14 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p
     goto _error;
   }
 
-  code = tdInitDataCols(pQueryHandle->rhelper.pDataCols[0], pSchema);
+  code = tdInitDataCols(pQueryHandle->rhelper.pDCols[0], pSchema);
   if (code != TSDB_CODE_SUCCESS) {
     tsdbError("%p failed to malloc buf for rhelper.pDataCols[0], %p", pQueryHandle, pQueryHandle->qinfo);
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
     goto _error;
   }
 
-  code = tdInitDataCols(pQueryHandle->rhelper.pDataCols[1], pSchema);
+  code = tdInitDataCols(pQueryHandle->rhelper.pDCols[1], pSchema);
   if (code != TSDB_CODE_SUCCESS) {
     tsdbError("%p failed to malloc buf for rhelper.pDataCols[1], %p", pQueryHandle, pQueryHandle->qinfo);
     terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
@@ -856,7 +855,7 @@ static int32_t doLoadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* p
   pBlockLoadInfo->slot = pQueryHandle->cur.slot;
   pBlockLoadInfo->tid = pCheckInfo->pTableObj->tableId.tid;
 
-  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  SDataCols* pCols = pQueryHandle->rhelper.pDCols[0];
   assert(pCols->numOfRows != 0 && pCols->numOfRows <= pBlock->numOfRows);
 
   pBlock->numOfRows = pCols->numOfRows;
@@ -882,7 +881,7 @@ static void moveDataToFront(STsdbQueryHandle* pQueryHandle, int32_t numOfRows, i
 static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle);
 static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos);
 
-static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo){
+static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo){
   SQueryFilePos* cur = &pQueryHandle->cur;
   STsdbCfg*      pCfg = &pQueryHandle->pTsdb->config;
   SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
@@ -965,7 +964,7 @@ static int32_t handleDataMergeIfNeeded(STsdbQueryHandle* pQueryHandle, SCompBloc
   return code;
 }
 
-static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) {
+static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SBlock* pBlock, STableCheckInfo* pCheckInfo, bool* exists) {
   SQueryFilePos* cur = &pQueryHandle->cur;
   int32_t code = TSDB_CODE_SUCCESS;
 
@@ -977,7 +976,7 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl
         return code;
       }
 
-      SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
+      SDataCols* pTSCol = pQueryHandle->rhelper.pDCols[0];
       assert(pTSCol->cols->type == TSDB_DATA_TYPE_TIMESTAMP && pTSCol->numOfRows == pBlock->numOfRows);
 
       if (pCheckInfo->lastKey > pBlock->keyFirst) {
@@ -1000,7 +999,7 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl
         return code;
       }
 
-      SDataCols* pTsCol = pQueryHandle->rhelper.pDataCols[0];
+      SDataCols* pTsCol = pQueryHandle->rhelper.pDCols[0];
       if (pCheckInfo->lastKey < pBlock->keyLast) {
         cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
@@ -1085,7 +1084,7 @@ int32_t doCopyRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, int32_t capacity
   char* pData = NULL;
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
 
-  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  SDataCols* pCols = pQueryHandle->rhelper.pDCols[0];
   TSKEY* tsArray = pCols->cols[0].pData;
 
   int32_t num = end - start + 1;
@@ -1309,7 +1308,7 @@ static void doCheckGeneratedBlockRange(STsdbQueryHandle* pQueryHandle) {
 static void copyAllRemainRowsFromFileBlock(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SDataBlockInfo* pBlockInfo, int32_t endPos) {
   SQueryFilePos* cur = &pQueryHandle->cur;
 
-  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  SDataCols* pCols = pQueryHandle->rhelper.pDCols[0];
   TSKEY* tsArray = pCols->cols[0].pData;
 
   int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1:-1;
@@ -1352,7 +1351,7 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
   int32_t order = ASCENDING_TRAVERSE(pQueryHandle->order)? TSDB_ORDER_DESC : TSDB_ORDER_ASC;
 
   SQueryFilePos* cur = &pQueryHandle->cur;
-  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  SDataCols* pCols = pQueryHandle->rhelper.pDCols[0];
 
   if (ASCENDING_TRAVERSE(pQueryHandle->order) && pQueryHandle->window.ekey >= pBlockInfo->window.ekey) {
     endPos = pBlockInfo->rows - 1;
@@ -1371,14 +1370,14 @@ int32_t getEndPosInDataBlock(STsdbQueryHandle* pQueryHandle, SDataBlockInfo* pBl
 
 // only return the qualified data to client in terms of query time window, data rows in the same block but do not
 // be included in the query time window will be discarded
-static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SCompBlock* pBlock) {
+static void doMergeTwoLevelData(STsdbQueryHandle* pQueryHandle, STableCheckInfo* pCheckInfo, SBlock* pBlock) {
   SQueryFilePos* cur = &pQueryHandle->cur;
   SDataBlockInfo blockInfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlock);
   STsdbCfg*      pCfg = &pQueryHandle->pTsdb->config;
 
   initTableMemIterator(pQueryHandle, pCheckInfo);
 
-  SDataCols* pCols = pQueryHandle->rhelper.pDataCols[0];
+  SDataCols* pCols = pQueryHandle->rhelper.pDCols[0];
   assert(pCols->cols[0].type == TSDB_DATA_TYPE_TIMESTAMP && pCols->cols[0].colId == PRIMARYKEY_TIMESTAMP_COL_INDEX &&
       cur->pos >= 0 && cur->pos < pBlock->numOfRows);
 
@@ -1670,7 +1669,7 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
       continue;
     }
 
-    SCompBlock* pBlock = pTableCheck->pCompInfo->blocks;
+    SBlock* pBlock = pTableCheck->pCompInfo->blocks;
     sup.numOfBlocksPerTable[numOfQualTables] = pTableCheck->numOfBlocks;
 
     char* buf = calloc(1, sizeof(STableBlockInfo) * pTableCheck->numOfBlocks);
@@ -1786,19 +1785,19 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
   STimeWindow win = TSWINDOW_INITIALIZER;
 
   while (true) {
-    pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+    tsdbRLockFS(REPO_FS(pQueryHandle->pTsdb));
 
-    if ((pQueryHandle->pFileGroup = tsdbGetFileGroupNext(&pQueryHandle->fileIter)) == NULL) {
-      pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+    if ((pQueryHandle->pFileGroup = tsdbFSIterNext(&pQueryHandle->fileIter)) == NULL) {
+      tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
       break;
     }
 
-    tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fileId, &win.skey, &win.ekey);
+    tsdbGetFidKeyRange(pCfg->daysPerFile, pCfg->precision, pQueryHandle->pFileGroup->fid, &win.skey, &win.ekey);
 
     // current file are not overlapped with query time window, ignore remain files
     if ((ASCENDING_TRAVERSE(pQueryHandle->order) && win.skey > pQueryHandle->window.ekey) ||
         (!ASCENDING_TRAVERSE(pQueryHandle->order) && win.ekey < pQueryHandle->window.ekey)) {
-      pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+      tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
       tsdbDebug("%p remain files are not qualified for qrange:%" PRId64 "-%" PRId64 ", ignore, %p", pQueryHandle,
                 pQueryHandle->window.skey, pQueryHandle->window.ekey, pQueryHandle->qinfo);
       pQueryHandle->pFileGroup = NULL;
@@ -1806,15 +1805,15 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
       break;
     }
 
-    if (tsdbSetAndOpenHelperFile(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) {
-      pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+    if (tsdbSetAndOpenReadFSet(&pQueryHandle->rhelper, pQueryHandle->pFileGroup) < 0) {
+      tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
       code = terrno;
       break;
     }
 
-    pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+    tsdbUnLockFS(REPO_FS(pQueryHandle->pTsdb));
 
-    if (tsdbLoadCompIdx(&pQueryHandle->rhelper, NULL) < 0) {
+    if (tsdbLoadBlockIdx(&pQueryHandle->rhelper) < 0) {
       code = terrno;
       break;
     }
@@ -1824,7 +1823,7 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
     }
 
     tsdbDebug("%p %d blocks found in file for %d table(s), fid:%d, %p", pQueryHandle, numOfBlocks, numOfTables,
-              pQueryHandle->pFileGroup->fileId, pQueryHandle->qinfo);
+              pQueryHandle->pFileGroup->fid, pQueryHandle->qinfo);
 
     assert(numOfBlocks >= 0);
     if (numOfBlocks == 0) {
@@ -1855,7 +1854,7 @@ static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exist
 
   assert(pQueryHandle->pFileGroup != NULL && pQueryHandle->numOfBlocks > 0);
   cur->slot = ASCENDING_TRAVERSE(pQueryHandle->order)? 0:pQueryHandle->numOfBlocks-1;
-  cur->fid = pQueryHandle->pFileGroup->fileId;
+  cur->fid = pQueryHandle->pFileGroup->fid;
 
   STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
   return getDataBlockRv(pQueryHandle, pBlockInfo, exists);
@@ -1878,7 +1877,7 @@ static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) {
 }
 
 static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) {
-  STsdbFileH*    pFileHandle = tsdbGetFile(pQueryHandle->pTsdb);
+  STsdbFS*       pFileHandle = REPO_FS(pQueryHandle->pTsdb);
   SQueryFilePos* cur = &pQueryHandle->cur;
 
   // find the start data block in file
@@ -1887,10 +1886,10 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
     STsdbCfg* pCfg = &pQueryHandle->pTsdb->config;
     int32_t   fid = getFileIdFromKey(pQueryHandle->window.skey, pCfg->daysPerFile, pCfg->precision);
 
-    pthread_rwlock_rdlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
-    tsdbInitFileGroupIter(pFileHandle, &pQueryHandle->fileIter, pQueryHandle->order);
-    tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);
-    pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
+    tsdbRLockFS(pFileHandle);
+    tsdbFSIterInit(&pQueryHandle->fileIter, pFileHandle, pQueryHandle->order);
+    tsdbFSIterSeek(&pQueryHandle->fileIter, fid);
+    tsdbUnLockFS(pFileHandle);
 
     return getFirstFileDataBlock(pQueryHandle, exists);
   } else {
@@ -2482,7 +2481,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
   }
 
   int64_t stime = taosGetTimestampUs();
-  tsdbLoadCompData(&pHandle->rhelper, pBlockInfo->compBlock, NULL);
+  tsdbLoadBlockStatis(&pHandle->rhelper, pBlockInfo->compBlock);
 
   int16_t* colIds = pHandle->defaultLoadColumn->pData;
 
@@ -2492,7 +2491,7 @@ int32_t tsdbRetrieveDataBlockStatisInfo(TsdbQueryHandleT* pQueryHandle, SDataSta
     pHandle->statis[i].colId = colIds[i];
   }
 
-  tsdbGetDataStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols);
+  tsdbGetBlockStatis(&pHandle->rhelper, pHandle->statis, (int)numOfCols);
 
   // always load the first primary timestamp column data
   SDataStatis* pPrimaryColStatis = &pHandle->statis[0];
@@ -2544,11 +2543,11 @@ SArray* tsdbRetrieveDataBlock(TsdbQueryHandleT* pQueryHandle, SArray* pIdList) {
       // data block has been loaded, todo extract method
       SDataBlockLoadInfo* pBlockLoadInfo = &pHandle->dataBlockLoadInfo;
 
-      if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fileId == pHandle->cur.fid &&
+      if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid &&
           pBlockLoadInfo->tid == pCheckInfo->pTableObj->tableId.tid) {
         return pHandle->pColumns;
       } else {  // only load the file block
-        SCompBlock* pBlock = pBlockInfo->compBlock;
+        SBlock* pBlock = pBlockInfo->compBlock;
         if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) {
           return NULL;
         }
@@ -2820,7 +2819,7 @@ static int32_t doQueryTableList(STable* pSTable, SArray* pRes, tExprNode* pExpr)
   return TSDB_CODE_SUCCESS;
 }
 
-int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
+int32_t tsdbQuerySTableByTagCond(STsdbRepo* tsdb, uint64_t uid, TSKEY skey, const char* pTagCond, size_t len,
                                  int16_t tagNameRelType, const char* tbnameCond, STableGroupInfo* pGroupInfo,
                                  SColIndex* pColIndex, int32_t numOfCols) {
   if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
@@ -2915,7 +2914,7 @@ int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY skey, co
   return terrno;
 }
 
-int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) {
+int32_t tsdbGetOneTableGroup(STsdbRepo* tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo* pGroupInfo) {
   if (tsdbRLockRepoMeta(tsdb) < 0) goto _error;
 
   STable* pTable = tsdbGetTableByUid(tsdbGetMeta(tsdb), uid);
@@ -2945,7 +2944,7 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T* tsdb, uint64_t uid, TSKEY startKey, ST
   return terrno;
 }
 
-int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) {
+int32_t tsdbGetTableGroupFromIdList(STsdbRepo* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo) {
   if (tsdbRLockRepoMeta(tsdb) < 0) {
     return terrno;
   }
@@ -3031,7 +3030,7 @@ void tsdbCleanupQueryHandle(TsdbQueryHandleT queryHandle) {
   // todo check error
   tsdbMayUnTakeMemSnapshot(pQueryHandle);
 
-  tsdbDestroyHelper(&pQueryHandle->rhelper);
+  tsdbDestroyReadH(&pQueryHandle->rhelper);
 
   tdFreeDataCols(pQueryHandle->pDataCols);
   pQueryHandle->pDataCols = NULL;

src/tsdb/src/tsdbReadImpl.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "tsdbint.h"
+
+#define TSDB_KEY_COL_OFFSET 0
+
+static void tsdbResetReadTable(SReadH *pReadh);
+static void tsdbResetReadFile(SReadH *pReadh);
+static int  tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols);
+static int  tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
+                                         int maxPoints, char *buffer, int bufferSize);
+static int  tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds,
+                                      int numOfColIds);
+static int  tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol);
+
+int tsdbInitReadH(SReadH *pReadh, STsdbRepo *pRepo) {
+  ASSERT(pReadh != NULL && pRepo != NULL);
+
+  STsdbCfg *pCfg = REPO_CFG(pRepo);
+
+  memset((void *)pReadh, 0, sizeof(*pReadh));
+  pReadh->pRepo = pRepo;
+
+  TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh));
+
+  pReadh->aBlkIdx = taosArrayInit(1024, sizeof(SBlockIdx));
+  if (pReadh->aBlkIdx == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  pReadh->pDCols[0] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
+  if (pReadh->pDCols[0] == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbDestroyReadH(pReadh);
+    return -1;
+  }
+
+  pReadh->pDCols[1] = tdNewDataCols(0, 0, pCfg->maxRowsPerFileBlock);
+  if (pReadh->pDCols[1] == NULL) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    tsdbDestroyReadH(pReadh);
+    return -1;
+  }
+
+  return 0;
+}
+
+void tsdbDestroyReadH(SReadH *pReadh) {
+  if (pReadh == NULL) return;
+
+  pReadh->pCBuf = taosTZfree(pReadh->pCBuf);
+  pReadh->pBuf = taosTZfree(pReadh->pBuf);
+  pReadh->pDCols[0] = tdFreeDataCols(pReadh->pDCols[0]);
+  pReadh->pDCols[1] = tdFreeDataCols(pReadh->pDCols[1]);
+  pReadh->pBlkData = taosTZfree(pReadh->pBlkData);
+  pReadh->pBlkInfo = taosTZfree(pReadh->pBlkInfo);
+  pReadh->cidx = 0;
+  pReadh->pBlkIdx = NULL;
+  pReadh->pTable = NULL;
+  pReadh->aBlkIdx = taosArrayDestroy(pReadh->aBlkIdx);
+  tsdbCloseDFileSet(TSDB_READ_FSET(pReadh));
+  pReadh->pRepo = NULL;
+}
+
+int tsdbSetAndOpenReadFSet(SReadH *pReadh, SDFileSet *pSet) {
+  ASSERT(pSet != NULL);
+  tsdbResetReadFile(pReadh);
+
+  pReadh->rSet = *pSet;
+  TSDB_FSET_SET_CLOSED(TSDB_READ_FSET(pReadh));
+  if (tsdbOpenDFileSet(TSDB_READ_FSET(pReadh), O_RDONLY) < 0) {
+    tsdbError("vgId:%d failed to open file set %d since %s", TSDB_READ_REPO_ID(pReadh), TSDB_FSET_FID(pSet),
+              tstrerror(terrno));
+    return -1;
+  }
+
+  return 0;
+}
+
+void tsdbCloseAndUnsetFSet(SReadH *pReadh) { tsdbResetReadFile(pReadh); }
+
+int tsdbLoadBlockIdx(SReadH *pReadh) {
+  SDFile *  pHeadf = TSDB_READ_HEAD_FILE(pReadh);
+  SBlockIdx blkIdx;
+
+  ASSERT(taosArrayGetSize(pReadh->aBlkIdx) == 0);
+
+  // No data at all, just return
+  if (pHeadf->info.offset <= 0) return 0;
+
+  if (tsdbSeekDFile(pHeadf, pHeadf->info.offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load SBlockIdx part while seek file %s since %s, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset,
+              pHeadf->info.len);
+    return -1;
+  }
+
+  if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pHeadf->info.len) < 0) return -1;
+
+  int64_t nread = tsdbReadDFile(pHeadf, TSDB_READ_BUF(pReadh), pHeadf->info.len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load SBlockIdx part while read file %s since %s, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pHeadf->info.offset,
+              pHeadf->info.len);
+    return -1;
+  }
+
+  if (nread < pHeadf->info.len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockIdx part in file %s is corrupted, offset:%u expected bytes:%u read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len, nread);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), pHeadf->info.len)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockIdx part in file %s is corrupted since wrong checksum, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pHeadf->info.offset, pHeadf->info.len);
+    return -1;
+  }
+
+  void *ptr = TSDB_READ_BUF(pReadh);
+  int   tsize = 0;
+  while (POINTER_DISTANCE(ptr, TSDB_READ_BUF(pReadh)) < (pHeadf->info.len - sizeof(TSCKSUM))) {
+    ptr = tsdbDecodeSBlockIdx(ptr, &blkIdx);
+    ASSERT(ptr != NULL);
+
+    if (taosArrayPush(pReadh->aBlkIdx, (void *)(&blkIdx)) < 0) {
+      terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+      return -1;
+    }
+
+    tsize++;
+    ASSERT(tsize == 1 || ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 2))->tid <
+                             ((SBlockIdx *)taosArrayGet(pReadh->aBlkIdx, tsize - 1))->tid);
+  }
+
+  return 0;
+}
+
+int tsdbSetReadTable(SReadH *pReadh, STable *pTable) {
+  STSchema *pSchema = tsdbGetTableSchemaImpl(pTable, false, false, -1);
+
+  pReadh->pTable = pTable;
+
+  if (tdInitDataCols(pReadh->pDCols[0], pSchema) < 0) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  if (tdInitDataCols(pReadh->pDCols[1], pSchema) < 0) {
+    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
+    return -1;
+  }
+
+  size_t size = taosArrayGetSize(pReadh->aBlkIdx);
+  if (size > 0) {
+    while (true) {
+      if (pReadh->cidx >= size) {
+        pReadh->pBlkIdx = NULL;
+        break;
+      }
+
+      SBlockIdx *pBlkIdx = taosArrayGet(pReadh->aBlkIdx, pReadh->cidx);
+      if (pBlkIdx->tid == TABLE_TID(pTable)) {
+        if (pBlkIdx->uid == TABLE_UID(pTable)) {
+          pReadh->pBlkIdx = pBlkIdx;
+        } else {
+          pReadh->pBlkIdx = NULL;
+        }
+        pReadh->cidx++;
+        break;
+      } else if (pBlkIdx->tid > TABLE_TID(pTable)) {
+        pReadh->pBlkIdx = NULL;
+        break;
+      } else {
+        pReadh->cidx++;
+      }
+    }
+  } else {
+    pReadh->pBlkIdx = NULL;
+  }
+
+  return 0;
+}
+
+int tsdbLoadBlockInfo(SReadH *pReadh, void *pTarget) {
+  ASSERT(pReadh->pBlkIdx != NULL);
+
+  SDFile *   pHeadf = TSDB_READ_HEAD_FILE(pReadh);
+  SBlockIdx *pBlkIdx = pReadh->pBlkIdx;
+
+  if (tsdbSeekDFile(pHeadf, pBlkIdx->offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load SBlockInfo part while seek file %s since %s, offset:%u len:%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  if (tsdbMakeRoom((void **)(&(pReadh->pBlkInfo)), pBlkIdx->len) < 0) return -1;
+
+  int64_t nread = tsdbReadDFile(pHeadf, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load SBlockInfo part while read file %s since %s, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), tstrerror(terrno), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  if (nread < pBlkIdx->len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockInfo part in file %s is corrupted, offset:%u expected bytes:%u read bytes:%" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len, nread);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkInfo), pBlkIdx->len)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d SBlockInfo part in file %s is corrupted since wrong checksum, offset:%u len :%u",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pHeadf), pBlkIdx->offset, pBlkIdx->len);
+    return -1;
+  }
+
+  ASSERT(pBlkIdx->tid == pReadh->pBlkInfo->tid && pBlkIdx->uid == pReadh->pBlkInfo->uid);
+
+  if (pTarget) {
+    memcpy(pTarget, (void *)(pReadh->pBlkInfo), pBlkIdx->len);
+  }
+
+  return 0;
+}
+
+int tsdbLoadBlockData(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo) {
+  ASSERT(pBlock->numOfSubBlocks > 0);
+
+  SBlock *iBlock = pBlock;
+  if (pBlock->numOfSubBlocks > 1) {
+    if (pBlkInfo) {
+      iBlock = (SBlock *)POINTER_SHIFT(pBlkInfo, pBlock->offset);
+    } else {
+      iBlock = (SBlock *)POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset);
+    }
+  }
+
+  if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[0]) < 0) return -1;
+  for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
+    iBlock++;
+    if (tsdbLoadBlockDataImpl(pReadh, iBlock, pReadh->pDCols[1]) < 0) return -1;
+    if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows) < 0) return -1;
+  }
+
+  ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
+  ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
+  ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
+
+  return 0;
+}
+
+int tsdbLoadBlockDataCols(SReadH *pReadh, SBlock *pBlock, SBlockInfo *pBlkInfo, int16_t *colIds, int numOfColsIds) {
+  ASSERT(pBlock->numOfSubBlocks > 0);
+
+  SBlock *iBlock = pBlock;
+  if (pBlock->numOfSubBlocks > 1) {
+    if (pBlkInfo) {
+      iBlock = POINTER_SHIFT(pBlkInfo, pBlock->offset);
+    } else {
+      iBlock = POINTER_SHIFT(pReadh->pBlkInfo, pBlock->offset);
+    }
+  }
+
+  if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[0], colIds, numOfColsIds) < 0) return -1;
+  for (int i = 1; i < pBlock->numOfSubBlocks; i++) {
+    iBlock++;
+    if (tsdbLoadBlockDataColsImpl(pReadh, iBlock, pReadh->pDCols[1], colIds, numOfColsIds) < 0) return -1;
+    if (tdMergeDataCols(pReadh->pDCols[0], pReadh->pDCols[1], pReadh->pDCols[1]->numOfRows) < 0) return -1;
+  }
+
+  ASSERT(pReadh->pDCols[0]->numOfRows == pBlock->numOfRows);
+  ASSERT(dataColsKeyFirst(pReadh->pDCols[0]) == pBlock->keyFirst);
+  ASSERT(dataColsKeyLast(pReadh->pDCols[0]) == pBlock->keyLast);
+
+  return 0;
+}
+
+int tsdbLoadBlockStatis(SReadH *pReadh, SBlock *pBlock) {
+  ASSERT(pBlock->numOfSubBlocks <= 1);
+
+  SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
+
+  if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load block statis part while seek file %s to offset %" PRId64 " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno));
+    return -1;
+  }
+
+  size_t size = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols);
+  if (tsdbMakeRoom((void **)(&(pReadh->pBlkData)), size) < 0) return -1;
+
+  int64_t nread = tsdbReadDFile(pDFile, (void *)(pReadh->pBlkData), size);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load block statis part while read file %s since %s, offset:%" PRId64 " len :%" PRIzu,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset, size);
+    return -1;
+  }
+
+  if (nread < size) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block statis part in file %s is corrupted, offset:%" PRId64 " expected bytes:%" PRIzu
+              " read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size, nread);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)(pReadh->pBlkData), (uint32_t)size)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%" PRIzu,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, size);
+    return -1;
+  }
+
+  return 0;
+}
+
+int tsdbEncodeSBlockIdx(void **buf, SBlockIdx *pIdx) {
+  int tlen = 0;
+
+  tlen += taosEncodeVariantI32(buf, pIdx->tid);
+  tlen += taosEncodeVariantU32(buf, pIdx->len);
+  tlen += taosEncodeVariantU32(buf, pIdx->offset);
+  tlen += taosEncodeFixedU8(buf, pIdx->hasLast);
+  tlen += taosEncodeVariantU32(buf, pIdx->numOfBlocks);
+  tlen += taosEncodeFixedU64(buf, pIdx->uid);
+  tlen += taosEncodeFixedU64(buf, pIdx->maxKey);
+
+  return tlen;
+}
+
+void *tsdbDecodeSBlockIdx(void *buf, SBlockIdx *pIdx) {
+  uint8_t  hasLast = 0;
+  uint32_t numOfBlocks = 0;
+  uint64_t value = 0;
+
+  if ((buf = taosDecodeVariantI32(buf, &(pIdx->tid))) == NULL) return NULL;
+  if ((buf = taosDecodeVariantU32(buf, &(pIdx->len))) == NULL) return NULL;
+  if ((buf = taosDecodeVariantU32(buf, &(pIdx->offset))) == NULL) return NULL;
+  if ((buf = taosDecodeFixedU8(buf, &(hasLast))) == NULL) return NULL;
+  pIdx->hasLast = hasLast;
+  if ((buf = taosDecodeVariantU32(buf, &(numOfBlocks))) == NULL) return NULL;
+  pIdx->numOfBlocks = numOfBlocks;
+  if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
+  pIdx->uid = (int64_t)value;
+  if ((buf = taosDecodeFixedU64(buf, &value)) == NULL) return NULL;
+  pIdx->maxKey = (TSKEY)value;
+
+  return buf;
+}
+
+void tsdbGetBlockStatis(SReadH *pReadh, SDataStatis *pStatis, int numOfCols) {
+  SBlockData *pBlockData = pReadh->pBlkData;
+
+  for (int i = 0, j = 0; i < numOfCols;) {
+    if (j >= pBlockData->numOfCols) {
+      pStatis[i].numOfNull = -1;
+      i++;
+      continue;
+    }
+
+    if (pStatis[i].colId == pBlockData->cols[j].colId) {
+      pStatis[i].sum = pBlockData->cols[j].sum;
+      pStatis[i].max = pBlockData->cols[j].max;
+      pStatis[i].min = pBlockData->cols[j].min;
+      pStatis[i].maxIndex = pBlockData->cols[j].maxIndex;
+      pStatis[i].minIndex = pBlockData->cols[j].minIndex;
+      pStatis[i].numOfNull = pBlockData->cols[j].numOfNull;
+      i++;
+      j++;
+    } else if (pStatis[i].colId < pBlockData->cols[j].colId) {
+      pStatis[i].numOfNull = -1;
+      i++;
+    } else {
+      j++;
+    }
+  }
+}
+
+static void tsdbResetReadTable(SReadH *pReadh) {
+  tdResetDataCols(pReadh->pDCols[0]);
+  tdResetDataCols(pReadh->pDCols[1]);
+  pReadh->cidx = 0;
+  pReadh->pBlkIdx = NULL;
+  pReadh->pTable = NULL;
+}
+
+static void tsdbResetReadFile(SReadH *pReadh) {
+  tsdbResetReadTable(pReadh);
+  taosArrayClear(pReadh->aBlkIdx);
+  tsdbCloseDFileSet(TSDB_READ_FSET(pReadh));
+}
+
+static int tsdbLoadBlockDataImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols) {
+  ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
+
+  SDFile *pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
+
+  tdResetDataCols(pDataCols);
+  if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlock->len) < 0) return -1;
+
+  SBlockData *pBlockData = (SBlockData *)TSDB_READ_BUF(pReadh);
+
+  if (tsdbSeekDFile(pDFile, pBlock->offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load block data part while seek file %s to offset %" PRId64 " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tstrerror(terrno));
+    return -1;
+  }
+
+  int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlock->len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load block data part while read file %s since %s, offset:%" PRId64 " len :%d",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), (int64_t)pBlock->offset,
+              pBlock->len);
+    return -1;
+  }
+
+  if (nread < pBlock->len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block data part in file %s is corrupted, offset:%" PRId64
+              " expected bytes:%d read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, pBlock->len, nread);
+    return -1;
+  }
+
+  int32_t tsize = TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols);
+  if (!taosCheckChecksumWhole((uint8_t *)TSDB_READ_BUF(pReadh), tsize)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block statis part in file %s is corrupted since wrong checksum, offset:%" PRId64 " len :%d",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), (int64_t)pBlock->offset, tsize);
+    return -1;
+  }
+
+  ASSERT(tsize < pBlock->len);
+  ASSERT(pBlockData->numOfCols == pBlock->numOfCols);
+
+  pDataCols->numOfRows = pBlock->numOfRows;
+
+  // Recover the data
+  int ccol = 0;  // loop iter for SBlockCol object
+  int dcol = 0;  // loop iter for SDataCols object
+  while (dcol < pDataCols->numOfCols) {
+    SDataCol *pDataCol = &(pDataCols->cols[dcol]);
+    if (dcol != 0 && ccol >= pBlockData->numOfCols) {
+      // Set current column as NULL and forward
+      dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints);
+      dcol++;
+      continue;
+    }
+
+    int16_t tcolId = 0;
+    int32_t toffset = TSDB_KEY_COL_OFFSET;
+    int32_t tlen = pBlock->keyLen;
+
+    if (dcol != 0) {
+      SBlockCol *pBlockCol = &(pBlockData->cols[ccol]);
+      tcolId = pBlockCol->colId;
+      toffset = pBlockCol->offset;
+      tlen = pBlockCol->len;
+    } else {
+      ASSERT(pDataCol->colId == tcolId);
+    }
+
+    if (tcolId == pDataCol->colId) {
+      if (pBlock->algorithm == TWO_STAGE_COMP) {
+        int zsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES;
+        if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), zsize) < 0) return -1;
+      }
+
+      if (tsdbCheckAndDecodeColumnData(pDataCol, POINTER_SHIFT(pBlockData, tsize + toffset), tlen, pBlock->algorithm,
+                                       pBlock->numOfRows, pDataCols->maxPoints, TSDB_READ_COMP_BUF(pReadh),
+                                       (int)taosTSizeof(TSDB_READ_COMP_BUF(pReadh))) < 0) {
+        tsdbError("vgId:%d file %s is broken at column %d block offset %" PRId64 " column offset %d",
+                  TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tcolId, (int64_t)pBlock->offset, toffset);
+        return -1;
+      }
+
+      if (dcol != 0) {
+        ccol++;
+      }
+      dcol++;
+    } else if (tcolId < pDataCol->colId) {
+      ccol++;
+    } else {
+      // Set current column as NULL and forward
+      dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints);
+      dcol++;
+    }
+  }
+
+  return 0;
+}
+
+static int tsdbCheckAndDecodeColumnData(SDataCol *pDataCol, void *content, int32_t len, int8_t comp, int numOfRows,
+                                        int maxPoints, char *buffer, int bufferSize) {
+  if (!taosCheckChecksumWhole((uint8_t *)content, len)) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    return -1;
+  }
+
+  // Decode the data
+  if (comp) {
+    // Need to decompress
+    int tlen = (*(tDataTypes[pDataCol->type].decompFunc))(content, len - sizeof(TSCKSUM), numOfRows, pDataCol->pData,
+                                                             pDataCol->spaceSize, comp, buffer, bufferSize);
+    if (tlen <= 0) {
+      tsdbError("Failed to decompress column, file corrupted, len:%d comp:%d numOfRows:%d maxPoints:%d bufferSize:%d",
+                len, comp, numOfRows, maxPoints, bufferSize);
+      terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+      return -1;
+    }
+    pDataCol->len = tlen;
+  } else {
+    // No need to decompress, just memcpy it
+    pDataCol->len = len - sizeof(TSCKSUM);
+    memcpy(pDataCol->pData, content, pDataCol->len);
+  }
+
+  if (IS_VAR_DATA_TYPE(pDataCol->type)) {
+    dataColSetOffset(pDataCol, numOfRows);
+  }
+  return 0;
+}
+
+static int tsdbLoadBlockDataColsImpl(SReadH *pReadh, SBlock *pBlock, SDataCols *pDataCols, int16_t *colIds,
+                                     int numOfColIds) {
+  ASSERT(pBlock->numOfSubBlocks == 0 || pBlock->numOfSubBlocks == 1);
+  ASSERT(colIds[0] == 0);
+
+  SDFile *  pDFile = (pBlock->last) ? TSDB_READ_LAST_FILE(pReadh) : TSDB_READ_DATA_FILE(pReadh);
+  SBlockCol blockCol = {0};
+
+  tdResetDataCols(pDataCols);
+
+  // If only load timestamp column, no need to load SBlockData part
+  if (numOfColIds > 1 && tsdbLoadBlockStatis(pReadh, pBlock) < 0) return -1;
+
+  pDataCols->numOfRows = pBlock->numOfRows;
+
+  int dcol = 0;
+  int ccol = 0;
+  for (int i = 0; i < numOfColIds; i++) {
+    int16_t    colId = colIds[i];
+    SDataCol * pDataCol = NULL;
+    SBlockCol *pBlockCol = NULL;
+
+    while (true) {
+      if (dcol >= pDataCols->numOfCols) {
+        pDataCol = NULL;
+        break;
+      }
+      pDataCol = &pDataCols->cols[dcol];
+      if (pDataCol->colId > colId) {
+        pDataCol = NULL;
+        break;
+      } else {
+        dcol++;
+        if (pDataCol->colId == colId) break;
+      }
+    }
+
+    if (pDataCol == NULL) continue;
+    ASSERT(pDataCol->colId == colId);
+
+    if (colId == 0) {  // load the key row
+      blockCol.colId = colId;
+      blockCol.len = pBlock->keyLen;
+      blockCol.type = pDataCol->type;
+      blockCol.offset = TSDB_KEY_COL_OFFSET;
+      pBlockCol = &blockCol;
+    } else {  // load non-key rows
+      while (true) {
+        if (ccol >= pBlock->numOfCols) {
+          pBlockCol = NULL;
+          break;
+        }
+
+        pBlockCol = &(pReadh->pBlkData->cols[ccol]);
+        if (pBlockCol->colId > colId) {
+          pBlockCol = NULL;
+          break;
+        } else {
+          ccol++;
+          if (pBlockCol->colId == colId) break;
+        }
+      }
+
+      if (pBlockCol == NULL) {
+        dataColSetNEleNull(pDataCol, pBlock->numOfRows, pDataCols->maxPoints);
+        continue;
+      }
+
+      ASSERT(pBlockCol->colId == pDataCol->colId);
+    }
+
+    if (tsdbLoadColData(pReadh, pDFile, pBlock, pBlockCol, pDataCol) < 0) return -1;
+  }
+
+  return 0;
+}
+
+static int tsdbLoadColData(SReadH *pReadh, SDFile *pDFile, SBlock *pBlock, SBlockCol *pBlockCol, SDataCol *pDataCol) {
+  ASSERT(pDataCol->colId == pBlockCol->colId);
+
+  STsdbRepo *pRepo = TSDB_READ_REPO(pReadh);
+  STsdbCfg * pCfg = REPO_CFG(pRepo);
+  int        tsize = pDataCol->bytes * pBlock->numOfRows + COMP_OVERFLOW_BYTES;
+
+  if (tsdbMakeRoom((void **)(&TSDB_READ_BUF(pReadh)), pBlockCol->len) < 0) return -1;
+  if (tsdbMakeRoom((void **)(&TSDB_READ_COMP_BUF(pReadh)), tsize) < 0) return -1;
+
+  int64_t offset = pBlock->offset + TSDB_BLOCK_STATIS_SIZE(pBlock->numOfCols) + pBlockCol->offset;
+  if (tsdbSeekDFile(pDFile, offset, SEEK_SET) < 0) {
+    tsdbError("vgId:%d failed to load block column data while seek file %s to offset %" PRId64 " since %s",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, tstrerror(terrno));
+    return -1;
+  }
+
+  int64_t nread = tsdbReadDFile(pDFile, TSDB_READ_BUF(pReadh), pBlockCol->len);
+  if (nread < 0) {
+    tsdbError("vgId:%d failed to load block column data while read file %s since %s, offset:%" PRId64 " len :%d",
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), tstrerror(terrno), offset, pBlockCol->len);
+    return -1;
+  }
+
+  if (nread < pBlockCol->len) {
+    terrno = TSDB_CODE_TDB_FILE_CORRUPTED;
+    tsdbError("vgId:%d block column data in file %s is corrupted, offset:%" PRId64 " expected bytes:%d" PRIzu
+              " read bytes: %" PRId64,
+              TSDB_READ_REPO_ID(pReadh), TSDB_FILE_FULL_NAME(pDFile), offset, pBlockCol->len, nread);
+    return -1;
+  }
+
+  if (tsdbCheckAndDecodeColumnData(pDataCol, pReadh->pBuf, pBlockCol->len, pBlock->algorithm, pBlock->numOfRows,
+                                   pCfg->maxRowsPerFileBlock, pReadh->pCBuf, (int32_t)taosTSizeof(pReadh->pCBuf)) < 0) {
+    tsdbError("vgId:%d file %s is broken at column %d offset %" PRId64, REPO_ID(pRepo), TSDB_FILE_FULL_NAME(pDFile),
+              pBlockCol->colId, offset);
+    return -1;
+  }
+
+  return 0;
+}
\ No newline at end of file

src/tsdb/src/tsdbScan.c

@@ -13,8 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdbMain.h"
+#include "tsdbint.h"
 
+#if 0
 #ifndef _TSDB_PLUGINS
 
 int tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; }
@@ -25,12 +26,13 @@ void tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream) {}
 
 int tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid) { return 0; }
 
-int tsdbScanSCompIdx(STsdbScanHandle* pScanHandle) { return 0; }
+int tsdbScanSBlockIdx(STsdbScanHandle* pScanHandle) { return 0; }
 
-int tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; }
+int tsdbScanSBlock(STsdbScanHandle* pScanHandle, int idx) { return 0; }
 
 int tsdbCloseScanFile(STsdbScanHandle* pScanHandle) { return 0; }
 
 void tsdbFreeScanHandle(STsdbScanHandle* pScanHandle) {}
 
+#endif
 #endif
\ No newline at end of file

src/tsdb/src/tsdbSync.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "taoserror.h"
+#include "tsdbint.h"
+
+// Sync handle
+typedef struct {
+  STsdbRepo *pRepo;
+  SRtn       rtn;
+  SOCKET     socketFd;
+  void *     pBuf;
+  bool       mfChanged;
+  SMFile *   pmf;
+  SMFile     mf;
+  SDFileSet  df;
+  SDFileSet *pdf;
+} SSyncH;
+
+#define SYNC_BUFFER(sh) ((sh)->pBuf)
+
+static void    tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd);
+static void    tsdbDestroySyncH(SSyncH *pSyncH);
+static int32_t tsdbSyncSendMeta(SSyncH *pSynch);
+static int32_t tsdbSyncRecvMeta(SSyncH *pSynch);
+static int32_t tsdbSendMetaInfo(SSyncH *pSynch);
+static int32_t tsdbRecvMetaInfo(SSyncH *pSynch);
+static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend);
+static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend);
+static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch);
+static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch);
+static bool    tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2);
+static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet);
+static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet);
+static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch);
+static int     tsdbReload(STsdbRepo *pRepo, bool isMfChanged);
+
+int32_t tsdbSyncSend(void *tsdb, SOCKET socketFd) {
+  STsdbRepo *pRepo = (STsdbRepo *)tsdb;
+  SSyncH     synch = {0};
+
+  tsdbInitSyncH(&synch, pRepo, socketFd);
+  // Disable TSDB commit
+  tsem_wait(&(pRepo->readyToCommit));
+
+  if (tsdbSyncSendMeta(&synch) < 0) {
+    tsdbError("vgId:%d, failed to send metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  if (tsdbSyncSendDFileSetArray(&synch) < 0) {
+    tsdbError("vgId:%d, failed to send filesets since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  // Enable TSDB commit
+  tsem_post(&(pRepo->readyToCommit));
+  tsdbDestroySyncH(&synch);
+  return 0;
+
+_err:
+  tsem_post(&(pRepo->readyToCommit));
+  tsdbDestroySyncH(&synch);
+  return -1;
+}
+
+int32_t tsdbSyncRecv(void *tsdb, SOCKET socketFd) {
+  STsdbRepo *pRepo = (STsdbRepo *)tsdb;
+  SSyncH synch = {0};
+
+  pRepo->state = TSDB_STATE_OK;
+
+  tsdbInitSyncH(&synch, pRepo, socketFd);
+  tsdbStartFSTxn(pRepo, 0, 0);
+
+  if (tsdbSyncRecvMeta(&synch) < 0) {
+    tsdbError("vgId:%d, failed to recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  if (tsdbSyncRecvDFileSetArray(&synch) < 0) {
+    tsdbError("vgId:%d, failed to recv filesets since %s", REPO_ID(pRepo), tstrerror(terrno));
+    goto _err;
+  }
+
+  tsdbEndFSTxn(pRepo);
+  tsdbDestroySyncH(&synch);
+
+  // Reload file change
+  tsdbReload(pRepo, synch.mfChanged);
+
+  return 0;
+
+_err:
+  tsdbEndFSTxnWithError(REPO_FS(pRepo));
+  tsdbDestroySyncH(&synch);
+  return -1;
+}
+
+static void tsdbInitSyncH(SSyncH *pSyncH, STsdbRepo *pRepo, SOCKET socketFd) {
+  pSyncH->pRepo = pRepo;
+  pSyncH->socketFd = socketFd;
+  tsdbGetRtnSnap(pRepo, &(pSyncH->rtn));
+}
+
+static void tsdbDestroySyncH(SSyncH *pSyncH) { taosTZfree(pSyncH->pBuf); }
+
+static int32_t tsdbSyncSendMeta(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  bool       toSendMeta = false;
+  SMFile     mf;
+
+  // Send meta info to remote
+  tsdbInfo("vgId:%d, metainfo will be sent", REPO_ID(pRepo));
+  if (tsdbSendMetaInfo(pSynch) < 0) {
+    tsdbError("vgId:%d, failed to send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  if (pRepo->fs->cstatus->pmf == NULL) {
+    // No meta file, not need to wait to retrieve meta file
+    tsdbInfo("vgId:%d, metafile not exist, no need to send", REPO_ID(pRepo));
+    return 0;
+  }
+
+  if (tsdbRecvDecision(pSynch, &toSendMeta) < 0) {
+    tsdbError("vgId:%d, failed to recv decision while send meta since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  if (toSendMeta) {
+    tsdbInitMFileEx(&mf, pRepo->fs->cstatus->pmf);
+    if (tsdbOpenMFile(&mf, O_RDONLY) < 0) {
+      tsdbError("vgId:%d, failed to open file while send metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+
+    int32_t writeLen = (int32_t)mf.info.size;
+    tsdbInfo("vgId:%d, metafile:%s will be sent, size:%d", REPO_ID(pRepo), mf.f.aname, writeLen);
+
+    int32_t ret = (int32_t)taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&mf), 0, writeLen);
+    if (ret != writeLen) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      tsdbError("vgId:%d, failed to send metafile since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret,
+                writeLen);
+      tsdbCloseMFile(&mf);
+      return -1;
+    }
+
+    tsdbCloseMFile(&mf);
+    tsdbInfo("vgId:%d, metafile is sent", REPO_ID(pRepo));
+  } else {
+    tsdbInfo("vgId:%d, metafile is same, no need to send", REPO_ID(pRepo));
+  }
+
+  return 0;
+}
+
+static int32_t tsdbSyncRecvMeta(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  SMFile *   pLMFile = pRepo->fs->cstatus->pmf;
+
+  // Recv meta info from remote
+  if (tsdbRecvMetaInfo(pSynch) < 0) {
+    tsdbError("vgId:%d, failed to recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  // No meta file, do nothing (rm local meta file)
+  if (pSynch->pmf == NULL) {
+    if (pLMFile == NULL) {
+      pSynch->mfChanged = false;
+    } else {
+      pSynch->mfChanged = true;
+    }
+    tsdbInfo("vgId:%d, metafile not exist in remote, no need to recv", REPO_ID(pRepo));
+    return 0;
+  }
+
+  if (pLMFile == NULL || memcmp(&(pSynch->pmf->info), &(pLMFile->info), sizeof(SMFInfo)) != 0) {
+    // Local has no meta file or has a different meta file, need to copy from remote
+    pSynch->mfChanged = true;
+
+    if (tsdbSendDecision(pSynch, true) < 0) {
+      tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+
+    tsdbInfo("vgId:%d, metafile will be received", REPO_ID(pRepo));
+
+    // Recv from remote
+    SMFile  mf;
+    SDiskID did = {.level = TFS_PRIMARY_LEVEL, .id = TFS_PRIMARY_ID};
+    tsdbInitMFile(&mf, did, REPO_ID(pRepo), FS_TXN_VERSION(REPO_FS(pRepo)));
+    if (tsdbCreateMFile(&mf, false) < 0) {
+      tsdbError("vgId:%d, failed to create file while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+
+    tsdbInfo("vgId:%d, metafile:%s is created", REPO_ID(pRepo), mf.f.aname);
+
+    int32_t readLen = (int32_t)pSynch->pmf->info.size;
+    int32_t ret = taosCopyFds(pSynch->socketFd, TSDB_FILE_FD(&mf), readLen);
+    if (ret != readLen) {
+      terrno = TAOS_SYSTEM_ERROR(errno);
+      tsdbError("vgId:%d, failed to recv metafile since %s, ret:%d readLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret,
+                readLen);
+      tsdbCloseMFile(&mf);
+      tsdbRemoveMFile(&mf);
+      return -1;
+    }
+
+    tsdbInfo("vgId:%d, metafile is received, size:%d", REPO_ID(pRepo), readLen);
+
+    mf.info = pSynch->pmf->info;
+    tsdbCloseMFile(&mf);
+    tsdbUpdateMFile(REPO_FS(pRepo), &mf);
+  } else {
+    pSynch->mfChanged = false;
+    tsdbInfo("vgId:%d, metafile is same, no need to recv", REPO_ID(pRepo));
+    if (tsdbSendDecision(pSynch, false) < 0) {
+      tsdbError("vgId:%d, failed to send decision while recv metafile since %s", REPO_ID(pRepo), tstrerror(terrno));
+      return -1;
+    }
+    tsdbUpdateMFile(REPO_FS(pRepo), pLMFile);
+  }
+
+  return 0;
+}
+
+static int32_t tsdbSendMetaInfo(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint32_t   tlen = 0;
+  SMFile *   pMFile = pRepo->fs->cstatus->pmf;
+
+  if (pMFile) {
+    tlen = tlen + tsdbEncodeSMFileEx(NULL, pMFile) + sizeof(TSCKSUM);
+  }
+
+  if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) {
+    tsdbError("vgId:%d, failed to makeroom while send metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  void *ptr = SYNC_BUFFER(pSynch);
+  taosEncodeFixedU32(&ptr, tlen);
+  void *tptr = ptr;
+  if (pMFile) {
+    tsdbEncodeSMFileEx(&ptr, pMFile);
+    taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen);
+  }
+
+  int32_t writeLen = tlen + sizeof(uint32_t);
+  int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen);
+  if (ret != writeLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to send metainfo since %s, ret:%d writeLen:%d", REPO_ID(pRepo), tstrerror(terrno), ret,
+              writeLen);
+    return -1;
+  }
+
+  tsdbInfo("vgId:%d, metainfo is sent, tlen:%d, writeLen:%d", REPO_ID(pRepo), tlen, writeLen);
+  return 0;
+}
+
+static int32_t tsdbRecvMetaInfo(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint32_t   tlen = 0;
+  char       buf[64] = {0};
+
+  int32_t readLen = sizeof(uint32_t);
+  int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen);
+  if (ret != readLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to recv metalen, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen);
+    return -1;
+  }
+
+  taosDecodeFixedU32(buf, &tlen);
+
+  tsdbInfo("vgId:%d, metalen is received, readLen:%d, tlen:%d", REPO_ID(pRepo), readLen, tlen);
+  if (tlen == 0) {
+    pSynch->pmf = NULL;
+    return 0;
+  }
+
+  if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) {
+    tsdbError("vgId:%d, failed to makeroom while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen);
+  if (ret != tlen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to recv metainfo, ret:%d tlen:%d", REPO_ID(pRepo), ret, tlen);
+    return -1;
+  }
+
+  tsdbInfo("vgId:%d, metainfo is received, tlen:%d", REPO_ID(pRepo), tlen);
+  if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) {
+    terrno = TSDB_CODE_TDB_MESSED_MSG;
+    tsdbError("vgId:%d, failed to checksum while recv metainfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  pSynch->pmf = &(pSynch->mf);
+  tsdbDecodeSMFileEx(SYNC_BUFFER(pSynch), pSynch->pmf);
+
+  return 0;
+}
+
+static int32_t tsdbSendDecision(SSyncH *pSynch, bool toSend) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint8_t    decision = toSend;
+
+  int32_t writeLen = sizeof(uint8_t);
+  int32_t ret = taosWriteMsg(pSynch->socketFd, (void *)(&decision), writeLen);
+  if (ret != writeLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to send decison, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen);
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t tsdbRecvDecision(SSyncH *pSynch, bool *toSend) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint8_t    decision = 0;
+
+  int32_t readLen = sizeof(uint8_t);
+  int32_t ret = taosReadMsg(pSynch->socketFd, (void *)(&decision), readLen);
+  if (ret != readLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to recv decison, ret:%d readLen:%d", REPO_ID(pRepo), ret, readLen);
+    return -1;
+  }
+
+  *toSend = decision;
+  return 0;
+}
+
+static int32_t tsdbSyncSendDFileSetArray(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  STsdbFS *  pfs = REPO_FS(pRepo);
+  SFSIter    fsiter;
+  SDFileSet *pSet;
+
+  tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
+
+  do {
+    pSet = tsdbFSIterNext(&fsiter);
+    if (tsdbSyncSendDFileSet(pSynch, pSet) < 0) {
+      tsdbError("vgId:%d, failed to send fileset:%d since %s", REPO_ID(pRepo), pSet ? pSet->fid : -1,
+                tstrerror(terrno));
+      return -1;
+    }
+
+    // No more file set to send, jut break
+    if (pSet == NULL) {
+      tsdbInfo("vgId:%d, no filesets any more", REPO_ID(pRepo));
+      break;
+    }
+  } while (true);
+
+  return 0;
+}
+
+static int32_t tsdbSyncRecvDFileSetArray(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  STsdbFS *  pfs = REPO_FS(pRepo);
+  SFSIter    fsiter;
+  SDFileSet *pLSet;  // Local file set
+
+  tsdbFSIterInit(&fsiter, pfs, TSDB_FS_ITER_FORWARD);
+
+  pLSet = tsdbFSIterNext(&fsiter);
+  if (tsdbRecvDFileSetInfo(pSynch) < 0) {
+    tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  while (true) {
+    if (pLSet == NULL && pSynch->pdf == NULL) {
+      tsdbInfo("vgId:%d, all filesets is disposed", REPO_ID(pRepo));
+      break;
+    } else {
+      tsdbInfo("vgId:%d, fileset local:%d remote:%d, will be disposed", REPO_ID(pRepo), pLSet != NULL ? pLSet->fid : -1,
+               pSynch->pdf != NULL ? pSynch->pdf->fid : -1);
+    }
+
+    if (pLSet && (pSynch->pdf == NULL || pLSet->fid < pSynch->pdf->fid)) {
+      // remote not has pLSet->fid set, just remove local (do nothing to remote the fset)
+      tsdbInfo("vgId:%d, fileset:%d smaller than remote:%d, remove it", REPO_ID(pRepo), pLSet->fid,
+               pSynch->pdf != NULL ? pSynch->pdf->fid : -1);
+      pLSet = tsdbFSIterNext(&fsiter);
+    } else {
+      if (pLSet && pSynch->pdf && pLSet->fid == pSynch->pdf->fid && tsdbIsTowFSetSame(pLSet, pSynch->pdf)) {
+        // Just keep local files and notify remote not to send
+        tsdbInfo("vgId:%d, fileset:%d is same and no need to recv", REPO_ID(pRepo), pLSet->fid);
+
+        if (tsdbUpdateDFileSet(pfs, pLSet) < 0) {
+          tsdbError("vgId:%d, failed to update fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
+          return -1;
+        }
+
+        if (tsdbSendDecision(pSynch, false) < 0) {
+          tsdbError("vgId:%d, filed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
+          return -1;
+        }
+      } else {
+        // Need to copy from remote
+        tsdbInfo("vgId:%d, fileset:%d will be received", REPO_ID(pRepo), pSynch->pdf->fid);
+
+        // Notify remote to send there file here
+        if (tsdbSendDecision(pSynch, true) < 0) {
+          tsdbError("vgId:%d, failed to send decision since %s", REPO_ID(pRepo), tstrerror(terrno));
+          return -1;
+        }
+
+        // Create local files and copy from remote
+        SDiskID   did;
+        SDFileSet fset;
+
+        tfsAllocDisk(tsdbGetFidLevel(pSynch->pdf->fid, &(pSynch->rtn)), &(did.level), &(did.id));
+        if (did.level == TFS_UNDECIDED_LEVEL) {
+          terrno = TSDB_CODE_TDB_NO_AVAIL_DISK;
+          tsdbError("vgId:%d, failed allc disk since %s", REPO_ID(pRepo), tstrerror(terrno));
+          return -1;
+        }
+
+        tsdbInitDFileSet(&fset, did, REPO_ID(pRepo), pSynch->pdf->fid, FS_TXN_VERSION(pfs));
+
+        // Create new FSET
+        if (tsdbCreateDFileSet(&fset, false) < 0) {
+          tsdbError("vgId:%d, failed to create fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
+          return -1;
+        }
+
+        for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+          SDFile *pDFile = TSDB_DFILE_IN_SET(&fset, ftype);         // local file
+          SDFile *pRDFile = TSDB_DFILE_IN_SET(pSynch->pdf, ftype);  // remote file
+
+          tsdbInfo("vgId:%d, file:%s will be received, osize:%" PRIu64 " rsize:%" PRIu64, REPO_ID(pRepo),
+                   pDFile->f.aname, pDFile->info.size, pRDFile->info.size);
+
+          int32_t writeLen = (int32_t)pRDFile->info.size;
+          int32_t ret = taosCopyFds(pSynch->socketFd, pDFile->fd, writeLen);
+          if (ret != writeLen) {
+            terrno = TAOS_SYSTEM_ERROR(errno);
+            tsdbError("vgId:%d, failed to recv file:%s since %s, ret:%d writeLen:%d", REPO_ID(pRepo), pDFile->f.aname,
+                      tstrerror(terrno), ret, writeLen);
+            tsdbCloseDFileSet(&fset);
+            tsdbRemoveDFileSet(&fset);
+            return -1;
+          }
+
+          // Update new file info
+          pDFile->info = pRDFile->info;
+          tsdbInfo("vgId:%d, file:%s is received, size:%d", REPO_ID(pRepo), pDFile->f.aname, writeLen);
+        }
+
+        tsdbCloseDFileSet(&fset);
+        if (tsdbUpdateDFileSet(pfs, &fset) < 0) {
+          tsdbInfo("vgId:%d, fileset:%d failed to update since %s", REPO_ID(pRepo), fset.fid, tstrerror(terrno));
+          return -1;
+        }
+
+        tsdbInfo("vgId:%d, fileset:%d is received", REPO_ID(pRepo), pSynch->pdf->fid);
+      }
+
+      // Move forward
+      if (tsdbRecvDFileSetInfo(pSynch) < 0) {
+        tsdbError("vgId:%d, failed to recv fileset since %s", REPO_ID(pRepo), tstrerror(terrno));
+        return -1;
+      }
+
+      if (pLSet) {
+        pLSet = tsdbFSIterNext(&fsiter);
+      }
+    }
+
+#if 0
+    if (pLSet == NULL) {
+      // Copy from remote >>>>>>>>>>>
+    } else {
+      if (pSynch->pdf == NULL) {
+        // Remove local file, just ignore ++++++++++++++
+        pLSet = tsdbFSIterNext(&fsiter);
+      } else {
+        if (pLSet->fid < pSynch->pdf->fid) {
+          // Remove local file, just ignore ++++++++++++
+          pLSet = tsdbFSIterNext(&fsiter);
+        } else if (pLSet->fid > pSynch->pdf->fid){
+          // Copy from remote >>>>>>>>>>>>>>
+          if (tsdbRecvDFileSetInfo(pSynch) < 0) {
+            // TODO
+            return -1;
+          }
+        } else {
+          if (true/*TODO: is same fset*/) {
+            // No need to copy ---------------------
+          } else {
+            // copy from remote >>>>>>>>>>>>>.
+          }
+        }
+      }
+    }
+#endif
+  }
+
+  return 0;
+}
+
+static bool tsdbIsTowFSetSame(SDFileSet *pSet1, SDFileSet *pSet2) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    SDFile *pDFile1 = TSDB_DFILE_IN_SET(pSet1, ftype);
+    SDFile *pDFile2 = TSDB_DFILE_IN_SET(pSet2, ftype);
+
+    if (memcmp((void *)(TSDB_FILE_INFO(pDFile1)), (void *)(TSDB_FILE_INFO(pDFile2)), sizeof(SDFInfo)) != 0) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+static int32_t tsdbSyncSendDFileSet(SSyncH *pSynch, SDFileSet *pSet) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  bool toSend = false;
+
+  if (tsdbSendDFileSetInfo(pSynch, pSet) < 0) {
+    tsdbError("vgId:%d, failed to send fileset:%d info since %s", REPO_ID(pRepo), pSet->fid, tstrerror(terrno));
+    return -1;
+  }
+
+  // No file any more, no need to send file, just return
+  if (pSet == NULL) {
+    return 0;
+  }
+
+  if (tsdbRecvDecision(pSynch, &toSend) < 0) {
+    tsdbError("vgId:%d, failed to recv decision while send fileset:%d since %s", REPO_ID(pRepo), pSet->fid,
+              tstrerror(terrno));
+    return -1;
+  }
+
+  if (toSend) {
+    tsdbInfo("vgId:%d, fileset:%d will be sent", REPO_ID(pRepo), pSet->fid);
+
+    for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+      SDFile df = *TSDB_DFILE_IN_SET(pSet, ftype);
+      
+      if (tsdbOpenDFile(&df, O_RDONLY) < 0) {
+        tsdbError("vgId:%d, failed to file:%s since %s", REPO_ID(pRepo), df.f.aname, tstrerror(terrno));
+        return -1;
+      }
+
+      int32_t writeLen = (int32_t)df.info.size;
+      tsdbInfo("vgId:%d, file:%s will be sent, size:%d", REPO_ID(pRepo), df.f.aname, writeLen);
+
+      int32_t ret = (int32_t)taosSendFile(pSynch->socketFd, TSDB_FILE_FD(&df), 0, writeLen);
+      if (ret != writeLen) {
+        terrno = TAOS_SYSTEM_ERROR(errno);
+        tsdbError("vgId:%d, failed to send file:%s since %s, ret:%d writeLen:%d", REPO_ID(pRepo), df.f.aname,
+                  tstrerror(terrno), ret, writeLen);
+        tsdbCloseDFile(&df);
+        return -1;
+      }
+
+      tsdbInfo("vgId:%d, file:%s is sent", REPO_ID(pRepo), df.f.aname);
+      tsdbCloseDFile(&df);
+    }
+
+    tsdbInfo("vgId:%d, fileset:%d is sent", REPO_ID(pRepo), pSet->fid);
+  } else {
+    tsdbInfo("vgId:%d, fileset:%d is same, no need to send", REPO_ID(pRepo), pSet->fid);
+  }
+
+  return 0;
+}
+
+static int32_t tsdbSendDFileSetInfo(SSyncH *pSynch, SDFileSet *pSet) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint32_t   tlen = 0;
+
+  if (pSet) {
+    tlen = tsdbEncodeDFileSetEx(NULL, pSet) + sizeof(TSCKSUM);
+  }
+
+  if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen + sizeof(tlen)) < 0) {
+    tsdbError("vgId:%d, failed to makeroom while send fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  void *ptr = SYNC_BUFFER(pSynch);
+  taosEncodeFixedU32(&ptr, tlen);
+  void *tptr = ptr;
+  if (pSet) {
+    tsdbEncodeDFileSetEx(&ptr, pSet);
+    taosCalcChecksumAppend(0, (uint8_t *)tptr, tlen);
+  }
+
+  int32_t writeLen = tlen + sizeof(uint32_t);
+  int32_t ret = taosWriteMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), writeLen);
+  if (ret != writeLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to send fileinfo, ret:%d writeLen:%d", REPO_ID(pRepo), ret, writeLen);
+    return -1;
+  }
+
+  return 0;
+}
+
+static int32_t tsdbRecvDFileSetInfo(SSyncH *pSynch) {
+  STsdbRepo *pRepo = pSynch->pRepo;
+  uint32_t   tlen;
+  char       buf[64] = {0};
+
+  int32_t readLen = sizeof(uint32_t);
+  int32_t ret = taosReadMsg(pSynch->socketFd, buf, readLen);
+  if (ret != readLen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  taosDecodeFixedU32(buf, &tlen);
+
+  tsdbInfo("vgId:%d, fileinfo len:%d is received", REPO_ID(pRepo), tlen);
+  if (tlen == 0) {
+    pSynch->pdf = NULL;
+    return 0;
+  }
+
+  if (tsdbMakeRoom((void **)(&SYNC_BUFFER(pSynch)), tlen) < 0) {
+    tsdbError("vgId:%d, failed to makeroom while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  ret = taosReadMsg(pSynch->socketFd, SYNC_BUFFER(pSynch), tlen);
+  if (ret != tlen) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    tsdbError("vgId:%d, failed to recv fileinfo, ret:%d readLen:%d", REPO_ID(pRepo), ret, tlen);
+    return -1;
+  }
+
+  if (!taosCheckChecksumWhole((uint8_t *)SYNC_BUFFER(pSynch), tlen)) {
+    terrno = TSDB_CODE_TDB_MESSED_MSG;
+    tsdbError("vgId:%d, failed to checksum while recv fileinfo since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  pSynch->pdf = &(pSynch->df);
+  tsdbDecodeDFileSetEx(SYNC_BUFFER(pSynch), pSynch->pdf);
+
+  return 0;
+}
+
+static int tsdbReload(STsdbRepo *pRepo, bool isMfChanged) {
+  // TODO: may need to stop and restart stream
+  if (isMfChanged) {
+    tsdbCloseMeta(pRepo);
+    tsdbFreeMeta(pRepo->tsdbMeta);
+    pRepo->tsdbMeta = tsdbNewMeta(REPO_CFG(pRepo));
+    tsdbOpenMeta(pRepo);
+    tsdbLoadMetaCache(pRepo, true);
+  }
+
+  tsdbUnRefMemTable(pRepo, pRepo->mem);
+  tsdbUnRefMemTable(pRepo, pRepo->imem);
+  pRepo->mem = NULL;
+  pRepo->imem = NULL;
+
+  if (tsdbRestoreInfo(pRepo) < 0) {
+    tsdbError("vgId:%d failed to restore info from file since %s", REPO_ID(pRepo), tstrerror(terrno));
+    return -1;
+  }
+
+  return 0;
+}
\ No newline at end of file

src/tsdb/tests/tsdbTests.cpp

@@ -12,7 +12,7 @@ static double getCurTime() {
 }
 
 typedef struct {
-  TSDB_REPO_T *pRepo;
+  STsdbRepo *pRepo;
   bool       isAscend;
   int        tid;
   uint64_t   uid;
@@ -143,7 +143,7 @@ TEST(TsdbTest, testInsertSpeed) {
   // Create and open repository
   tsdbSetCfg(&tsdbCfg, 1, 16, 4, -1, -1, -1, -1, -1, -1, -1);
   tsdbCreateRepo(rootDir, &tsdbCfg);
-  TSDB_REPO_T *repo = tsdbOpenRepo(rootDir, NULL);
+  STsdbRepo *repo = tsdbOpenRepo(rootDir, NULL);
   ASSERT_NE(repo, nullptr);
 
   // Create table

src/util/CMakeLists.txt

@@ -33,3 +33,7 @@ ELSEIF(TD_DARWIN)
   TARGET_LINK_LIBRARIES(tutil m)
   TARGET_LINK_LIBRARIES(tutil iconv)
 ENDIF()
+
+IF (TD_STORAGE)
+  TARGET_LINK_LIBRARIES(tutil storage)
+ENDIF ()
\ No newline at end of file

src/util/inc/tarray.h

@@ -21,9 +21,11 @@ extern "C" {
 #endif
 
 #include "os.h"
+#include "talgo.h"
 
 #define TARRAY_MIN_SIZE 8
 #define TARRAY_GET_ELEM(array, index) ((void*)((char*)((array)->pData) + (index) * (array)->elemSize))
+#define TARRAY_ELEM_IDX(array, ele) (POINTER_DISTANCE(ele, (array)->pData) / (array)->elemSize)
 
 typedef struct SArray {
   size_t size;
@@ -44,9 +46,20 @@ void* taosArrayInit(size_t size, size_t elemSize);
  *
  * @param pArray
  * @param pData
+ * @param nEles
  * @return
  */
-void* taosArrayPush(SArray* pArray, void* pData);
+void *taosArrayPushBatch(SArray *pArray, const void *pData, int nEles);
+
+/**
+ *
+ * @param pArray
+ * @param pData
+ * @return
+ */
+static FORCE_INLINE void* taosArrayPush(SArray* pArray, const void* pData) {
+  return taosArrayPushBatch(pArray, pData, 1);
+}
 
 /**
  *
@@ -92,6 +105,14 @@ size_t taosArrayGetSize(const SArray* pArray);
  */
 void* taosArrayInsert(SArray* pArray, size_t index, void* pData);
 
+/**
+ * set data in array
+ * @param pArray
+ * @param index
+ * @param pData
+ */
+void taosArraySet(SArray* pArray, size_t index, void* pData);
+
 /**
  * remove data entry of the given index
  * @param pArray
@@ -122,7 +143,7 @@ void taosArrayClear(SArray* pArray);
  * destroy array list
  * @param pArray
  */
-void taosArrayDestroy(SArray* pArray);
+void* taosArrayDestroy(SArray* pArray);
 
 /**
  *
@@ -150,14 +171,14 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn);
  * @param compar
  * @param key
  */
-void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn);
+void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags);
 
 /**
  * search the array
  * @param pArray
  * @param key
  */
-char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn);
+char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags);
 
 #ifdef __cplusplus
 }

src/util/inc/tconfig.h

@@ -48,6 +48,7 @@ enum {
   TAOS_CFG_VTYPE_STRING,
   TAOS_CFG_VTYPE_IPSTR,
   TAOS_CFG_VTYPE_DIRECTORY,
+  TAOS_CFG_VTYPE_DATA_DIRCTORY,
 };
 
 enum {

src/util/inc/tlist.h

@@ -47,7 +47,7 @@ typedef struct {
 #define listNodeFree(n) free(n);
 
 SList *    tdListNew(int eleSize);
-void       tdListFree(SList *list);
+void *     tdListFree(SList *list);
 void       tdListEmpty(SList *list);
 void       tdListPrependNode(SList *list, SListNode *node);
 void       tdListAppendNode(SList *list, SListNode *node);

src/util/src/tarray.c

@@ -55,24 +55,29 @@ static int32_t taosArrayResize(SArray* pArray) {
   return 0;
 }
 
-void* taosArrayPush(SArray* pArray, void* pData) {
+void* taosArrayPushBatch(SArray* pArray, const void* pData, int nEles) {
   if (pArray == NULL || pData == NULL) {
     return NULL;
   }
 
-  if (pArray->size >= pArray->capacity) {
-    int32_t ret = taosArrayResize(pArray);
-    
-    // failed to push data into buffer due to the failure of memory allocation
-    if (ret != 0) {
+  if (pArray->size + nEles > pArray->capacity) {
+    size_t tsize = (pArray->capacity << 1u);
+    while (pArray->size + nEles > tsize) {
+      tsize = (tsize << 1u);
+    }
+
+    pArray->pData = realloc(pArray->pData, tsize * pArray->elemSize);
+    if (pArray->pData == NULL) {
       return NULL;
     }
+
+    pArray->capacity = tsize;
   }
 
   void* dst = TARRAY_GET_ELEM(pArray, pArray->size);
-  memcpy(dst, pData, pArray->elemSize);
+  memcpy(dst, pData, pArray->elemSize * nEles);
 
-  pArray->size += 1;
+  pArray->size += nEles;
   return dst;
 }
 
@@ -133,6 +138,11 @@ void* taosArrayInsert(SArray* pArray, size_t index, void* pData) {
   return dst;
 }
 
+void taosArraySet(SArray* pArray, size_t index, void* pData) {
+  assert(index < pArray->size);
+  memcpy(TARRAY_GET_ELEM(pArray, index), pData, pArray->elemSize);
+}
+
 void taosArrayRemove(SArray* pArray, size_t index) {
   assert(index < pArray->size);
   
@@ -184,13 +194,13 @@ void taosArrayClear(SArray* pArray) {
   pArray->size = 0;
 }
 
-void taosArrayDestroy(SArray* pArray) {
-  if (pArray == NULL) {
-    return;
+void* taosArrayDestroy(SArray* pArray) {
+  if (pArray) {
+    free(pArray->pData);
+    free(pArray);
   }
 
-  free(pArray->pData);
-  free(pArray);
+  return NULL;
 }
 
 void taosArrayDestroyEx(SArray* pArray, void (*fp)(void*)) {
@@ -217,11 +227,11 @@ void taosArraySort(SArray* pArray, int (*compar)(const void*, const void*)) {
   qsort(pArray->pData, pArray->size, pArray->elemSize, compar);
 }
 
-void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn) {
+void* taosArraySearch(const SArray* pArray, const void* key, __compar_fn_t comparFn, int flags) {
   assert(pArray != NULL && comparFn != NULL);
   assert(key != NULL);
 
-  return bsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn);
+  return taosbsearch(key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags);
 }
 
 void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) {
@@ -229,11 +239,11 @@ void taosArraySortString(SArray* pArray, __compar_fn_t comparFn) {
   qsort(pArray->pData, pArray->size, pArray->elemSize, comparFn);
 }
 
-char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn) {
+char* taosArraySearchString(const SArray* pArray, const char* key, __compar_fn_t comparFn, int flags) {
   assert(pArray != NULL);
   assert(key != NULL);
 
-  void* p = bsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn);
+  void* p = taosbsearch(&key, pArray->pData, pArray->size, pArray->elemSize, comparFn, flags);
   if (p == NULL) {
     return NULL;
   }

src/util/src/tcompare.c

@@ -277,7 +277,7 @@ int32_t taosArrayCompareString(const void* a, const void* b) {
 
 static int32_t compareFindStrInArray(const void* pLeft, const void* pRight) {
   const SArray* arr = (const SArray*) pRight;
-  return taosArraySearchString(arr, pLeft, taosArrayCompareString) == NULL ? 0 : 1;
+  return taosArraySearchString(arr, pLeft, taosArrayCompareString, TD_EQ) == NULL ? 0 : 1;
 }
 
 static int32_t compareWStrPatternComp(const void* pLeft, const void* pRight) {

src/util/src/tconfig.c

@@ -20,7 +20,7 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
-#include "tsystem.h"
+#include "tsocket.h"
 #include "tutil.h"
 
 SGlobalCfg tsGlobalConfig[TSDB_CFG_MAX_NUM] = {{0}};
@@ -112,32 +112,34 @@ static void taosReadInt8Config(SGlobalCfg *cfg, char *input_value) {
   }
 }
 
-static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
+static bool taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
   int   length = (int)strlen(input_value);
   char *option = (char *)cfg->ptr;
   if (length <= 0 || length > cfg->ptrLength) {
-    uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s",
-           cfg->option, input_value, cfg->ptrLength, option);
+    uError("config option:%s, input value:%s, length out of range[0, %d], use default value:%s", cfg->option,
+           input_value, cfg->ptrLength, option);
+    return false;
   } else {
     if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_FILE) {
       wordexp_t full_path;
       if (0 != wordexp(input_value, &full_path, 0)) {
         printf("\nconfig dir: %s wordexp fail! reason:%s\n", input_value, strerror(errno));
         wordfree(&full_path);
-        return;
+        return false;
       }
-      
+
       if (full_path.we_wordv != NULL && full_path.we_wordv[0] != NULL) {
         strcpy(option, full_path.we_wordv[0]);
       }
-      
+
       wordfree(&full_path);
 
       int code = taosMkDir(option, 0755);
       if (code != 0) {
         terrno = TAOS_SYSTEM_ERROR(errno);
-        uError("config option:%s, input value:%s, directory not exist, create fail:%s",
-             cfg->option, input_value, strerror(errno)); 
+        uError("config option:%s, input value:%s, directory not exist, create fail:%s", cfg->option, input_value,
+               strerror(errno));
+        return false;
       }
       cfg->cfgStatus = TAOS_CFG_CSTATUS_FILE;
     } else {
@@ -145,6 +147,8 @@ static void taosReadDirectoryConfig(SGlobalCfg *cfg, char *input_value) {
             tsCfgStatusStr[cfg->cfgStatus], option);
     }
   }
+
+  return true;
 }
 
 static void taosReadIpStrConfig(SGlobalCfg *cfg, char *input_value) {
@@ -214,7 +218,7 @@ SGlobalCfg *taosGetConfigOption(const char *option) {
   return NULL;
 }
 
-static void taosReadConfigOption(const char *option, char *value) {
+static void taosReadConfigOption(const char *option, char *value, char *value2, char *value3) {
   for (int i = 0; i < tsGlobalConfigNum; ++i) {
     SGlobalCfg *cfg = tsGlobalConfig + i;
     if (!(cfg->cfgType & TSDB_CFG_CTYPE_B_CONFIG)) continue;
@@ -242,6 +246,11 @@ static void taosReadConfigOption(const char *option, char *value) {
       case TAOS_CFG_VTYPE_DIRECTORY:
         taosReadDirectoryConfig(cfg, value);
         break;
+      case TAOS_CFG_VTYPE_DATA_DIRCTORY:
+        if (taosReadDirectoryConfig(cfg, value)) {
+          taosReadDataDirCfg(value, value2, value3);
+        }
+        break;
       default:
         uError("config option:%s, input value:%s, can't be recognized", option, value);
         break;
@@ -322,8 +331,8 @@ void taosReadGlobalLogCfg() {
 }
 
 bool taosReadGlobalCfg() {
-  char * line, *option, *value, *value1;
-  int    olen, vlen, vlen1;
+  char * line, *option, *value, *value2, *value3;
+  int    olen, vlen, vlen2, vlen3;
   char   fileName[PATH_MAX] = {0};
 
   sprintf(fileName, "%s/taos.cfg", configDir);
@@ -346,8 +355,8 @@ bool taosReadGlobalCfg() {
   while (!feof(fp)) {
     memset(line, 0, len);
 
-    option = value = NULL;
-    olen = vlen = 0;
+    option = value = value2 = value3 = NULL;
+    olen = vlen = vlen2 = vlen3 = 0;
 
     tgetline(&line, &len, fp);
     line[len - 1] = 0;
@@ -360,11 +369,14 @@ bool taosReadGlobalCfg() {
     if (vlen == 0) continue;
     value[vlen] = 0;
 
-    // For dataDir, the format is:
-    // dataDir    /mnt/disk1    0
-    paGetToken(value + vlen + 1, &value1, &vlen1);
-    
-    taosReadConfigOption(option, value);
+    paGetToken(value + vlen + 1, &value2, &vlen2);
+    if (vlen2 != 0) {
+      value2[vlen2] = 0;
+      paGetToken(value2 + vlen2 + 1, &value3, &vlen3);
+      if (vlen3 != 0) value3[vlen3] = 0;
+    }
+
+    taosReadConfigOption(option, value, value2, value3);
   }
 
   fclose(fp);
@@ -419,6 +431,8 @@ void taosPrintGlobalCfg() {
   }
 
   taosPrintOsInfo();
+  taosPrintDataDirCfg();
+  uInfo("==================================");
 }
 
 static void taosDumpCfg(SGlobalCfg *cfg) {

src/util/src/tkvstore.c

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#define _DEFAULT_SOURCE
-#define TAOS_RANDOM_FILE_FAIL_TEST
-#include "os.h"
-#include "hash.h"
-#include "taoserror.h"
-#include "tchecksum.h"
-#include "tcoding.h"
-#include "tkvstore.h"
-#include "tulog.h"
-
-#define TD_KVSTORE_HEADER_SIZE 512
-#define TD_KVSTORE_MAJOR_VERSION 1
-#define TD_KVSTORE_MAINOR_VERSION 0
-#define TD_KVSTORE_SNAP_SUFFIX ".snap"
-#define TD_KVSTORE_NEW_SUFFIX ".new"
-#define TD_KVSTORE_INIT_MAGIC 0xFFFFFFFF
-
-typedef struct {
-  uint64_t uid;
-  int64_t  offset;
-  int64_t  size;
-} SKVRecord;
-
-static int       tdInitKVStoreHeader(int fd, char *fname);
-static int       tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo);
-static void *    tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo);
-static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH);
-static char *    tdGetKVStoreSnapshotFname(char *fdata);
-static char *    tdGetKVStoreNewFname(char *fdata);
-static void      tdFreeKVStore(SKVStore *pStore);
-static int       tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo);
-static int       tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t *version);
-static int       tdEncodeKVRecord(void **buf, SKVRecord *pRecord);
-static void *    tdDecodeKVRecord(void *buf, SKVRecord *pRecord);
-static int       tdRestoreKVStore(SKVStore *pStore);
-
-int tdCreateKVStore(char *fname) {
-  int fd = open(fname, O_RDWR | O_CREAT | O_BINARY, 0755);
-  if (fd < 0) {
-    uError("failed to open file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (tdInitKVStoreHeader(fd, fname) < 0) goto _err;
-
-  if (fsync(fd) < 0) {
-    uError("failed to fsync file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (close(fd) < 0) {
-    uError("failed to close file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  return 0;
-
-_err:
-  if (fd >= 0) close(fd);
-  (void)remove(fname);
-  return -1;
-}
-
-int tdDestroyKVStore(char *fname) {
-  if (remove(fname) < 0) {
-    uError("failed to remove file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return 0;
-}
-
-SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) {
-  SStoreInfo info = {0};
-  uint32_t   version = 0;
-
-  SKVStore *pStore = tdNewKVStore(fname, iFunc, aFunc, appH);
-  if (pStore == NULL) return NULL;
-
-  pStore->fd = open(pStore->fname, O_RDWR | O_BINARY);
-  if (pStore->fd < 0) {
-    uError("failed to open file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  pStore->sfd = open(pStore->fsnap, O_RDONLY | O_BINARY);
-  if (pStore->sfd < 0) {
-    if (errno != ENOENT) {
-      uError("failed to open file %s since %s", pStore->fsnap, strerror(errno));
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-  } else {
-    uDebug("file %s exists, try to recover the KV store", pStore->fsnap);
-    if (tdLoadKVStoreHeader(pStore->sfd, pStore->fsnap, &info, &version) < 0) {
-      if (terrno != TSDB_CODE_COM_FILE_CORRUPTED) goto _err;
-    } else {
-      if (version != KVSTORE_FILE_VERSION) {
-        uError("file %s version %u is not the same as program version %u, this may cause problem", pStore->fsnap,
-               version, KVSTORE_FILE_VERSION);
-      }
-
-      if (taosFtruncate(pStore->fd, info.size) < 0) {
-        uError("failed to truncate %s to %" PRId64 " size since %s", pStore->fname, info.size, strerror(errno));
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        goto _err;
-      }
-
-      if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &info) < 0) goto _err;
-      if (fsync(pStore->fd) < 0) {
-        uError("failed to fsync file %s since %s", pStore->fname, strerror(errno));
-        goto _err;
-      }
-    }
-
-    close(pStore->sfd);
-    pStore->sfd = -1;
-    (void)remove(pStore->fsnap);
-  }
-
-  if (tdLoadKVStoreHeader(pStore->fd, pStore->fname, &info, &version) < 0) goto _err;
-  if (version != KVSTORE_FILE_VERSION) {
-    uError("file %s version %u is not the same as program version %u, this may cause problem", pStore->fname, version,
-           KVSTORE_FILE_VERSION);
-  }
-
-  pStore->info.size = TD_KVSTORE_HEADER_SIZE;
-  pStore->info.magic = info.magic;
-
-  if (tdRestoreKVStore(pStore) < 0) goto _err;
-
-  close(pStore->fd);
-  pStore->fd = -1;
-
-  return pStore;
-
-_err:
-  if (pStore->fd > 0) {
-    close(pStore->fd);
-    pStore->fd = -1;
-  }
-  if (pStore->sfd > 0) {
-    close(pStore->sfd);
-    pStore->sfd = -1;
-  }
-  tdFreeKVStore(pStore);
-  return NULL;
-}
-
-void tdCloseKVStore(SKVStore *pStore) { tdFreeKVStore(pStore); }
-
-int tdKVStoreStartCommit(SKVStore *pStore) {
-  ASSERT(pStore->fd < 0);
-
-  pStore->fd = open(pStore->fname, O_RDWR | O_BINARY);
-  if (pStore->fd < 0) {
-    uError("failed to open file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  pStore->sfd = open(pStore->fsnap, O_WRONLY | O_CREAT | O_BINARY, 0755);
-  if (pStore->sfd < 0) {
-    uError("failed to open file %s since %s", pStore->fsnap, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (taosSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
-    uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (fsync(pStore->sfd) < 0) {
-    uError("failed to fsync file %s since %s", pStore->fsnap, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  if (close(pStore->sfd) < 0) {
-    uError("failed to close file %s since %s", pStore->fsnap, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-  pStore->sfd = -1;
-
-  if (lseek(pStore->fd, 0, SEEK_END) < 0) {
-    uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  ASSERT(pStore->info.size == lseek(pStore->fd, 0, SEEK_CUR));
-
-  return 0;
-
-_err:
-  if (pStore->sfd > 0) {
-    close(pStore->sfd);
-    pStore->sfd = -1;
-    (void)remove(pStore->fsnap);
-  }
-  if (pStore->fd > 0) {
-    close(pStore->fd);
-    pStore->fd = -1;
-  }
-  return -1;
-}
-
-int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen) {
-  SKVRecord rInfo = {0};
-  char      buf[64] = "\0";
-  char *    pBuf = buf;
-
-  rInfo.offset = lseek(pStore->fd, 0, SEEK_CUR);
-  if (rInfo.offset < 0) {
-    uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  rInfo.uid = uid;
-  rInfo.size = contLen;
-
-  int tlen = tdEncodeKVRecord((void *)(&pBuf), &rInfo);
-  ASSERT(tlen == POINTER_DISTANCE(pBuf, buf));
-  ASSERT(tlen == sizeof(SKVRecord));
-
-  if (taosWrite(pStore->fd, buf, tlen) < tlen) {
-    uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosWrite(pStore->fd, cont, contLen) < contLen) {
-    uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  pStore->info.magic =
-      taosCalcChecksum(pStore->info.magic, (uint8_t *)POINTER_SHIFT(cont, contLen - sizeof(TSCKSUM)), sizeof(TSCKSUM));
-  pStore->info.size += (sizeof(SKVRecord) + contLen);
-  SKVRecord *pRecord = taosHashGet(pStore->map, (void *)&uid, sizeof(uid));
-  if (pRecord != NULL) {  // just to insert
-    pStore->info.tombSize += pRecord->size;
-  } else {
-    pStore->info.nRecords++;
-  }
-
-  taosHashPut(pStore->map, (void *)(&uid), sizeof(uid), (void *)(&rInfo), sizeof(rInfo));
-  uTrace("put uid %" PRIu64 " into kvStore %s", uid, pStore->fname);
-
-  return 0;
-}
-
-int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
-  SKVRecord rInfo = {0};
-  char      buf[128] = "\0";
-
-  SKVRecord *pRecord = taosHashGet(pStore->map, (void *)(&uid), sizeof(uid));
-  if (pRecord == NULL) {
-    uError("failed to drop KV store record with key %" PRIu64 " since not find", uid);
-    return -1;
-  }
-
-  rInfo.offset = -pRecord->offset;
-  rInfo.uid = pRecord->uid;
-  rInfo.size = pRecord->size;
-
-  void *pBuf = buf;
-  tdEncodeKVRecord(&pBuf, &rInfo);
-
-  if (taosWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
-    uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  pStore->info.magic = taosCalcChecksum(pStore->info.magic, (uint8_t *)buf, (uint32_t)POINTER_DISTANCE(pBuf, buf));
-  pStore->info.size += POINTER_DISTANCE(pBuf, buf);
-  pStore->info.nDels++;
-  pStore->info.nRecords--;
-  pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
-
-  taosHashRemove(pStore->map, (void *)(&uid), sizeof(uid));
-  uDebug("drop uid %" PRIu64 " from KV store %s", uid, pStore->fname);
-
-  return 0;
-}
-
-int tdKVStoreEndCommit(SKVStore *pStore) {
-  ASSERT(pStore->fd > 0);
-
-  if (tdUpdateKVStoreHeader(pStore->fd, pStore->fname, &(pStore->info)) < 0) return -1;
-
-  if (fsync(pStore->fd) < 0) {
-    uError("failed to fsync file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (close(pStore->fd) < 0) {
-    uError("failed to close file %s since %s", pStore->fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-  pStore->fd = -1;
-
-  (void)remove(pStore->fsnap);
-  return 0;
-}
-
-void tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size) {
-  char       buf[TD_KVSTORE_HEADER_SIZE] = "\0";
-  SStoreInfo info = {0};
-
-  int fd = open(fname, O_RDONLY | O_BINARY);
-  if (fd < 0) goto _err;
-
-  if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) goto _err;
-  if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) goto _err;
-
-  void *pBuf = (void *)buf;
-  pBuf = tdDecodeStoreInfo(pBuf, &info);
-  off_t offset = lseek(fd, 0, SEEK_END);
-  if (offset < 0) goto _err;
-  close(fd);
-
-  *magic = info.magic;
-  *size = offset;
-
-  return;
-
-_err:
-  if (fd >= 0) close(fd);
-  *magic = TD_KVSTORE_INIT_MAGIC;
-  *size = 0;
-}
-
-static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo, uint32_t *version) {
-  char buf[TD_KVSTORE_HEADER_SIZE] = "\0";
-
-  if (lseek(fd, 0, SEEK_SET) < 0) {
-    uError("failed to lseek file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (taosRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
-    uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  if (!taosCheckChecksumWhole((uint8_t *)buf, TD_KVSTORE_HEADER_SIZE)) {
-    uError("file %s is broken", fname);
-    terrno = TSDB_CODE_COM_FILE_CORRUPTED;
-    return -1;
-  }
-
-  void *pBuf = (void *)buf;
-  pBuf = tdDecodeStoreInfo(pBuf, pInfo);
-  pBuf = taosDecodeFixedU32(pBuf, version);
-
-  return 0;
-}
-
-static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
-  char buf[TD_KVSTORE_HEADER_SIZE] = "\0";
-
-  if (lseek(fd, 0, SEEK_SET) < 0) {
-    uError("failed to lseek file %s since %s", fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  void *pBuf = buf;
-  tdEncodeStoreInfo(&pBuf, pInfo);
-  taosEncodeFixedU32(&pBuf, KVSTORE_FILE_VERSION);
-  ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE);
-
-  taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE);
-  if (taosWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
-    uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    return -1;
-  }
-
-  return 0;
-}
-
-static int tdInitKVStoreHeader(int fd, char *fname) {
-  SStoreInfo info = {TD_KVSTORE_HEADER_SIZE, 0, 0, 0, TD_KVSTORE_INIT_MAGIC};
-
-  return tdUpdateKVStoreHeader(fd, fname, &info);
-}
-
-static int tdEncodeStoreInfo(void **buf, SStoreInfo *pInfo) {
-  int tlen = 0;
-  tlen += taosEncodeVariantI64(buf, pInfo->size);
-  tlen += taosEncodeVariantI64(buf, pInfo->tombSize);
-  tlen += taosEncodeVariantI64(buf, pInfo->nRecords);
-  tlen += taosEncodeVariantI64(buf, pInfo->nDels);
-  tlen += taosEncodeFixedU32(buf, pInfo->magic);
-
-  return tlen;
-}
-
-static void *tdDecodeStoreInfo(void *buf, SStoreInfo *pInfo) {
-  buf = taosDecodeVariantI64(buf, &(pInfo->size));
-  buf = taosDecodeVariantI64(buf, &(pInfo->tombSize));
-  buf = taosDecodeVariantI64(buf, &(pInfo->nRecords));
-  buf = taosDecodeVariantI64(buf, &(pInfo->nDels));
-  buf = taosDecodeFixedU32(buf, &(pInfo->magic));
-
-  return buf;
-}
-
-static SKVStore *tdNewKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH) {
-  SKVStore *pStore = (SKVStore *)calloc(1, sizeof(SKVStore));
-  if (pStore == NULL) goto _err;
-
-  pStore->fname = strdup(fname);
-  if (pStore->fname == NULL) {
-    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  pStore->fsnap = tdGetKVStoreSnapshotFname(fname);
-  if (pStore->fsnap == NULL) {
-    goto _err;
-  }
-
-  pStore->fnew = tdGetKVStoreNewFname(fname);
-  if (pStore->fnew == NULL) goto _err;
-
-  pStore->fd = -1;
-  pStore->sfd = -1;
-  pStore->nfd = -1;
-  pStore->iFunc = iFunc;
-  pStore->aFunc = aFunc;
-  pStore->appH = appH;
-  pStore->map = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, false);
-  if (pStore->map == NULL) {
-    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-    goto _err;
-  }
-
-  return pStore;
-
-_err:
-  tdFreeKVStore(pStore);
-  return NULL;
-}
-
-static void tdFreeKVStore(SKVStore *pStore) {
-  if (pStore) {
-    tfree(pStore->fname);
-    tfree(pStore->fsnap);
-    tfree(pStore->fnew);
-    taosHashCleanup(pStore->map);
-    free(pStore);
-  }
-}
-
-static char *tdGetKVStoreSnapshotFname(char *fdata) {
-  size_t size = strlen(fdata) + strlen(TD_KVSTORE_SNAP_SUFFIX) + 1;
-  char * fname = malloc(size);
-  if (fname == NULL) {
-    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-    return NULL;
-  }
-  sprintf(fname, "%s%s", fdata, TD_KVSTORE_SNAP_SUFFIX);
-  return fname;
-}
-
-static char *tdGetKVStoreNewFname(char *fdata) {
-  size_t size = strlen(fdata) + strlen(TD_KVSTORE_NEW_SUFFIX) + 1;
-  char * fname = malloc(size);
-  if (fname == NULL) {
-    terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-    return NULL;
-  }
-  sprintf(fname, "%s%s", fdata, TD_KVSTORE_NEW_SUFFIX);
-  return fname;
-}
-
-static int tdEncodeKVRecord(void **buf, SKVRecord *pRecord) {
-  int tlen = 0;
-  tlen += taosEncodeFixedU64(buf, pRecord->uid);
-  tlen += taosEncodeFixedI64(buf, pRecord->offset);
-  tlen += taosEncodeFixedI64(buf, pRecord->size);
-
-  return tlen;
-}
-
-static void *tdDecodeKVRecord(void *buf, SKVRecord *pRecord) {
-  buf = taosDecodeFixedU64(buf, &(pRecord->uid));
-  buf = taosDecodeFixedI64(buf, &(pRecord->offset));
-  buf = taosDecodeFixedI64(buf, &(pRecord->size));
-
-  return buf;
-}
-
-static int tdRestoreKVStore(SKVStore *pStore) {
-  char                  tbuf[128] = "\0";
-  void *                buf = NULL;
-  int64_t               maxBufSize = 0;
-  SKVRecord             rInfo = {0};
-  SKVRecord            *pRecord = NULL;
-
-  ASSERT(TD_KVSTORE_HEADER_SIZE == lseek(pStore->fd, 0, SEEK_CUR));
-  ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE);
-
-  while (true) {
-    int64_t tsize = taosRead(pStore->fd, tbuf, sizeof(SKVRecord));
-    if (tsize == 0) break;
-    if (tsize < sizeof(SKVRecord)) {
-      uError("failed to read %" PRIzu " bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname,
-             pStore->info.size, strerror(errno));
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    char *pBuf = tdDecodeKVRecord(tbuf, &rInfo);
-    ASSERT(POINTER_DISTANCE(pBuf, tbuf) == sizeof(SKVRecord));
-    ASSERT((rInfo.offset > 0) ? (pStore->info.size == rInfo.offset) : true);
-
-    if (rInfo.offset < 0) {
-      taosHashRemove(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid));
-      pStore->info.size += sizeof(SKVRecord);
-      pStore->info.nRecords--;
-      pStore->info.nDels++;
-      pStore->info.tombSize += (rInfo.size + sizeof(SKVRecord) * 2);
-    } else {
-      ASSERT(rInfo.offset > 0 && rInfo.size > 0);
-      if (taosHashPut(pStore->map, (void *)(&rInfo.uid), sizeof(rInfo.uid), &rInfo, sizeof(rInfo)) < 0) {
-        uError("failed to put record in KV store %s", pStore->fname);
-        terrno = TSDB_CODE_COM_OUT_OF_MEMORY;
-        goto _err;
-      }
-
-      maxBufSize = MAX(maxBufSize, rInfo.size);
-
-      if (lseek(pStore->fd, (off_t)rInfo.size, SEEK_CUR) < 0) {
-        uError("failed to lseek file %s since %s", pStore->fname, strerror(errno));
-        terrno = TAOS_SYSTEM_ERROR(errno);
-        goto _err;
-      }
-
-      pStore->info.size += (sizeof(SKVRecord) + rInfo.size);
-      pStore->info.nRecords++;
-    }
-  }
-
-  buf = malloc((size_t)maxBufSize);
-  if (buf == NULL) {
-    uError("failed to allocate %" PRId64 " bytes in KV store %s", maxBufSize, pStore->fname);
-    terrno = TAOS_SYSTEM_ERROR(errno);
-    goto _err;
-  }
-
-  pRecord = taosHashIterate(pStore->map, NULL);
-  while (pRecord) {
-    if (lseek(pStore->fd, (off_t)(pRecord->offset + sizeof(SKVRecord)), SEEK_SET) < 0) {
-      uError("failed to lseek file %s since %s, offset %" PRId64, pStore->fname, strerror(errno), pRecord->offset);
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    if (taosRead(pStore->fd, buf, (size_t)pRecord->size) < pRecord->size) {
-      uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname,
-             strerror(errno), pRecord->offset);
-      terrno = TAOS_SYSTEM_ERROR(errno);
-      goto _err;
-    }
-
-    if (pStore->iFunc) {
-      if ((*pStore->iFunc)(pStore->appH, buf, (int)pRecord->size) < 0) {
-        uError("failed to restore record uid %" PRIu64 " in kv store %s at offset %" PRId64 " size %" PRId64
-               " since %s",
-               pRecord->uid, pStore->fname, pRecord->offset, pRecord->size, tstrerror(terrno));
-        goto _err;
-      }
-    }
-
-    pRecord = taosHashIterate(pStore->map, pRecord);
-  }
-
-  if (pStore->aFunc) (*pStore->aFunc)(pStore->appH);
-
-  tfree(buf);
-  return 0;
-
-_err:
-  taosHashCancelIterate(pStore->map, pRecord);
-  tfree(buf);
-  return -1;
-}

src/util/src/tlist.c

@@ -38,11 +38,13 @@ void tdListEmpty(SList *list) {
   list->numOfEles = 0;
 }
 
-void tdListFree(SList *list) {
+void *tdListFree(SList *list) {
   if (list) {
     tdListEmpty(list);
     free(list);
   }
+
+  return NULL;
 }
 
 void tdListPrependNode(SList *list, SListNode *node) {

src/util/src/tsocket.c

@@ -483,5 +483,5 @@ int32_t taosCopyFds(SOCKET sfd, int32_t dfd, int64_t len) {
     leftLen -= readLen;
   }
 
-  return 0;
+  return (int32_t)len;
 }

src/util/src/ttimer.c

@@ -188,7 +188,11 @@ static void removeTimer(uintptr_t id) {
 }
 
 static int64_t getMonotonicMs(void) {
+#ifdef WINDOWS
   return (int64_t) getMonotonicUs() / 1000;
+#else
+  return taosGetTimestampMs();
+#endif
 }
 
 static void addToWheel(tmr_obj_t* timer, uint32_t delay) {
@@ -537,7 +541,8 @@ static void taosTmrModuleInit(void) {
 }
 
 void* taosTmrInit(int maxNumOfTmrs, int resolution, int longest, const char* label) {
-  tmrInfo("ttimer monotonic clock source:%s", monotonicInit());
+  const char* ret = monotonicInit();
+  tmrInfo("ttimer monotonic clock source:%s", ret);
 
   pthread_once(&tmrModuleInit, taosTmrModuleInit);
 

src/vnode/CMakeLists.txt

@@ -11,4 +11,4 @@ INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 
 ADD_LIBRARY(vnode ${SRC})
-TARGET_LINK_LIBRARIES(vnode tsdb tcq)
+TARGET_LINK_LIBRARIES(vnode tsdb tcq common)

src/vnode/inc/vnodeMain.h

@@ -26,8 +26,6 @@ int32_t vnodeDrop(int32_t vgId);
 int32_t vnodeOpen(int32_t vgId);
 int32_t vnodeAlter(void *pVnode, SCreateVnodeMsg *pVnodeCfg);
 int32_t vnodeClose(int32_t vgId);
-
-int32_t vnodeReset(SVnodeObj *pVnode);
 void    vnodeCleanUp(SVnodeObj *pVnode);
 void    vnodeDestroy(SVnodeObj *pVnode);
 

src/vnode/inc/vnodeSync.h

@@ -25,7 +25,8 @@ uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t ei
 int32_t  vnodeGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId);
 void     vnodeNotifyRole(int32_t vgId, int8_t role);
 void     vnodeCtrlFlow(int32_t vgId, int32_t level);
-int32_t  vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion);
+void     vnodeStartSyncFile(int32_t vgId);
+void     vnodeStopSyncFile(int32_t vgId, uint64_t fversion);
 void     vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code);
 int32_t  vnodeWriteToCache(int32_t vgId, void *wparam, int32_t qtype, void *rparam);
 int32_t  vnodeGetVersion(int32_t vgId, uint64_t *fver, uint64_t *wver);

src/vnode/src/vnodeMain.c

@@ -18,7 +18,7 @@
 #include "taoserror.h"
 #include "taosmsg.h"
 #include "tglobal.h"
-// #include "tfs.h"
+#include "tfs.h"
 #include "query.h"
 #include "dnode.h"
 #include "vnodeCfg.h"
@@ -41,32 +41,19 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
     return TSDB_CODE_SUCCESS;
   }
 
-  if (mkdir(tsVnodeDir, 0755) != 0 && errno != EEXIST) {
-    vError("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), tsVnodeDir);
-    if (errno == EACCES) {
-      return TSDB_CODE_VND_NO_DISK_PERMISSIONS;
-    } else if (errno == ENOSPC) {
-      return TSDB_CODE_VND_NO_DISKSPACE;
-    } else if (errno == ENOENT) {
-      return TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR;
-    } else {
-      return TSDB_CODE_VND_INIT_FAILED;
-    }
+  if (tfsMkdir("vnode") < 0) {
+    vError("vgId:%d, failed to create vnode dir, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno));
+    return terrno;
   }
 
   char rootDir[TSDB_FILENAME_LEN] = {0};
   sprintf(rootDir, "%s/vnode%d", tsVnodeDir, pVnodeCfg->cfg.vgId);
-  if (mkdir(rootDir, 0755) != 0 && errno != EEXIST) {
-    vError("vgId:%d, failed to create vnode, reason:%s dir:%s", pVnodeCfg->cfg.vgId, strerror(errno), rootDir);
-    if (errno == EACCES) {
-      return TSDB_CODE_VND_NO_DISK_PERMISSIONS;
-    } else if (errno == ENOSPC) {
-      return TSDB_CODE_VND_NO_DISKSPACE;
-    } else if (errno == ENOENT) {
-      return TSDB_CODE_VND_NO_SUCH_FILE_OR_DIR;
-    } else {
-      return TSDB_CODE_VND_INIT_FAILED;
-    }
+
+  char vnodeDir[TSDB_FILENAME_LEN] = "\0";
+  snprintf(vnodeDir, TSDB_FILENAME_LEN, "/vnode/vnode%d", pVnodeCfg->cfg.vgId);
+  if (tfsMkdir(vnodeDir) < 0) {
+    vError("vgId:%d, failed to create vnode dir %s, reason:%s", pVnodeCfg->cfg.vgId, vnodeDir, strerror(errno));
+    return terrno;
   }
 
   code = vnodeWriteCfg(pVnodeCfg);
@@ -75,22 +62,24 @@ int32_t vnodeCreate(SCreateVnodeMsg *pVnodeCfg) {
     return code;
   }
 
-  STsdbCfg tsdbCfg = {0};
-  tsdbCfg.tsdbId              = pVnodeCfg->cfg.vgId;
-  tsdbCfg.cacheBlockSize      = pVnodeCfg->cfg.cacheBlockSize;
-  tsdbCfg.totalBlocks         = pVnodeCfg->cfg.totalBlocks;
-  tsdbCfg.daysPerFile         = pVnodeCfg->cfg.daysPerFile;
-  tsdbCfg.keep                = pVnodeCfg->cfg.daysToKeep;
-  tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
-  tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
-  tsdbCfg.precision           = pVnodeCfg->cfg.precision;
-  tsdbCfg.compression         = pVnodeCfg->cfg.compression;
-  tsdbCfg.update              = pVnodeCfg->cfg.update;
-  tsdbCfg.cacheLastRow        = pVnodeCfg->cfg.cacheLastRow;
-
-  char tsdbDir[TSDB_FILENAME_LEN] = {0};
-  sprintf(tsdbDir, "%s/vnode%d/tsdb", tsVnodeDir, pVnodeCfg->cfg.vgId);
-  if (tsdbCreateRepo(tsdbDir, &tsdbCfg) < 0) {
+  // STsdbCfg tsdbCfg = {0};
+  // tsdbCfg.tsdbId              = pVnodeCfg->cfg.vgId;
+  // tsdbCfg.cacheBlockSize      = pVnodeCfg->cfg.cacheBlockSize;
+  // tsdbCfg.totalBlocks         = pVnodeCfg->cfg.totalBlocks;
+  // tsdbCfg.daysPerFile         = pVnodeCfg->cfg.daysPerFile;
+  // tsdbCfg.keep                = pVnodeCfg->cfg.daysToKeep;
+  // tsdbCfg.keep1               = pVnodeCfg->cfg.daysToKeep1;
+  // tsdbCfg.keep2               = pVnodeCfg->cfg.daysToKeep2;
+  // tsdbCfg.minRowsPerFileBlock = pVnodeCfg->cfg.minRowsPerFileBlock;
+  // tsdbCfg.maxRowsPerFileBlock = pVnodeCfg->cfg.maxRowsPerFileBlock;
+  // tsdbCfg.precision           = pVnodeCfg->cfg.precision;
+  // tsdbCfg.compression         = pVnodeCfg->cfg.compression;
+  // tsdbCfg.update              = pVnodeCfg->cfg.update;
+  // tsdbCfg.cacheLastRow        = pVnodeCfg->cfg.cacheLastRow;
+
+  // char tsdbDir[TSDB_FILENAME_LEN] = {0};
+  // sprintf(tsdbDir, "vnode/vnode%d/tsdb", pVnodeCfg->cfg.vgId);
+  if (tsdbCreateRepo(pVnodeCfg->cfg.vgId) < 0) {
     vError("vgId:%d, failed to create tsdb in vnode, reason:%s", pVnodeCfg->cfg.vgId, tstrerror(terrno));
     return TSDB_CODE_VND_INIT_FAILED;
   }
@@ -247,14 +236,13 @@ int32_t vnodeOpen(int32_t vgId) {
   appH.cqH = pVnode->cq;
   appH.cqCreateFunc = cqCreate;
   appH.cqDropFunc = cqDrop;
-  sprintf(temp, "%s/tsdb", rootDir);
 
   terrno = 0;
-  pVnode->tsdb = tsdbOpenRepo(temp, &appH);
+  pVnode->tsdb = tsdbOpenRepo(&(pVnode->tsdbCfg), &appH);
   if (pVnode->tsdb == NULL) {
     vnodeCleanUp(pVnode);
     return terrno;
-  } else if (terrno != TSDB_CODE_SUCCESS) {
+  } else if (tsdbGetState(pVnode->tsdb) != TSDB_STATE_OK) {
     vError("vgId:%d, failed to open tsdb, replica:%d reason:%s", pVnode->vgId, pVnode->syncCfg.replica,
            tstrerror(terrno));
     if (pVnode->syncCfg.replica <= 1) {
@@ -301,20 +289,23 @@ int32_t vnodeOpen(int32_t vgId) {
   vDebug("vgId:%d, vnode is opened in %s, pVnode:%p", pVnode->vgId, rootDir, pVnode);
 
   vnodeAddIntoHash(pVnode);
-
+  
   SSyncInfo syncInfo;
   syncInfo.vgId = pVnode->vgId;
   syncInfo.version = pVnode->version;
   syncInfo.syncCfg = pVnode->syncCfg;
   tstrncpy(syncInfo.path, rootDir, TSDB_FILENAME_LEN);
-  syncInfo.getWalInfo = vnodeGetWalInfo;
-  syncInfo.getFileInfo = vnodeGetFileInfo;
-  syncInfo.writeToCache = vnodeWriteToCache;
+  syncInfo.getWalInfoFp = vnodeGetWalInfo;
+  syncInfo.writeToCacheFp = vnodeWriteToCache;
   syncInfo.confirmForward = vnodeConfirmForard; 
-  syncInfo.notifyRole = vnodeNotifyRole;
-  syncInfo.notifyFlowCtrl = vnodeCtrlFlow;
-  syncInfo.notifyFileSynced = vnodeNotifyFileSynced;
-  syncInfo.getVersion = vnodeGetVersion;
+  syncInfo.notifyRoleFp = vnodeNotifyRole;
+  syncInfo.notifyFlowCtrlFp = vnodeCtrlFlow;
+  syncInfo.startSyncFileFp = vnodeStartSyncFile;
+  syncInfo.stopSyncFileFp = vnodeStopSyncFile;
+  syncInfo.getVersionFp = vnodeGetVersion;
+  syncInfo.sendFileFp = tsdbSyncSend;
+  syncInfo.recvFileFp = tsdbSyncRecv;
+  syncInfo.pTsdb = pVnode->tsdb;
   pVnode->sync = syncStart(&syncInfo);
 
   if (pVnode->sync <= 0) {
@@ -344,7 +335,7 @@ int32_t vnodeClose(int32_t vgId) {
 void vnodeDestroy(SVnodeObj *pVnode) {
   int32_t code = 0;
   int32_t vgId = pVnode->vgId;
-
+  
   if (pVnode->qMgmt) {
     qCleanupQueryMgmt(pVnode->qMgmt);
     pVnode->qMgmt = NULL;
@@ -396,17 +387,17 @@ void vnodeDestroy(SVnodeObj *pVnode) {
   if (pVnode->dropped) {
     char rootDir[TSDB_FILENAME_LEN] = {0};    
     char newDir[TSDB_FILENAME_LEN] = {0};
-    sprintf(rootDir, "%s/vnode%d", tsVnodeDir, vgId);
-    sprintf(newDir, "%s/vnode%d", tsVnodeBakDir, vgId);
+    sprintf(rootDir, "%s/vnode%d", "vnode", vgId);
+    sprintf(newDir, "%s/vnode%d", "vnode_bak", vgId);
 
     if (0 == tsEnableVnodeBak) {
       vInfo("vgId:%d, vnode backup not enabled", pVnode->vgId);
     } else {
-      taosRemoveDir(newDir);
-      taosRename(rootDir, newDir);
+      tfsRmdir(newDir);
+      tfsRename(rootDir, newDir);
     }
 
-    taosRemoveDir(rootDir);
+    tfsRmdir(rootDir);
     dnodeSendStatusMsgToMnode();
   }
 
@@ -466,37 +457,3 @@ static int32_t vnodeProcessTsdbStatus(void *arg, int32_t status, int32_t eno) {
 
   return 0;
 }
-
-int32_t vnodeReset(SVnodeObj *pVnode) {
-  char rootDir[128] = "\0";
-  sprintf(rootDir, "%s/tsdb", pVnode->rootDir);
-
-  if (!vnodeSetResetStatus(pVnode)) {
-    return -1;
-  }
-
-  void *tsdb = pVnode->tsdb;
-  pVnode->tsdb = NULL;
-  
-  // acquire vnode
-  int32_t refCount = atomic_add_fetch_32(&pVnode->refCount, 1);
-
-  if (refCount > 3) {
-    tsem_wait(&pVnode->sem);
-  }
-
-  // close tsdb, then open tsdb
-  tsdbCloseRepo(tsdb, 0);
-  STsdbAppH appH = {0};
-  appH.appH = (void *)pVnode;
-  appH.notifyStatus = vnodeProcessTsdbStatus;
-  appH.cqH = pVnode->cq;
-  appH.cqCreateFunc = cqCreate;
-  appH.cqDropFunc = cqDrop;
-  pVnode->tsdb = tsdbOpenRepo(rootDir, &appH);
-
-  vnodeSetReadyStatus(pVnode);
-  vnodeRelease(pVnode);
-
-  return 0;
-}

src/vnode/src/vnodeSync.c

@@ -20,6 +20,7 @@
 #include "dnode.h"
 #include "vnodeVersion.h"
 #include "vnodeMain.h"
+#include "vnodeStatus.h"
 
 uint32_t vnodeGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fver) {
   SVnodeObj *pVnode = vnodeAcquire(vgId);
@@ -83,22 +84,34 @@ void vnodeCtrlFlow(int32_t vgId, int32_t level) {
   vnodeRelease(pVnode);
 }
 
-int32_t vnodeNotifyFileSynced(int32_t vgId, uint64_t fversion) {
+void vnodeStartSyncFile(int32_t vgId) {
   SVnodeObj *pVnode = vnodeAcquire(vgId);
   if (pVnode == NULL) {
-    vError("vgId:%d, vnode not found while notify file synced", vgId);
-    return 0;
+    vError("vgId:%d, vnode not found while start filesync", vgId);
+    return;
+  }
+
+  vDebug("vgId:%d, datafile will be synced", vgId);
+  vnodeSetResetStatus(pVnode);
+
+  vnodeRelease(pVnode);
+}
+
+void vnodeStopSyncFile(int32_t vgId, uint64_t fversion) {
+  SVnodeObj *pVnode = vnodeAcquire(vgId);
+  if (pVnode == NULL) {
+    vError("vgId:%d, vnode not found while stop filesync", vgId);
+    return;
   }
 
   pVnode->fversion = fversion;
   pVnode->version = fversion;
   vnodeSaveVersion(pVnode);
 
-  vDebug("vgId:%d, data file is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
-  int32_t code = vnodeReset(pVnode);
+  vDebug("vgId:%d, datafile is synced, fver:%" PRIu64 " vver:%" PRIu64, vgId, fversion, fversion);
+  vnodeSetReadyStatus(pVnode);
 
   vnodeRelease(pVnode);
-  return code;
 }
 
 void vnodeConfirmForard(int32_t vgId, void *wparam, int32_t code) {

tests/pytest/pytest_1.sh

@@ -15,7 +15,8 @@ python3 ./test.py -f insert/nchar.py
 #python3 ./test.py -f insert/nchar-boundary.py
 python3 ./test.py -f insert/nchar-unicode.py
 python3 ./test.py -f insert/multi.py
-python3 insert/retentionpolicy.py
+python3 ./test.py -f insert/randomNullCommit.py
+#python3 insert/retentionpolicy.py
 python3 ./test.py -f insert/alterTableAndInsert.py
 python3 ./test.py -f insert/insertIntoTwoTables.py
 #python3 ./test.py -f insert/before_1970.py

tests/pytest/pytest_2.sh

 
 
 # update
-python3 ./test.py -f update/allow_update.py
+#python3 ./test.py -f update/allow_update.py
 python3 ./test.py -f update/allow_update-0.py
 python3 ./test.py -f update/append_commit_data.py
 python3 ./test.py -f update/append_commit_last-0.py
 python3 ./test.py -f update/append_commit_last.py
-python3 ./test.py -f update/merge_commit_data.py
-python3 ./test.py -f update/merge_commit_data-0.py
-python3 ./test.py -f update/merge_commit_data2.py
-python3 ./test.py -f update/merge_commit_data2_update0.py
-python3 ./test.py -f update/merge_commit_last-0.py
-python3 ./test.py -f update/merge_commit_last.py
-python3 ./test.py -f update/bug_td2279.py
+#python3 ./test.py -f update/merge_commit_data.py
+#python3 ./test.py -f update/merge_commit_data-0.py
+#python3 ./test.py -f update/merge_commit_data2.py
+#python3 ./test.py -f update/merge_commit_data2_update0.py
+#python3 ./test.py -f update/merge_commit_last-0.py
+#python3 ./test.py -f update/merge_commit_last.py
+#python3 ./test.py -f update/bug_td2279.py
 
 # wal
 python3 ./test.py -f wal/addOldWalTest.py