Merge branch 'develop' into hotfix/TD-2854

CMakeLists.txt

@@ -15,6 +15,7 @@ SET(TD_ADMIN FALSE)
 SET(TD_GRANT FALSE)
 SET(TD_MQTT FALSE)
 SET(TD_TSDB_PLUGINS FALSE)
+SET(TD_STORAGE FALSE)
 
 SET(TD_COVER FALSE)
 SET(TD_MEM_CHECK FALSE)

Jenkinsfile

@@ -45,6 +45,7 @@ def pre_test(){
     git pull
     git fetch origin +refs/pull/${CHANGE_ID}/merge
     git checkout -qf FETCH_HEAD
+    git --no-pager diff --name-only FETCH_HEAD $(git merge-base FETCH_HEAD develop)|grep -v -E '.*md|.*src/connector|Jenkinsfile' || exit 0
     cd ${WK}
     git reset --hard HEAD~10
     git checkout develop
@@ -79,13 +80,14 @@ pipeline {
               changeRequest()
           }
       parallel {
-        stage('python_1') {
+        stage('python_1_s1') {
           agent{label 'p1'}
           steps {
             
             pre_test()
-            timeout(time: 90, unit: 'MINUTES'){
+            timeout(time: 45, unit: 'MINUTES'){
               sh '''
+              date
               cd ${WKC}/tests
               find pytest -name '*'sql|xargs rm -rf
               ./test-all.sh p1
@@ -94,23 +96,35 @@ pipeline {
             
           }
         }
-        stage('python_2') {
+        stage('python_2_s5') {
           agent{label 'p2'}
           steps {
             
             pre_test()
+            timeout(time: 45, unit: 'MINUTES'){
             sh '''
+            date
             cd ${WKC}/tests
             find pytest -name '*'sql|xargs rm -rf
             ./test-all.sh p2
             date'''
-            sh '''
-            cd ${WKC}/tests
-            ./test-all.sh b4fq
-            '''
+            }
+          }
+        }
+        stage('python_3_s6') {
+          agent{label 'p3'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh p3
+              date'''
+            }
           }
         }
-        stage('test_b1') {
+        stage('test_b1_s2') {
           agent{label 'b1'}
           steps {     
             timeout(time: 90, unit: 'MINUTES'){       
@@ -123,7 +137,7 @@ pipeline {
           }
         }
 
-        stage('test_crash_gen') {
+        stage('test_crash_gen_s3') {
           agent{label "b2"}
           steps {
             pre_test()
@@ -139,7 +153,7 @@ pipeline {
                 ./handle_crash_gen_val_log.sh
                 '''
             }
-            timeout(time: 90, unit: 'MINUTES'){
+            timeout(time: 45, unit: 'MINUTES'){
               sh '''
               date
               cd ${WKC}/tests
@@ -150,7 +164,7 @@ pipeline {
           }
         }
 
-        stage('test_valgrind') {
+        stage('test_valgrind_s4') {
           agent{label "b3"}
 
           steps {
@@ -162,7 +176,7 @@ pipeline {
                 ./handle_val_log.sh
                 '''
             }     
-            timeout(time: 90, unit: 'MINUTES'){      
+            timeout(time: 45, unit: 'MINUTES'){      
               sh '''
               date
               cd ${WKC}/tests
@@ -171,8 +185,58 @@ pipeline {
             }
           }
         }
-   
-        
+        stage('test_b4_s7') {
+          agent{label 'b4'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b4fq
+              date'''
+            }
+          }
+        }
+        stage('test_b5_s8') {
+          agent{label 'b5'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b5fq
+              date'''
+            }
+          }
+        }
+        stage('test_b6_s9') {
+          agent{label 'b6'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b6fq
+              date'''
+            }
+          }
+        }
+        stage('test_b7_s10') {
+          agent{label 'b7'}
+          steps {     
+            timeout(time: 45, unit: 'MINUTES'){       
+              pre_test()
+              sh '''
+              date
+              cd ${WKC}/tests
+              ./test-all.sh b7fq
+              date'''
+            }
+          }
+        }        
     }
   }
   }

cmake/define.inc

@@ -21,6 +21,10 @@ IF (TD_TSDB_PLUGINS)
   ADD_DEFINITIONS(-D_TSDB_PLUGINS)
 ENDIF ()
 
+IF (TD_STORAGE)
+  ADD_DEFINITIONS(-D_STORAGE)
+ENDIF ()
+
 IF (TD_GODLL)
   ADD_DEFINITIONS(-D_TD_GO_DLL_)
 ENDIF ()

src/CMakeLists.txt

@@ -5,6 +5,7 @@ PROJECT(TDengine)
 ADD_SUBDIRECTORY(os)
 ADD_SUBDIRECTORY(common)
 ADD_SUBDIRECTORY(util)
+ADD_SUBDIRECTORY(tfs)
 ADD_SUBDIRECTORY(rpc)
 ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(query)

src/client/src/tscSQLParser.c

@@ -3440,6 +3440,7 @@ static int32_t getTagCondString(tSQLExpr* pExpr, char** str) {
 
 static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr* pTableCond, SStringBuilder* sb) {
   const char* msg0 = "invalid table name list";
+  const char* msg1 = "not string following like";
 
   if (pTableCond == NULL) {
     return TSDB_CODE_SUCCESS;
@@ -3457,6 +3458,10 @@ static int32_t getTablenameCond(SSqlCmd* pCmd, SQueryInfo* pQueryInfo, tSQLExpr*
   if (pTableCond->nSQLOptr == TK_IN) {
     ret = tablenameListToString(pRight, sb);
   } else if (pTableCond->nSQLOptr == TK_LIKE) {
+    if (pRight->nSQLOptr != TK_STRING) {
+      return invalidSqlErrMsg(tscGetErrorMsgPayload(pCmd), msg1);
+    }
+    
     ret = tablenameCondToString(pRight, sb);
   }
 

src/client/src/tscSub.c

@@ -61,7 +61,7 @@ TSKEY tscGetSubscriptionProgress(void* sub, int64_t uid, TSKEY dflt) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = 0};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
   if (p == NULL) {
     return dflt;
   }
@@ -76,7 +76,7 @@ void tscUpdateSubscriptionProgress(void* sub, int64_t uid, TSKEY ts) {
   SSub* pSub = (SSub*)sub;
 
   SSubscriptionProgress target = {.uid = uid, .key = ts};
-  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+  SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
   if (p != NULL) {
     p->key = ts;
     tscDebug("subscribe:%s, uid:%"PRIu64" update sub start ts:%"PRId64, pSub->topic, p->uid, p->key);
@@ -270,7 +270,7 @@ static int tscUpdateSubscription(STscObj* pObj, SSub* pSub) {
   if (UTIL_TABLE_IS_NORMAL_TABLE(pTableMetaInfo)) {
     STableMeta * pTableMeta = pTableMetaInfo->pTableMeta;
     SSubscriptionProgress target = {.uid = pTableMeta->id.uid, .key = 0};
-    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress);
+    SSubscriptionProgress* p = taosArraySearch(pSub->progress, &target, tscCompareSubscriptionProgress, TD_EQ);
     if (p == NULL) {
       taosArrayClear(pSub->progress);
       taosArrayPush(pSub->progress, &target);

src/client/src/tscSubquery.c

@@ -2410,7 +2410,7 @@ static void multiVnodeInsertFinalize(void* param, TAOS_RES* tres, int numOfRows)
 
   // record the total inserted rows
   if (numOfRows > 0) {
-    pParentObj->res.numOfRows += numOfRows;
+    atomic_add_fetch_32(&pParentObj->res.numOfRows, numOfRows);
   }
 
   if (taos_errno(tres) != TSDB_CODE_SUCCESS) {

src/client/src/tscSystem.c

@@ -18,7 +18,6 @@
 #include "tref.h"
 #include "trpc.h"
 #include "tnote.h"
-#include "tsystem.h"
 #include "ttimer.h"
 #include "tutil.h"
 #include "tsched.h"
@@ -49,7 +48,7 @@ int32_t   tscNumOfThreads = 1;     // num of rpc threads
 static    pthread_mutex_t rpcObjMutex; // mutex to protect open the rpc obj concurrently 
 static pthread_once_t tscinit = PTHREAD_ONCE_INIT;
 
-void tscCheckDiskUsage(void *UNUSED_PARAM(para), void* UNUSED_PARAM(param)) {
+void tscCheckDiskUsage(void *UNUSED_PARAM(para), void *UNUSED_PARAM(param)) {
   taosGetDisk();
   taosTmrReset(tscCheckDiskUsage, 1000, NULL, tscTmr, &tscCheckDiskUsageTmr);
 }
@@ -88,7 +87,7 @@ int32_t tscAcquireRpc(const char *key, const char *user, const char *secretEncry
   rpcInit.sessions = tsMaxConnections;
   rpcInit.connType = TAOS_CONN_CLIENT;
   rpcInit.user = (char *)user;
-  rpcInit.idleTime = 2000; 
+  rpcInit.idleTime = tsShellActivityTimer * 1000; 
   rpcInit.ckey = "key"; 
   rpcInit.spi = 1; 
   rpcInit.secret = (char *)secretEncrypt;

src/common/inc/tdataformat.h

@@ -68,9 +68,9 @@ typedef struct {
 typedef struct {
   int      version;    // version
   int      numOfCols;  // Number of columns appended
-  int      tlen;       // maximum length of a SDataRow without the header part
+  int      tlen;       // maximum length of a SDataRow without the header part (sizeof(VarDataOffsetT) + sizeof(VarDataLenT) + (bytes))
   uint16_t flen;       // First part length in a SDataRow after the header part
-  uint16_t vlen;       // pure value part length, excluded the overhead
+  uint16_t vlen;       // pure value part length, excluded the overhead (bytes only)
   STColumn columns[];
 } STSchema;
 
@@ -278,7 +278,7 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows);
 void       tdResetDataCols(SDataCols *pCols);
 int        tdInitDataCols(SDataCols *pCols, STSchema *pSchema);
 SDataCols *tdDupDataCols(SDataCols *pCols, bool keepData);
-void       tdFreeDataCols(SDataCols *pCols);
+SDataCols *tdFreeDataCols(SDataCols *pCols);
 void       tdAppendDataRowToDataCol(SDataRow row, STSchema *pSchema, SDataCols *pCols);
 int        tdMergeDataCols(SDataCols *target, SDataCols *src, int rowsToMerge);
 

src/common/inc/tglobal.h

@@ -16,6 +16,8 @@
 #ifndef TDENGINE_COMMON_GLOBAL_H
 #define TDENGINE_COMMON_GLOBAL_H
 
+#include "taosdef.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif
@@ -147,7 +149,6 @@ extern char    tsDataDir[];
 extern char    tsLogDir[];
 extern char    tsScriptDir[];
 extern int64_t tsMsPerDay[3];
-extern char    tsVnodeBakDir[];
 
 // system info
 extern char    tsOsName[];
@@ -196,6 +197,14 @@ extern int32_t wDebugFlag;
 extern int32_t cqDebugFlag;
 extern int32_t debugFlag;
 
+typedef struct {
+  char dir[TSDB_FILENAME_LEN];
+  int  level;
+  int  primary;
+} SDiskCfg;
+extern int32_t  tsDiskCfgNum;
+extern SDiskCfg tsDiskCfg[];
+
 #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize)
 
 void    taosInitGlobalCfg();
@@ -204,6 +213,9 @@ void    taosSetAllDebugFlag();
 bool    taosCfgDynamicOptions(char *msg);
 int     taosGetFqdnPortFromEp(const char *ep, char *fqdn, uint16_t *port);
 bool    taosCheckBalanceCfgOptions(const char *option, int32_t *vnodeId, int32_t *dnodeId);
+void    taosAddDataDir(int index, char *v1, int level, int primary);
+void    taosReadDataDirCfg(char *v1, char *v2, char *v3);
+void    taosPrintDataDirCfg();
 
 #ifdef __cplusplus
 }

src/common/src/tdataformat.c

@@ -289,23 +289,31 @@ SDataCols *tdNewDataCols(int maxRowSize, int maxCols, int maxRows) {
     return NULL;
   }
 
-  pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
-  if (pCols->cols == NULL) {
-    uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno));
-    tdFreeDataCols(pCols);
-    return NULL;
+  pCols->maxPoints = maxRows;
+
+  if (maxCols > 0) {
+    pCols->cols = (SDataCol *)calloc(maxCols, sizeof(SDataCol));
+    if (pCols->cols == NULL) {
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols,
+             strerror(errno));
+      tdFreeDataCols(pCols);
+      return NULL;
+    }
+
+    pCols->maxCols = maxCols;
   }
 
   pCols->maxRowSize = maxRowSize;
-  pCols->maxCols = maxCols;
-  pCols->maxPoints = maxRows;
   pCols->bufSize = maxRowSize * maxRows;
 
-  pCols->buf = malloc(pCols->bufSize);
-  if (pCols->buf == NULL) {
-    uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols, strerror(errno));
-    tdFreeDataCols(pCols);
-    return NULL;
+  if (pCols->bufSize > 0) {
+    pCols->buf = malloc(pCols->bufSize);
+    if (pCols->buf == NULL) {
+      uDebug("malloc failure, size:%" PRId64 " failed, reason:%s", (int64_t)sizeof(SDataCol) * maxCols,
+             strerror(errno));
+      tdFreeDataCols(pCols);
+      return NULL;
+    }
   }
 
   return pCols;
@@ -337,12 +345,13 @@ int tdInitDataCols(SDataCols *pCols, STSchema *pSchema) {
   return 0;
 }
 
-void tdFreeDataCols(SDataCols *pCols) {
+SDataCols *tdFreeDataCols(SDataCols *pCols) {
   if (pCols) {
     tfree(pCols->buf);
     tfree(pCols->cols);
     free(pCols);
   }
+  return NULL;
 }
 
 SDataCols *tdDupDataCols(SDataCols *pDataCols, bool keepData) {

src/common/src/tglobal.c

@@ -182,7 +182,14 @@ char   tsDnodeDir[TSDB_FILENAME_LEN] = {0};
 char   tsMnodeDir[TSDB_FILENAME_LEN] = {0};
 char   tsDataDir[TSDB_FILENAME_LEN] = {0};
 char   tsScriptDir[TSDB_FILENAME_LEN] = {0};
-char   tsVnodeBakDir[TSDB_FILENAME_LEN] = {0};
+
+int32_t  tsDiskCfgNum = 0;
+
+#ifndef _STORAGE
+SDiskCfg tsDiskCfg[1];
+#else
+SDiskCfg tsDiskCfg[TSDB_MAX_DISKS];
+#endif
 
 /*
  * minimum scale for whole system, millisecond by default
@@ -227,6 +234,7 @@ int32_t sDebugFlag = 135;
 int32_t wDebugFlag = 135;
 int32_t tsdbDebugFlag = 131;
 int32_t cqDebugFlag = 131;
+int32_t fsDebugFlag = 135;
 
 int32_t (*monStartSystemFp)() = NULL;
 void (*monStopSystemFp)() = NULL;
@@ -334,6 +342,39 @@ bool taosCfgDynamicOptions(char *msg) {
   return false;
 }
 
+void taosAddDataDir(int index, char *v1, int level, int primary) {
+  tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN);
+  tsDiskCfg[index].level = level;
+  tsDiskCfg[index].primary = primary;
+  uTrace("dataDir:%s, level:%d primary:%d is configured", v1, level, primary);
+}
+
+#ifndef _STORAGE
+void taosReadDataDirCfg(char *v1, char *v2, char *v3) {
+  if (tsDiskCfgNum == 1) {
+    SDiskCfg *cfg = &tsDiskCfg[0];
+    uInfo("dataDir:%s, level:%d primary:%d is replaced by %s", cfg->dir, cfg->level, cfg->primary, v1);
+  }
+  taosAddDataDir(0, v1, 0, 1);
+  tsDiskCfgNum = 1;
+}
+
+void taosPrintDataDirCfg() {
+  for (int i = 0; i < tsDiskCfgNum; ++i) {
+    SDiskCfg *cfg = &tsDiskCfg[i];
+    uInfo(" dataDir: %s", cfg->dir);
+  }
+}
+#endif
+
+static void taosCheckDataDirCfg() {
+  if (tsDiskCfgNum <= 0) {
+    taosAddDataDir(0, tsDataDir, 0, 1);
+    tsDiskCfgNum = 1;
+    uTrace("dataDir:%s, level:0 primary:1 is configured by default", tsDataDir);
+  }
+}
+
 static void doInitGlobalConfig(void) {
   osInit();
   srand(taosSafeRand());
@@ -415,7 +456,7 @@ static void doInitGlobalConfig(void) {
 
   cfg.option = "dataDir";
   cfg.ptr = tsDataDir;
-  cfg.valType = TAOS_CFG_VTYPE_DIRECTORY;
+  cfg.valType = TAOS_CFG_VTYPE_DATA_DIRCTORY;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG;
   cfg.minValue = 0;
   cfg.maxValue = 0;
@@ -1448,6 +1489,7 @@ int32_t taosCheckGlobalCfg() {
     snprintf(tsSecond, sizeof(tsSecond), "%s:%u", fqdn, port);
   }
 
+  taosCheckDataDirCfg();
   taosGetSystemInfo();
 
   tsSetLocale();

src/connector/jdbc/pom.xml

@@ -36,6 +36,7 @@
         <maven-compiler-plugin.version>3.6.0</maven-compiler-plugin.version>
         <commons-logging.version>1.1.2</commons-logging.version>
         <commons-lang3.version>3.5</commons-lang3.version>
+        <maven.test.jvmargs></maven.test.jvmargs>
     </properties>
     <dependencies>
         <dependency>
@@ -122,11 +123,14 @@
                 <artifactId>maven-surefire-plugin</artifactId>
                 <version>2.12.4</version>
                 <configuration>
+                    <forkMode>pertest</forkMode>
+                    <argLine>${maven.test.jvmargs}</argLine>
                     <includes>
                         <include>**/*Test.java</include>
                     </includes>
                     <excludes>
                         <exclude>**/AppMemoryLeakTest.java</exclude>
+                        <exclude>**/TaosInfoMonitorTest.java</exclude>
                         <exclude>**/FailOverTest.java</exclude>
                     </excludes>
                     <testFailureIgnore>true</testFailureIgnore>

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBJNIConnector.java

@@ -14,6 +14,8 @@
  *****************************************************************************/
 package com.taosdata.jdbc;
 
+import com.taosdata.jdbc.utils.TaosInfo;
+
 import java.sql.SQLException;
 import java.sql.SQLWarning;
 import java.util.List;
@@ -21,6 +23,8 @@ import java.util.List;
 public class TSDBJNIConnector {
     private static volatile Boolean isInitialized = false;
 
+    private TaosInfo taosInfo = TaosInfo.getInstance();
+
     static {
         System.loadLibrary("taos");
         System.out.println("java.library.path:" + System.getProperty("java.library.path"));
@@ -91,7 +95,8 @@ public class TSDBJNIConnector {
      */
     public boolean connect(String host, int port, String dbName, String user, String password) throws SQLException {
         if (this.taos != TSDBConstants.JNI_NULL_POINTER) {
-            this.closeConnectionImp(this.taos);
+//            this.closeConnectionImp(this.taos);
+            closeConnection();
             this.taos = TSDBConstants.JNI_NULL_POINTER;
         }
 
@@ -99,7 +104,8 @@ public class TSDBJNIConnector {
         if (this.taos == TSDBConstants.JNI_NULL_POINTER) {
             throw new SQLException(TSDBConstants.WrapErrMsg(this.getErrMsg(0L)), "", this.getErrCode(0l));
         }
-
+        // invoke connectImp only here
+        taosInfo.conn_open_increment();
         return true;
     }
 
@@ -120,6 +126,7 @@ public class TSDBJNIConnector {
         Long pSql = 0l;
         try {
             pSql = this.executeQueryImp(sql.getBytes(TaosGlobalConfig.getCharset()), this.taos);
+            taosInfo.stmt_count_increment();
         } catch (Exception e) {
             e.printStackTrace();
             this.freeResultSetImp(this.taos, pSql);
@@ -244,10 +251,11 @@ public class TSDBJNIConnector {
     private native int fetchRowImp(long connection, long resultSet, TSDBResultSetRowData rowData);
 
     public int fetchBlock(long resultSet, TSDBResultSetBlockData blockData) {
-		return this.fetchBlockImp(this.taos, resultSet, blockData);
+        return this.fetchBlockImp(this.taos, resultSet, blockData);
     }
-    
+
     private native int fetchBlockImp(long connection, long resultSet, TSDBResultSetBlockData blockData);
+
     /**
      * Execute close operation from C to release connection pointer by JNI
      *
@@ -262,6 +270,8 @@ public class TSDBJNIConnector {
         } else {
             throw new SQLException("Undefined error code returned by TDengine when closing a connection");
         }
+        // invoke closeConnectionImpl only here
+        taosInfo.connect_close_increment();
     }
 
     private native int closeConnectionImp(long connection);

src/connector/jdbc/src/main/java/com/taosdata/jdbc/TSDBStatement.java

@@ -14,13 +14,15 @@
  *****************************************************************************/
 package com.taosdata.jdbc;
 
+import com.taosdata.jdbc.utils.TaosInfo;
+
 import java.sql.*;
 import java.util.ArrayList;
 import java.util.List;
-import java.util.concurrent.TimeUnit;
 
 public class TSDBStatement implements Statement {
-    private TSDBJNIConnector connector = null;
+    private TSDBJNIConnector connector;
+    private TaosInfo taosInfo = TaosInfo.getInstance();
 
     /**
      * To store batched commands

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNode.java

-package com.taosdata.jdbc.utils;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.InputStreamReader;
-import java.util.*;
-import java.util.concurrent.TimeUnit;
-
-public class TDNode {
-    
-    private int index;
-    private int running;
-    private int deployed;
-    private boolean testCluster;    
-    private String path;    
-    private String cfgDir;
-    private String dataDir;
-    private String logDir;
-    private String cfgPath;
-
-    public TDNode(int index) {
-        this.index = index;
-        running = 0;
-        deployed = 0;
-        testCluster = false;            
-    }
-
-    public void setPath(String path) {
-        this.path = path;
-    }
-
-    public void setTestCluster(boolean testCluster) {
-        this.testCluster = testCluster;
-    }
-
-    public void setRunning(int running) {
-        this.running = running;
-    }
-
-    public void searchTaosd(File dir, ArrayList<String> taosdPath) {        
-        File[] fileList = dir.listFiles();
-        
-        if(fileList == null || fileList.length == 0) {
-            return;
-        }
-        
-        for(File file : fileList) {
-            if(file.isFile()) {
-                if(file.getName().equals("taosd")) {                        
-                    taosdPath.add(file.getAbsolutePath());
-                }
-            } else {
-                searchTaosd(file, taosdPath);
-            } 
-        }        
-    }
-
-    public void start() {
-        String selfPath = System.getProperty("user.dir");
-        String binPath = "";
-        String projDir = selfPath + "/../../../";
-
-        try {
-            ArrayList<String> taosdPath = new ArrayList<>();
-            
-            File dir = new File(projDir);
-            String realProjDir = dir.getCanonicalPath();
-            dir = new File(realProjDir);
-            System.out.println("project Dir: " + projDir);
-            searchTaosd(dir, taosdPath);
-                        
-            if(taosdPath.size() == 0) {
-                System.out.println("The project path doens't exist");
-                return;
-            } else {
-                for(String p : taosdPath) {
-                    if(!p.contains("packaging")) {
-                        binPath = p;
-                    }
-                }
-            }
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-
-        if(binPath.isEmpty()) {
-            System.out.println("taosd not found");
-            return;
-        } else {
-            System.out.println("taosd found in " + binPath);
-        }
-
-        if(this.deployed == 0) {
-            System.out.println("dnode" + index + "is not deployed");
-            return;
-        }
-
-        String cmd = "nohup " + binPath + " -c " + cfgDir + " > /dev/null 2>&1 & "; 
-        System.out.println("start taosd cmd: " + cmd);
-        
-        try{
-            Runtime.getRuntime().exec(cmd);  
-            TimeUnit.SECONDS.sleep(5);
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-        
-        this.running = 1;        
-    }
-
-    public Integer getTaosdPid() {
-        String cmd = "ps -ef|grep -w taosd| grep -v grep | awk '{print $2}'"; 
-        String[] cmds = {"sh", "-c", cmd};
-        try {
-            Process process = Runtime.getRuntime().exec(cmds);
-            BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()));
-            String line = null;
-            Integer res = null;
-            while((line = reader.readLine()) != null) {
-                if(!line.isEmpty()) {
-                    res = Integer.valueOf(line);
-                    break;
-                }
-            }
-            
-            return res;
-        } catch (Exception e) {
-            e.printStackTrace();
-        } 
-        return null;
-    }
-
-    public void stop() {        
-
-        if (this.running != 0) {
-            Integer pid = null;                        
-            while((pid = getTaosdPid()) != null) {
-                
-                String killCmd = "kill -term " + pid;                                      
-                String[] killCmds = {"sh", "-c", killCmd};
-                try {                                
-                    Runtime.getRuntime().exec(killCmds).waitFor();                    
-
-                    TimeUnit.SECONDS.sleep(2);
-                } catch (Exception e) {
-                    e.printStackTrace();
-                }                
-            }
-
-            try {
-                for(int port = 6030; port < 6041; port ++) {
-                    String fuserCmd = "fuser -k -n tcp " + port;
-                    Runtime.getRuntime().exec(fuserCmd).waitFor();
-                }
-            } catch (Exception e) {
-                e.printStackTrace();
-            }
-
-            this.running = 0;
-            System.out.println("dnode:" + this.index + " is stopped by kill -term");
-        }
-    }
-
-    public void startIP() {
-        try{
-            String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " up";        
-            Runtime.getRuntime().exec(cmd).waitFor();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    public void stopIP() {
-        try{
-            String cmd = "sudo ifconfig lo:" + index + "192.168.0." + index + " down";            
-            Runtime.getRuntime().exec(cmd).waitFor();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    public void setCfgConfig(String option, String value) {
-        try{
-            String cmd = "echo " + option + " " + value + " >> " + this.cfgPath;
-            String[] cmdLine = {"sh", "-c", cmd};
-            Process ps = Runtime.getRuntime().exec(cmdLine);
-            ps.waitFor();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }
-    }
-
-    public String getDnodeRootDir() {
-        String dnodeRootDir = this.path + "/sim/psim/dnode" + this.index;
-        return dnodeRootDir;
-    }
-
-    public String getDnodesRootDir() {
-        String dnodesRootDir = this.path + "/sim/psim" + this.index;
-        return dnodesRootDir;
-    }
-
-    public void deploy() {
-        this.logDir = this.path + "/sim/dnode" + this.index + "/log";
-        this.dataDir = this.path + "/sim/dnode" + this.index + "/data";
-        this.cfgDir = this.path + "/sim/dnode" + this.index + "/cfg";
-        this.cfgPath = this.path + "/sim/dnode" + this.index + "/cfg/taos.cfg";
-
-        try {        
-            String cmd = "rm -rf " + this.logDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-            
-            cmd = "rm -rf " + this.cfgDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-
-            cmd = "rm -rf " + this.dataDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-
-            cmd = "mkdir -p " + this.logDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-
-            cmd = "mkdir -p " + this.cfgDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-
-            cmd = "mkdir -p " + this.dataDir;
-            Runtime.getRuntime().exec(cmd).waitFor();
-
-            cmd = "touch " + this.cfgPath;
-            Runtime.getRuntime().exec(cmd).waitFor();
-        } catch (Exception e) {
-            e.printStackTrace();
-        }        
-
-        if(this.testCluster) {
-            startIP();
-            setCfgConfig("masterIp", "192.168.0.1");
-            setCfgConfig("secondIp", "192.168.0.2");
-            setCfgConfig("publicIp", "192.168.0." + this.index);
-            setCfgConfig("internalIp", "192.168.0." + this.index);
-            setCfgConfig("privateIp", "192.168.0." + this.index);
-        }
-        setCfgConfig("dataDir", this.dataDir);
-        setCfgConfig("logDir", this.logDir);
-        setCfgConfig("numOfLogLines", "1000000/00");
-        setCfgConfig("mnodeEqualVnodeNum", "0");
-        setCfgConfig("walLevel", "1");
-        setCfgConfig("statusInterval", "1");
-        setCfgConfig("numOfMnodes", "3");
-        setCfgConfig("numOfThreadsPerCore", "2.0");
-        setCfgConfig("monitor", "0");
-        setCfgConfig("maxVnodeConnections", "30000");
-        setCfgConfig("maxMgmtConnections", "30000");
-        setCfgConfig("maxMeterConnections", "30000");
-        setCfgConfig("maxShellConns", "30000");
-        setCfgConfig("locale", "en_US.UTF-8");
-        setCfgConfig("charset", "UTF-8");
-        setCfgConfig("asyncLog", "0");
-        setCfgConfig("anyIp", "0");
-        setCfgConfig("dDebugFlag", "135");
-        setCfgConfig("mDebugFlag", "135");
-        setCfgConfig("sdbDebugFlag", "135");
-        setCfgConfig("rpcDebugFlag", "135");
-        setCfgConfig("tmrDebugFlag", "131");
-        setCfgConfig("cDebugFlag", "135");
-        setCfgConfig("httpDebugFlag", "135");
-        setCfgConfig("monitorDebugFlag", "135");
-        setCfgConfig("udebugFlag", "135");
-        setCfgConfig("jnidebugFlag", "135");
-        setCfgConfig("qdebugFlag", "135"); 
-        this.deployed = 1;
-    }
-}
\ No newline at end of file

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TDNodes.java

-package com.taosdata.jdbc.utils;
-
-import java.io.File;
-import java.util.*;
-
-public class TDNodes {
-    private ArrayList<TDNode> tdNodes; 
-    private boolean testCluster;     
-
-    public TDNodes () {
-        tdNodes = new ArrayList<>();
-        for(int i = 1; i < 11; i ++) {            
-            tdNodes.add(new TDNode(i));
-        }               
-    }
-
-    public void setTestCluster(boolean testCluster) {
-        this.testCluster = testCluster;
-    }
-
-    public void check(int index) {
-        if(index < 1 || index > 10) {
-            System.out.println("index: " + index + " should on a scale of [1, 10]");
-            return;
-        }
-    } 
-
-    public void deploy(int index) {
-        try {     
-            File file = new File(System.getProperty("user.dir") + "/../../../");
-            String projectRealPath = file.getCanonicalPath();
-            check(index);        
-            tdNodes.get(index - 1).setTestCluster(this.testCluster);            
-            tdNodes.get(index - 1).setPath(projectRealPath); 
-            tdNodes.get(index - 1).deploy();
-        } catch (Exception e) {
-            e.printStackTrace();
-            System.out.println("deploy Test Exception");
-        }
-    }
-    
-    public void cfg(int index, String option, String value) {
-        check(index);
-        tdNodes.get(index - 1).setCfgConfig(option, value);
-    }
-
-    public TDNode getTDNode(int index) {
-        check(index);
-        return tdNodes.get(index - 1);
-    }
-    
-    public void start(int index) {
-        check(index);
-        tdNodes.get(index - 1).start();
-    }
-
-    public void stop(int index) {
-        check(index);
-        tdNodes.get(index - 1).stop();
-    }
-    
-    public void startIP(int index) {
-        check(index);
-        tdNodes.get(index - 1).startIP();
-    }
-
-    public void stopIP(int index) {
-        check(index);
-        tdNodes.get(index - 1).stopIP();
-    }
-
-}
\ No newline at end of file

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfo.java

+package com.taosdata.jdbc.utils;
+
+import javax.management.*;
+import java.lang.management.ManagementFactory;
+import java.util.concurrent.atomic.AtomicLong;
+
+public class TaosInfo implements TaosInfoMBean {
+
+    private static volatile TaosInfo instance;
+    private AtomicLong connect_open = new AtomicLong();
+    private AtomicLong connect_close = new AtomicLong();
+    private AtomicLong statement_count = new AtomicLong();
+
+    static {
+        try {
+            MBeanServer server = ManagementFactory.getPlatformMBeanServer();
+            ObjectName name = new ObjectName("TaosInfoMBean:name=TaosInfo");
+            server.registerMBean(TaosInfo.getInstance(), name);
+
+        } catch (MalformedObjectNameException | InstanceAlreadyExistsException | MBeanRegistrationException | NotCompliantMBeanException e) {
+            e.printStackTrace();
+        }
+    }
+
+    @Override
+    public long getConnect_open() {
+        return connect_open.get();
+    }
+
+    @Override
+    public long getConnect_close() {
+        return connect_close.get();
+    }
+
+    @Override
+    public long getConnect_active() {
+        return connect_open.get() - connect_close.get();
+    }
+
+    @Override
+    public long getStatement_count() {
+        return statement_count.get();
+    }
+
+    /*******************************************************/
+
+    public void conn_open_increment() {
+        connect_open.incrementAndGet();
+    }
+
+    public void connect_close_increment() {
+        connect_close.incrementAndGet();
+    }
+
+    public void stmt_count_increment() {
+        statement_count.incrementAndGet();
+    }
+
+    /********************************************************************************/
+    private TaosInfo() {
+    }
+
+    public static TaosInfo getInstance() {
+        if (instance == null) {
+            synchronized (TaosInfo.class) {
+                if (instance == null) {
+                    instance = new TaosInfo();
+                }
+            }
+        }
+        return instance;
+    }
+
+}

src/connector/jdbc/src/main/java/com/taosdata/jdbc/utils/TaosInfoMBean.java

+package com.taosdata.jdbc.utils;
+
+public interface TaosInfoMBean {
+
+    long getConnect_open();
+
+    long getConnect_close();
+
+    long getConnect_active();
+
+    long getStatement_count();
+
+}

src/connector/jdbc/src/test/java/com/taosdata/jdbc/cases/TaosInfoMonitorTest.java

+package com.taosdata.jdbc.cases;
+
+import org.junit.Test;
+
+import java.sql.*;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+public class TaosInfoMonitorTest {
+
+    @Test
+    public void testCreateTooManyConnection() throws ClassNotFoundException {
+        Class.forName("com.taosdata.jdbc.TSDBDriver");
+        final String url = "jdbc:TAOS://127.0.0.1:6030/?user=root&password=taosdata";
+
+        List<Connection> connectionList = IntStream.range(0, 100).mapToObj(i -> {
+            try {
+                TimeUnit.MILLISECONDS.sleep(100);
+                return DriverManager.getConnection(url);
+            } catch (SQLException | InterruptedException e) {
+                e.printStackTrace();
+            }
+            return null;
+        }).collect(Collectors.toList());
+
+        connectionList.stream().forEach(conn -> {
+            try (Statement stmt = conn.createStatement()) {
+                ResultSet rs = stmt.executeQuery("show databases");
+                while (rs.next()) {
+
+                }
+                TimeUnit.MILLISECONDS.sleep(100);
+            } catch (SQLException | InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
+
+        connectionList.stream().forEach(conn -> {
+            try {
+                conn.close();
+                TimeUnit.MILLISECONDS.sleep(100);
+            } catch (SQLException | InterruptedException e) {
+                e.printStackTrace();
+            }
+        });
+    }
+}

src/cq/src/cqMain.c

@@ -125,8 +125,6 @@ void cqFree(void *handle) {
   pthread_mutex_unlock(&pContext->mutex);
 
   if (delete) {
-    pthread_mutex_unlock(&pContext->mutex);
-    
     pthread_mutex_destroy(&pContext->mutex);
     
     taosTmrCleanUp(pContext->tmrCtrl);
@@ -186,6 +184,18 @@ void *cqOpen(void *ahandle, const SCqCfg *pCfg) {
   return pContext;
 }
 
+static void freeSCqContext(void *handle) {
+  if (handle == NULL) {
+    return;
+  }
+  SCqContext *pContext = handle;
+  pthread_mutex_destroy(&pContext->mutex);
+    
+  taosTmrCleanUp(pContext->tmrCtrl);
+  pContext->tmrCtrl = NULL;
+  cDebug("vgId:%d, CQ is closed", pContext->vgId);
+  free(pContext);
+}
 void cqClose(void *handle) {
   if (tsEnableStream == 0) {
     return;
@@ -217,6 +227,8 @@ void cqClose(void *handle) {
     
     taosRemoveRef(cqObjRef, rid);
   }
+
+  freeSCqContext(pContext);
 }
 
 void cqStart(void *handle) {

src/dnode/CMakeLists.txt

@@ -27,7 +27,7 @@ IF (TD_GRANT)
   TARGET_LINK_LIBRARIES(taosd grant)
 ENDIF ()
 
-IF ((TD_LINUX OR TD_WINDOWS) AND TD_MQTT)
+IF (TD_MQTT)
   TARGET_LINK_LIBRARIES(taosd mqtt)
 ENDIF ()
 
@@ -43,5 +43,6 @@ ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}
     COMMAND ${CMAKE_COMMAND} -E echo dataDir ${TD_TESTS_OUTPUT_DIR}/data > ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMAND ${CMAKE_COMMAND} -E echo logDir ${TD_TESTS_OUTPUT_DIR}/log  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMAND ${CMAKE_COMMAND} -E echo charset UTF-8  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
+    COMMAND ${CMAKE_COMMAND} -E echo monitor 0  >> ${TD_TESTS_OUTPUT_DIR}/cfg/taos.cfg
     COMMENT "prepare taosd environment")
 ADD_CUSTOM_TARGET(${PREPARE_ENV_TARGET} ALL WORKING_DIRECTORY ${TD_EXECUTABLE_OUTPUT_PATH} DEPENDS ${PREPARE_ENV_CMD})

src/dnode/src/dnodeMain.c

@@ -21,7 +21,7 @@
 #include "tconfig.h"
 #include "tfile.h"
 #include "twal.h"
-// #include "tfs.h"
+#include "tfs.h"
 #include "tsync.h"
 #include "dnodeStep.h"
 #include "dnodePeer.h"
@@ -189,32 +189,35 @@ static void dnodeCheckDataDirOpenned(char *dir) {
 }
 
 static int32_t dnodeInitStorage() {
-  if (dnodeCreateDir(tsDataDir) < 0) {
-   dError("failed to create dir: %s, reason: %s", tsDataDir, strerror(errno));
-   return -1;
+  if (tfsInit(tsDiskCfg, tsDiskCfgNum) < 0) {
+    dError("failed to init TFS since %s", tstrerror(terrno));
+    return -1;
   }
+  strncpy(tsDataDir, TFS_PRIMARY_PATH(), TSDB_FILENAME_LEN);
   sprintf(tsMnodeDir, "%s/mnode", tsDataDir);
   sprintf(tsVnodeDir, "%s/vnode", tsDataDir);
   sprintf(tsDnodeDir, "%s/dnode", tsDataDir);
-  sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir);
+  // sprintf(tsVnodeBakDir, "%s/vnode_bak", tsDataDir);
 
   //TODO(dengyihao): no need to init here 
   if (dnodeCreateDir(tsMnodeDir) < 0) {
    dError("failed to create dir: %s, reason: %s", tsMnodeDir, strerror(errno));
    return -1;
   } 
-  //TODO(dengyihao): no need to init here
-  if (dnodeCreateDir(tsVnodeDir) < 0) {
-   dError("failed to create dir: %s, reason: %s", tsVnodeDir, strerror(errno));
-   return -1;
-  }
+
   if (dnodeCreateDir(tsDnodeDir) < 0) {
    dError("failed to create dir: %s, reason: %s", tsDnodeDir, strerror(errno));
    return -1;
-  } 
-  if (dnodeCreateDir(tsVnodeBakDir) < 0) {
-   dError("failed to create dir: %s, reason: %s", tsVnodeBakDir, strerror(errno));
-   return -1;
+  }
+
+  if (tfsMkdir("vnode") < 0) {
+    dError("failed to create vnode dir since %s", tstrerror(terrno));
+    return -1;
+  }
+
+  if (tfsMkdir("vnode_bak") < 0) {
+    dError("failed to create vnode_bak dir since %s", tstrerror(terrno));
+    return -1;
   }
 
   dnodeCheckDataDirOpenned(tsDnodeDir);
@@ -223,7 +226,7 @@ static int32_t dnodeInitStorage() {
   return 0;
 }
 
-static void dnodeCleanupStorage() {}
+static void dnodeCleanupStorage() { tfsDestroy(); }
 
 bool  dnodeIsFirstDeploy() {
   return strcmp(tsFirst, tsLocalEp) == 0;

src/dnode/src/dnodeVMgmt.c

@@ -174,7 +174,7 @@ static int32_t dnodeProcessAlterVnodeMsg(SRpcMsg *rpcMsg) {
     vnodeRelease(pVnode);
     return code;
   } else {
-    dError("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
+    dInfo("vgId:%d, vnode not exist, can't alter it", pAlter->cfg.vgId);
     return TSDB_CODE_VND_INVALID_VGROUP_ID;
   }
 }

src/inc/taoserror.h

@@ -242,6 +242,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM,     0, 0x060F, "No table d
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS,      0, 0x0610, "File already exists")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE,        0, 0x0611, "Need to reconfigure table")
 TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO,    0, 0x0612, "Invalid information to create table")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK,            0, 0x0613, "No available disk")
+TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG,               0, 0x0614, "TSDB messed message")
 
 // query
 TAOS_DEFINE_ERROR(TSDB_CODE_QRY_INVALID_QHANDLE,          0, 0x0700, "Invalid handle")

src/inc/tfs.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_TFS_H
+#define TD_TFS_H
+
+#include "tglobal.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  int level;
+  int id;
+} SDiskID;
+
+#define TFS_UNDECIDED_LEVEL -1
+#define TFS_UNDECIDED_ID -1
+#define TFS_PRIMARY_LEVEL 0
+#define TFS_PRIMARY_ID 0
+
+// FS APIs ====================================
+typedef struct {
+  int64_t tsize;
+  int64_t avail;
+} SFSMeta;
+
+int  tfsInit(SDiskCfg *pDiskCfg, int ndisk);
+void tfsDestroy();
+void tfsUpdateInfo(SFSMeta *pFSMeta);
+void tfsGetMeta(SFSMeta *pMeta);
+void tfsAllocDisk(int expLevel, int *level, int *id);
+
+const char *TFS_PRIMARY_PATH();
+const char *TFS_DISK_PATH(int level, int id);
+
+// TFILE APIs ====================================
+typedef struct {
+  int  level;
+  int  id;
+  char rname[TSDB_FILENAME_LEN];  // REL name
+  char aname[TSDB_FILENAME_LEN];  // ABS name
+} TFILE;
+
+#define TFILE_LEVEL(pf) ((pf)->level)
+#define TFILE_ID(pf) ((pf)->id)
+#define TFILE_NAME(pf) ((pf)->aname)
+#define TFILE_REL_NAME(pf) ((pf)->rname)
+
+#define tfsopen(pf, flags) open(TFILE_NAME(pf), flags)
+#define tfsclose(fd) close(fd)
+#define tfsremove(pf) remove(TFILE_NAME(pf))
+#define tfscopy(sf, df) taosCopy(TFILE_NAME(sf), TFILE_NAME(df))
+#define tfsrename(sf, df) rename(TFILE_NAME(sf), TFILE_NAME(df))
+
+void  tfsInitFile(TFILE *pf, int level, int id, const char *bname);
+bool  tfsIsSameFile(const TFILE *pf1, const TFILE *pf2);
+int   tfsEncodeFile(void **buf, TFILE *pf);
+void *tfsDecodeFile(void *buf, TFILE *pf);
+void  tfsbasename(const TFILE *pf, char *dest);
+void  tfsdirname(const TFILE *pf, char *dest);
+
+// DIR APIs ====================================
+int tfsMkdirAt(const char *rname, int level, int id);
+int tfsMkdirRecurAt(const char *rname, int level, int id);
+int tfsMkdir(const char *rname);
+int tfsRmdir(const char *rname);
+int tfsRename(char *orname, char *nrname);
+
+typedef struct TDIR TDIR;
+
+TDIR *       tfsOpendir(const char *rname);
+const TFILE *tfsReaddir(TDIR *tdir);
+void         tfsClosedir(TDIR *tdir);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file

src/inc/tsdb.h

@@ -40,7 +40,8 @@ extern "C" {
 
 // TSDB STATE DEFINITION
 #define TSDB_STATE_OK 0x0
-#define TSDB_STATE_BAD_FILE 0x1
+#define TSDB_STATE_BAD_META 0x1
+#define TSDB_STATE_BAD_DATA 0x2
 
 // --------- TSDB APPLICATION HANDLE DEFINITION
 typedef struct {
@@ -48,7 +49,7 @@ typedef struct {
   void *cqH;
   int (*notifyStatus)(void *, int status, int eno);
   int (*eventCallBack)(void *);
-  void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char* dstTable, char *sqlStr, STSchema *pSchema);
+  void *(*cqCreateFunc)(void *handle, uint64_t uid, int32_t sid, const char *dstTable, char *sqlStr, STSchema *pSchema);
   void (*cqDropFunc)(void *handle);
 } STsdbAppH;
 
@@ -76,17 +77,17 @@ typedef struct {
   int64_t pointsWritten;  // total data points written
 } STsdbStat;
 
-typedef void TSDB_REPO_T;  // use void to hide implementation details from outside
+typedef struct STsdbRepo STsdbRepo;
 
-STsdbCfg *tsdbGetCfg(const TSDB_REPO_T *repo);
+STsdbCfg *tsdbGetCfg(const STsdbRepo *repo);
 
 // --------- TSDB REPOSITORY DEFINITION
-int          tsdbCreateRepo(char *rootDir, STsdbCfg *pCfg);
-int32_t      tsdbDropRepo(char *rootDir);
-TSDB_REPO_T *tsdbOpenRepo(char *rootDir, STsdbAppH *pAppH);
-int          tsdbCloseRepo(TSDB_REPO_T *repo, int toCommit);
-int32_t      tsdbConfigRepo(TSDB_REPO_T *repo, STsdbCfg *pCfg);
-int          tsdbGetState(TSDB_REPO_T *repo);
+int32_t    tsdbCreateRepo(int repoid);
+int32_t    tsdbDropRepo(int repoid);
+STsdbRepo *tsdbOpenRepo(STsdbCfg *pCfg, STsdbAppH *pAppH);
+int        tsdbCloseRepo(STsdbRepo *repo, int toCommit);
+int32_t    tsdbConfigRepo(STsdbRepo *repo, STsdbCfg *pCfg);
+int        tsdbGetState(STsdbRepo *repo);
 
 // --------- TSDB TABLE DEFINITION
 typedef struct {
@@ -110,8 +111,8 @@ typedef struct {
 
 void tsdbClearTableCfg(STableCfg *config);
 
-void* tsdbGetTableTagVal(const void* pTable, int32_t colId, int16_t type, int16_t bytes);
-char* tsdbGetTableName(void *pTable);
+void *tsdbGetTableTagVal(const void *pTable, int32_t colId, int16_t type, int16_t bytes);
+char *tsdbGetTableName(void *pTable);
 
 #define TSDB_TABLEID(_table) ((STableId*) (_table))
 #define TSDB_PREV_ROW  0x1
@@ -119,12 +120,11 @@ char* tsdbGetTableName(void *pTable);
 
 STableCfg *tsdbCreateTableCfgFromMsg(SMDCreateTableMsg *pMsg);
 
-int   tsdbCreateTable(TSDB_REPO_T *repo, STableCfg *pCfg);
-int   tsdbDropTable(TSDB_REPO_T *pRepo, STableId tableId);
-int   tsdbUpdateTableTagValue(TSDB_REPO_T *repo, SUpdateTableTagValMsg *pMsg);
-// TSKEY tsdbGetTableLastKey(TSDB_REPO_T *repo, uint64_t uid);
+int tsdbCreateTable(STsdbRepo *repo, STableCfg *pCfg);
+int tsdbDropTable(STsdbRepo *pRepo, STableId tableId);
+int tsdbUpdateTableTagValue(STsdbRepo *repo, SUpdateTableTagValMsg *pMsg);
 
-uint32_t tsdbGetFileInfo(TSDB_REPO_T *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size);
+uint32_t tsdbGetFileInfo(STsdbRepo *repo, char *name, uint32_t *index, uint32_t eindex, int64_t *size);
 
 // the TSDB repository info
 typedef struct STsdbRepoInfo {
@@ -134,7 +134,7 @@ typedef struct STsdbRepoInfo {
   int64_t  tsdbTotalDiskSize;  // the total disk size taken by this TSDB repository
   // TODO: Other informations to add
 } STsdbRepoInfo;
-STsdbRepoInfo *tsdbGetStatus(TSDB_REPO_T *pRepo);
+STsdbRepoInfo *tsdbGetStatus(STsdbRepo *pRepo);
 
 // the meter information report structure
 typedef struct {
@@ -152,7 +152,7 @@ typedef struct {
  *
  * @return the number of points inserted, -1 for failure and the error number is set
  */
-int32_t tsdbInsertData(TSDB_REPO_T *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
+int32_t tsdbInsertData(STsdbRepo *repo, SSubmitMsg *pMsg, SShellSubmitRspMsg *pRsp);
 
 // -- FOR QUERY TIME SERIES DATA
 
@@ -168,9 +168,9 @@ typedef struct STsdbQueryCond {
 } STsdbQueryCond;
 
 typedef struct SMemRef {
-  int32_t  ref;
-  void    *mem;
-  void    *imem;
+  int32_t ref;
+  void *  mem;
+  void *  imem;
 } SMemRef;
 
 typedef struct SDataBlockInfo {
@@ -182,14 +182,14 @@ typedef struct SDataBlockInfo {
 } SDataBlockInfo;
 
 typedef struct {
-  void  *pTable;
-  TSKEY  lastKey;
+  void *pTable;
+  TSKEY lastKey;
 } STableKeyInfo;
 
 typedef struct {
   size_t    numOfTables;
-  SArray   *pGroupList;
-  SHashObj *map;         // speedup acquire the tableQueryInfo by table uid
+  SArray *  pGroupList;
+  SHashObj *map;  // speedup acquire the tableQueryInfo by table uid
 } STableGroupInfo;
 
 /**
@@ -202,7 +202,8 @@ typedef struct {
  * @param qinfo      query info handle from query processor
  * @return
  */
-TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo, SMemRef* pRef);
+TsdbQueryHandleT *tsdbQueryTables(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfoGroup, void *qinfo,
+                                  SMemRef *pRef);
 
 /**
  * Get the last row of the given query time window for all the tables in STableGroupInfo object.
@@ -214,14 +215,15 @@ TsdbQueryHandleT *tsdbQueryTables(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STab
  * @param tableInfo  table list.
  * @return
  */
-TsdbQueryHandleT tsdbQueryLastRow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo, SMemRef* pRef);
+TsdbQueryHandleT tsdbQueryLastRow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *tableInfo, void *qinfo,
+                                  SMemRef *pRef);
 
 /**
  * get the queried table object list
  * @param pHandle
  * @return
  */
-SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
+SArray *tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
 
 /**
  * get the group list according to table id from client
@@ -231,8 +233,8 @@ SArray* tsdbGetQueriedTableList(TsdbQueryHandleT *pHandle);
  * @param qinfo
  * @return
  */
-TsdbQueryHandleT tsdbQueryRowsInExternalWindow(TSDB_REPO_T *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
-                                               void *qinfo, SMemRef* pRef);
+TsdbQueryHandleT tsdbQueryRowsInExternalWindow(STsdbRepo *tsdb, STsdbQueryCond *pCond, STableGroupInfo *groupList,
+                                               void *qinfo, SMemRef *pRef);
 
 
 /**
@@ -268,7 +270,7 @@ SArray* tsdbGetExternalRow(TsdbQueryHandleT *pHandle, SMemRef* pMemRef, int16_t
  * @param pBlockInfo
  * @return
  */
-void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo* pBlockInfo);
+void tsdbRetrieveDataBlockInfo(TsdbQueryHandleT *pQueryHandle, SDataBlockInfo *pBlockInfo);
 
 /**
  *
@@ -299,7 +301,7 @@ SArray *tsdbRetrieveDataBlock(TsdbQueryHandleT *pQueryHandle, SArray *pColumnIdL
  * @param stableid. super table sid
  * @param pTagCond. tag query condition
  */
-int32_t tsdbQuerySTableByTagCond(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len,
+int32_t tsdbQuerySTableByTagCond(STsdbRepo *tsdb, uint64_t uid, TSKEY key, const char *pTagCond, size_t len,
                                  int16_t tagNameRelType, const char *tbnameCond, STableGroupInfo *pGroupList,
                                  SColIndex *pColIndex, int32_t numOfCols);
 
@@ -317,7 +319,7 @@ void tsdbDestroyTableGroup(STableGroupInfo *pGroupList);
  * @param pGroupInfo  the generated result
  * @return
  */
-int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
+int32_t tsdbGetOneTableGroup(STsdbRepo *tsdb, uint64_t uid, TSKEY startKey, STableGroupInfo *pGroupInfo);
 
 /**
  *
@@ -326,7 +328,7 @@ int32_t tsdbGetOneTableGroup(TSDB_REPO_T *tsdb, uint64_t uid, TSKEY startKey, ST
  * @param pGroupInfo
  * @return
  */
-int32_t tsdbGetTableGroupFromIdList(TSDB_REPO_T* tsdb, SArray* pTableIdList, STableGroupInfo* pGroupInfo);
+int32_t tsdbGetTableGroupFromIdList(STsdbRepo *tsdb, SArray *pTableIdList, STableGroupInfo *pGroupInfo);
 
 /**
  * clean up the query handle
@@ -345,10 +347,14 @@ void tsdbReportStat(void *repo, int64_t *totalPoints, int64_t *totalStorage, int
 
 int  tsdbInitCommitQueue();
 void tsdbDestroyCommitQueue();
-int  tsdbSyncCommit(TSDB_REPO_T *repo);
+int  tsdbSyncCommit(STsdbRepo *repo);
 void tsdbIncCommitRef(int vgId);
 void tsdbDecCommitRef(int vgId);
 
+// For TSDB file sync
+int tsdbSyncSend(void *pRepo, SOCKET socketFd);
+int tsdbSyncRecv(void *pRepo, SOCKET socketFd);
+
 #ifdef __cplusplus
 }
 #endif

src/inc/tsync.h

@@ -56,16 +56,6 @@ typedef struct {
   int32_t  role[TAOS_SYNC_MAX_REPLICA];
 } SNodesRole;
 
-/* 
-  if name is empty(name[0] is zero), get the file from index or after, but not larger than eindex. If a file
-  is found between index and eindex, index shall be updated, name shall be set, size shall be set to 
-  file size, and file magic number shall be returned. 
-
-  if name is provided(name[0] is not zero), get the named file at the specified index. If not there, return
-  zero. If it is there, set the size to file size, and return file magic number. Index shall not be updated.
-*/
-typedef uint32_t (*FGetFileInfo)(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion); 
-
 // get the wal file from index or after
 // return value, -1: error, 1:more wal files, 0:last WAL. if name[0]==0, no WAL file
 typedef int32_t  (*FGetWalInfo)(int32_t vgId, char *fileName, int64_t *fileId); 
@@ -83,24 +73,31 @@ typedef void     (*FNotifyRole)(int32_t vgId, int8_t role);
 typedef void     (*FNotifyFlowCtrl)(int32_t vgId, int32_t level);
 
 // when data file is synced successfully, notity app
-typedef int32_t  (*FNotifyFileSynced)(int32_t vgId, uint64_t fversion);
+typedef void     (*FStartSyncFile)(int32_t vgId);
+typedef void     (*FStopSyncFile)(int32_t vgId, uint64_t fversion);
 
 // get file version
 typedef int32_t  (*FGetVersion)(int32_t vgId, uint64_t *fver, uint64_t *vver);
 
+typedef int32_t  (*FSendFile)(void *tsdb, SOCKET socketFd);
+typedef int32_t  (*FRecvFile)(void *tsdb, SOCKET socketFd);
+
 typedef struct {
   int32_t  vgId;       // vgroup ID
   uint64_t version;    // initial version
   SSyncCfg syncCfg;    // configuration from mgmt
   char     path[TSDB_FILENAME_LEN];  // path to the file
-  FGetFileInfo      getFileInfo;
-  FGetWalInfo       getWalInfo;
-  FWriteToCache     writeToCache;
+  void *   pTsdb;
+  FGetWalInfo       getWalInfoFp;
+  FWriteToCache     writeToCacheFp;
   FConfirmForward   confirmForward;
-  FNotifyRole       notifyRole;
-  FNotifyFlowCtrl   notifyFlowCtrl;
-  FNotifyFileSynced notifyFileSynced;
-  FGetVersion       getVersion;
+  FNotifyRole       notifyRoleFp;
+  FNotifyFlowCtrl   notifyFlowCtrlFp;
+  FStartSyncFile    startSyncFileFp;
+  FStopSyncFile     stopSyncFileFp;
+  FGetVersion       getVersionFp;
+  FSendFile         sendFileFp;
+  FRecvFile         recvFileFp;
 } SSyncInfo;
 
 typedef void *tsync_h;

src/kit/shell/src/shellEngine.c

@@ -470,7 +470,7 @@ static int dumpResultToFile(const char* fname, TAOS_RES* tres) {
 
   wordexp_t full_path;
 
-  if (wordexp(fname, &full_path, 0) != 0) {
+  if (wordexp((char *)fname, &full_path, 0) != 0) {
     fprintf(stderr, "ERROR: invalid file name: %s\n", fname);
     return -1;
   }

src/kit/taosdemox/taosdemox.c

@@ -173,6 +173,7 @@ typedef struct SSuperTable_S {
   int          childTblCount;
   bool         superTblExists;    // 0: no, 1: yes
   bool         childTblExists;    // 0: no, 1: yes  
+  int          batchCreateTableNum;  // 0: no batch,  > 0: batch table number in one sql
   int8_t       autoCreateTable;                  // 0: create sub table, 1: auto create sub table
   char         childTblPrefix[MAX_TB_NAME_SIZE];
   char         dataSource[MAX_TB_NAME_SIZE];  // rand_gen or sample
@@ -808,13 +809,14 @@ static void init_rand_data() {
 
 static void printfInsertMeta() {
   printf("\033[1m\033[40;32m================ insert.json parse result START ================\033[0m\n");
-  printf("host:                    \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port);
-  printf("user:                    \033[33m%s\033[0m\n", g_Dbs.user);
-  printf("password:                \033[33m%s\033[0m\n", g_Dbs.password);
-  printf("resultFile:              \033[33m%s\033[0m\n", g_Dbs.resultFile);
-  printf("thread count:            \033[33m%d\033[0m\n", g_Dbs.threadCount);
-
-  printf("database count:          \033[33m%d\033[0m\n", g_Dbs.dbCount);
+  printf("host:                       \033[33m%s:%u\033[0m\n", g_Dbs.host, g_Dbs.port);
+  printf("user:                       \033[33m%s\033[0m\n", g_Dbs.user);
+  printf("password:                   \033[33m%s\033[0m\n", g_Dbs.password);
+  printf("resultFile:                 \033[33m%s\033[0m\n", g_Dbs.resultFile);
+  printf("thread num of insert data:  \033[33m%d\033[0m\n", g_Dbs.threadCount);
+  printf("thread num of create table: \033[33m%d\033[0m\n", g_Dbs.threadCountByCreateTbl);
+
+  printf("database count:             \033[33m%d\033[0m\n", g_Dbs.dbCount);
   for (int i = 0; i < g_Dbs.dbCount; i++) {
     printf("database[\033[33m%d\033[0m]:\n", i);
     printf("  database name:         \033[33m%s\033[0m\n", g_Dbs.db[i].dbName);
@@ -944,11 +946,12 @@ static void printfInsertMeta() {
 
 static void printfInsertMetaToFile(FILE* fp) {
   fprintf(fp, "================ insert.json parse result START================\n");
-  fprintf(fp, "host:                    %s:%u\n", g_Dbs.host, g_Dbs.port);
-  fprintf(fp, "user:                    %s\n", g_Dbs.user);
-  fprintf(fp, "password:                %s\n", g_Dbs.password);
-  fprintf(fp, "resultFile:              %s\n", g_Dbs.resultFile);
-  fprintf(fp, "thread count:            %d\n", g_Dbs.threadCount);
+  fprintf(fp, "host:                       %s:%u\n", g_Dbs.host, g_Dbs.port);
+  fprintf(fp, "user:                       %s\n", g_Dbs.user);
+  fprintf(fp, "password:                   %s\n", g_Dbs.password);
+  fprintf(fp, "resultFile:                 %s\n", g_Dbs.resultFile);
+  fprintf(fp, "thread num of insert data:  %d\n", g_Dbs.threadCount);
+  fprintf(fp, "thread num of create table: %d\n", g_Dbs.threadCountByCreateTbl);
 
   fprintf(fp, "database count:          %d\n", g_Dbs.dbCount);
   for (int i = 0; i < g_Dbs.dbCount; i++) {
@@ -1730,19 +1733,27 @@ static int createDatabases() {
 
 
 void * createTable(void *sarg) 
-{
-  char command[BUFFER_SIZE] = "\0";
-  
+{  
   threadInfo *winfo = (threadInfo *)sarg; 
   SSuperTable* superTblInfo = winfo->superTblInfo;
 
   int64_t  lastPrintTime = taosGetTimestampMs();
 
+  char* buffer = calloc(superTblInfo->maxSqlLen, 1);
+
+  int len = 0;
+  int batchNum = 0;
   //printf("Creating table from %d to %d\n", winfo->start_table_id, winfo->end_table_id);
   for (int i = winfo->start_table_id; i <= winfo->end_table_id; i++) {
     if (0 == g_Dbs.use_metric) {
-      snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable);
+      snprintf(buffer, BUFFER_SIZE, "create table if not exists %s.%s%d %s;", winfo->db_name, superTblInfo->childTblPrefix, i, superTblInfo->colsOfCreatChildTable);
     } else {
+      if (0 == len) {  
+        batchNum = 0;
+        memset(buffer, 0, superTblInfo->maxSqlLen);
+        len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "create table ");
+      }
+      
       char* tagsValBuf = NULL;
       if (0 == superTblInfo->tagSource) {
         tagsValBuf = generateTagVaulesForStb(superTblInfo);
@@ -1750,13 +1761,22 @@ void * createTable(void *sarg)
         tagsValBuf = getTagValueFromTagSample(superTblInfo, i % superTblInfo->tagSampleCount);
       }
       if (NULL == tagsValBuf) {
+        free(buffer);
         return NULL;
       }
-      snprintf(command, BUFFER_SIZE, "create table if not exists %s.%s%d using %s.%s tags %s;", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf);
+      
+      len += snprintf(buffer + len, superTblInfo->maxSqlLen - len, "if not exists %s.%s%d using %s.%s tags %s ", winfo->db_name, superTblInfo->childTblPrefix, i, winfo->db_name, superTblInfo->sTblName, tagsValBuf);
       free(tagsValBuf);
+      batchNum++;
+
+      if ((batchNum < superTblInfo->batchCreateTableNum) && ((superTblInfo->maxSqlLen - len) >= (superTblInfo->lenOfTagOfOneRow + 256))) {
+        continue;
+      }
     }
-    
-    if (0 != queryDbExec(winfo->taos, command, NO_INSERT_TYPE)){
+
+    len = 0;
+    if (0 != queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE)){
+      free(buffer);
       return NULL;
     }
 
@@ -1766,7 +1786,12 @@ void * createTable(void *sarg)
       lastPrintTime = currentPrintTime;
     }
   }
-
+  
+  if (0 != len) {
+    (void)queryDbExec(winfo->taos, buffer, NO_INSERT_TYPE);
+  }
+  
+  free(buffer);
   return NULL;
 }
 
@@ -2422,6 +2447,16 @@ static bool getMetaFromInsertJsonFile(cJSON* root) {
         printf("failed to read json, auto_create_table not found");
         goto PARSE_OVER;
       }
+      
+      cJSON* batchCreateTbl = cJSON_GetObjectItem(stbInfo, "batch_create_tbl_num");
+      if (batchCreateTbl && batchCreateTbl->type == cJSON_Number) {
+        g_Dbs.db[i].superTbls[j].batchCreateTableNum = batchCreateTbl->valueint;
+      } else if (!batchCreateTbl) {
+        g_Dbs.db[i].superTbls[j].batchCreateTableNum = 2000;
+      } else {
+        printf("failed to read json, batch_create_tbl_num not found");
+        goto PARSE_OVER;
+      }      
 
       cJSON *childTblExists = cJSON_GetObjectItem(stbInfo, "child_table_exists"); // yes, no
       if (childTblExists && childTblExists->type == cJSON_String && childTblExists->valuestring != NULL) {
@@ -3679,14 +3714,14 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
     b = ntables % threads;
   }
 
-  TAOS* taos;
-  if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
-    taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
-    if (NULL == taos) {
-      printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
-      exit(-1);
-    }
-  }
+  //TAOS* taos;
+  //if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
+  //  taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
+  //  if (NULL == taos) {
+  //    printf("connect to server fail, reason: %s\n", taos_errstr(NULL));
+  //    exit(-1);
+  //  }
+  //}
 
   int32_t timePrec = TSDB_TIME_PRECISION_MILLI;
   if (0 != precision[0]) {
@@ -3719,7 +3754,12 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
     t_info->start_time = start_time;
 
     if (0 == strncasecmp(superTblInfo->insertMode, "taosc", 5)) {
-      t_info->taos = taos;
+      //t_info->taos = taos;
+      t_info->taos = taos_connect(g_Dbs.host, g_Dbs.user, g_Dbs.password, db_name, g_Dbs.port);
+      if (NULL == t_info->taos) {
+        printf("connect to server fail from insert sub thread, reason: %s\n", taos_errstr(NULL));
+        exit(-1);
+      }
     } else {
       t_info->taos = NULL;
       #ifdef TD_LOWA_CURL
@@ -3754,6 +3794,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
     threadInfo *t_info = infos + i;
 
     tsem_destroy(&(t_info->lock_sem));
+    taos_close(t_info->taos);
 
     superTblInfo->totalAffectedRows += t_info->totalAffectedRows;
     superTblInfo->totalRowsInserted += t_info->totalRowsInserted;
@@ -3766,7 +3807,7 @@ void startMultiThreadInsertData(int threads, char* db_name, char* precision, SSu
 
   double end = getCurrentTime();
   
-  taos_close(taos);
+  //taos_close(taos);
 
   free(pids);
   free(infos);  

src/mnode/src/mnodePeer.c

@@ -17,7 +17,6 @@
 #include "os.h"
 #include "taoserror.h"
 #include "tsched.h"
-#include "tsystem.h"
 #include "tutil.h"
 #include "tgrant.h"
 #include "tbn.h"

src/mnode/src/mnodeSdb.c

@@ -242,11 +242,6 @@ void sdbUpdateMnodeRoles() {
   mnodeUpdateMnodeEpSet(NULL);
 }
 
-static uint32_t sdbGetFileInfo(int32_t vgId, char *name, uint32_t *index, uint32_t eindex, int64_t *size, uint64_t *fversion) {
-  sdbUpdateMnodeRoles();
-  return 0;
-}
-
 static int32_t sdbGetWalInfo(int32_t vgId, char *fileName, int64_t *fileId) {
   return walGetWalFile(tsSdbMgmt.wal, fileName, fileId);
 }
@@ -262,7 +257,9 @@ static void sdbNotifyRole(int32_t vgId, int8_t role) {
   sdbUpdateMnodeRoles();
 }
 
-static int32_t sdbNotifyFileSynced(int32_t vgId, uint64_t fversion) { return 0; }
+static void sdbStartFileSync(int32_t vgId) {}
+
+static void sdbStopFileSync(int32_t vgId, uint64_t fversion) {}
 
 static void sdbNotifyFlowCtrl(int32_t vgId, int32_t level) {}
 
@@ -396,14 +393,14 @@ int32_t sdbUpdateSync(void *pMnodes) {
   syncInfo.version = sdbGetVersion();
   syncInfo.syncCfg = syncCfg;
   sprintf(syncInfo.path, "%s", tsMnodeDir);
-  syncInfo.getFileInfo = sdbGetFileInfo;
-  syncInfo.getWalInfo = sdbGetWalInfo;
-  syncInfo.writeToCache = sdbWriteFwdToQueue;
+  syncInfo.getWalInfoFp = sdbGetWalInfo;
+  syncInfo.writeToCacheFp = sdbWriteFwdToQueue;
   syncInfo.confirmForward = sdbConfirmForward;
-  syncInfo.notifyRole = sdbNotifyRole;
-  syncInfo.notifyFileSynced = sdbNotifyFileSynced;
-  syncInfo.notifyFlowCtrl = sdbNotifyFlowCtrl;
-  syncInfo.getVersion = sdbGetSyncVersion;
+  syncInfo.notifyRoleFp = sdbNotifyRole;
+  syncInfo.startSyncFileFp = sdbStartFileSync;
+  syncInfo.stopSyncFileFp = sdbStopFileSync;
+  syncInfo.notifyFlowCtrlFp = sdbNotifyFlowCtrl;
+  syncInfo.getVersionFp = sdbGetSyncVersion;
   tsSdbMgmt.cfg = syncCfg;
 
   if (tsSdbMgmt.sync) {

src/os/inc/osDarwin.h

@@ -105,6 +105,8 @@ typedef int(*__compar_fn_t)(const void *, const void *);
   #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
 #endif
 
+#define TAOS_OS_FUNC_PTHREAD_RWLOCK
+
 int64_t tsosStr2int64(char *str);
 
 #include "eok.h"

src/os/inc/osFile.h

@@ -26,6 +26,7 @@ int64_t taosReadImp(int32_t fd, void *buf, int64_t count);
 int64_t taosWriteImp(int32_t fd, void *buf, int64_t count);
 int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence);
 int32_t taosRenameFile(char *fullPath, char *suffix, char delimiter, char **dstPath);
+int64_t taosCopy(char *from, char *to);
 
 #define taosRead(fd, buf, count) taosReadImp(fd, buf, count)
 #define taosWrite(fd, buf, count) taosWriteImp(fd, buf, count)

src/os/inc/osMemory.h

@@ -35,7 +35,7 @@ void   taosDumpMemoryLeak();
 void * taosTMalloc(size_t size);
 void * taosTCalloc(size_t nmemb, size_t size);
 void * taosTRealloc(void *ptr, size_t size);
-void   taosTZfree(void *ptr);
+void * taosTZfree(void *ptr);
 size_t taosTSizeof(void *ptr);
 void   taosTMemset(void *ptr, int c);
 

src/os/inc/osSemphone.h

@@ -28,6 +28,21 @@ extern "C" {
   #define tsem_destroy sem_destroy
 #endif
 
+#ifdef TAOS_OS_FUNC_PTHREAD_RWLOCK
+  #define pthread_rwlock_t pthread_mutex_t
+  #define pthread_rwlock_init(lock, NULL) pthread_mutex_init(lock, NULL)
+  #define pthread_rwlock_destroy(lock) pthread_mutex_destroy(lock)
+  #define pthread_rwlock_wrlock(lock) pthread_mutex_lock(lock)
+  #define pthread_rwlock_rdlock(lock) pthread_mutex_lock(lock)
+  #define pthread_rwlock_unlock(lock) pthread_mutex_unlock(lock)
+
+  #define pthread_spinlock_t pthread_mutex_t
+  #define pthread_spin_init(lock, NULL) pthread_mutex_init(lock, NULL)
+  #define pthread_spin_destroy(lock) pthread_mutex_destroy(lock)
+  #define pthread_spin_lock(lock) pthread_mutex_lock(lock)
+  #define pthread_spin_unlock(lock) pthread_mutex_unlock(lock)
+#endif
+
 // TAOS_OS_FUNC_SEMPHONE_PTHREAD
 bool    taosCheckPthreadValid(pthread_t thread);
 int64_t taosGetSelfPthreadId();

src/os/inc/osSysinfo.h

@@ -21,10 +21,16 @@ extern "C" {
 #endif
 
 // TAOS_OS_FUNC_SYSINFO
+typedef struct {
+  int64_t tsize;
+  int64_t avail;
+} SysDiskSize;
+
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize);
 void taosGetSystemInfo();
 bool taosGetProcIO(float *readKB, float *writeKB);
 bool taosGetBandSpeed(float *bandSpeedKb);
-bool taosGetDisk();
+void taosGetDisk();
 bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) ;
 bool taosGetProcMemory(float *memoryUsedMB) ;
 bool taosGetSysMemory(float *memoryUsedMB);

src/os/inc/osWindows.h

@@ -201,13 +201,15 @@ int        gettimeofday(struct timeval *ptv, void *pTimeZone);
 
 typedef struct {
   int    we_wordc;
-  char **we_wordv;
+  char  *we_wordv[1];
   int    we_offs;
-  char   wordPos[20];
+  char   wordPos[1025];
 } wordexp_t;
-int  wordexp(const char *words, wordexp_t *pwordexp, int flags);
+int  wordexp(char *words, wordexp_t *pwordexp, int flags);
 void wordfree(wordexp_t *pwordexp);
 
+char *realpath(char *path, char *resolved_path);
+
 #define openlog(a, b, c)
 #define closelog()
 #define LOG_ERR 0

src/os/src/darwin/darwinSysInfo.c

@@ -18,6 +18,7 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
+#include "taoserror.h"
 #include <errno.h>
 #include <libproc.h>
 
@@ -70,8 +71,6 @@ void taosGetSystemInfo() {
   taosGetSystemLocale();
 }
 
-bool taosGetDisk() { return true; }
-
 bool taosGetProcIO(float *readKB, float *writeKB) {
   *readKB = 0;
   *writeKB = 0;
@@ -106,6 +105,19 @@ int taosSystem(const char *cmd) {
 
 void taosSetCoreDump() {}
 
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
+  struct statvfs info;
+  if (statvfs(tsDataDir, &info)) {
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  } else {
+    diskSize->tsize = info.f_blocks * info.f_frsize;
+    diskSize->avail = info.f_bavail * info.f_frsize;
+    return 0;
+  }
+}
+
 char cmdline[1024];
 
 char *taosGetCmdlineByPID(int pid) {

src/os/src/detail/osFile.c

@@ -119,6 +119,40 @@ int64_t taosLSeekImp(int32_t fd, int64_t offset, int32_t whence) {
   return (int64_t)lseek(fd, (long)offset, whence);
 }
 
+int64_t taosCopy(char *from, char *to) {
+  char    buffer[4096];
+  int     fidto = -1, fidfrom = -1;
+  int64_t size = 0;
+  int64_t bytes;
+
+  fidfrom = open(from, O_RDONLY | O_BINARY);
+  if (fidfrom < 0) goto _err;
+
+  fidto = open(to, O_WRONLY | O_CREAT | O_EXCL | O_BINARY, 0755);
+  if (fidto < 0) goto _err;
+
+  while (true) {
+    bytes = taosRead(fidfrom, buffer, sizeof(buffer));
+    if (bytes < 0) goto _err;
+    if (bytes == 0) break;
+
+    size += bytes;
+
+    if (taosWrite(fidto, (void *)buffer, bytes) < bytes) goto _err;
+    if (bytes < sizeof(buffer)) break;
+  }
+
+  close(fidfrom);
+  close(fidto);
+  return size;
+
+_err:
+  if (fidfrom >= 0) close(fidfrom);
+  if (fidto >= 0) close(fidto);
+  remove(to);
+  return -1;
+}
+
 #ifndef TAOS_OS_FUNC_FILE_SENDIFLE
 
 int64_t taosSendFile(SOCKET dfd, int32_t sfd, int64_t *offset, int64_t size) {

src/os/src/detail/osMemory.c

@@ -512,8 +512,9 @@ void * taosTRealloc(void *ptr, size_t size) {
   return (void *)((char *)tptr + sizeof(size_t));
 }
 
-void taosTZfree(void *ptr) {
+void* taosTZfree(void* ptr) {
   if (ptr) {
-    free((void *)((char *)ptr - sizeof(size_t)));
+    free((void*)((char*)ptr - sizeof(size_t)));
   }
+  return NULL;
 }
\ No newline at end of file

src/os/src/detail/osSysinfo.c

@@ -18,6 +18,7 @@
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
+#include "taoserror.h"
 
 #ifndef TAOS_OS_FUNC_SYSINFO
 
@@ -316,37 +317,17 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   return true;
 }
 
-bool taosGetDisk() {
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
   struct statvfs info;
-  const double   unit = 1024 * 1024 * 1024;
-
-  if (tscEmbedded) {
-    if (statvfs(tsDataDir, &info)) {
-      uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
-      return false;
-    } else {
-      tsTotalDataDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
-      tsAvailDataDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
-    }
-  }
-
-  if (statvfs(tsLogDir, &info)) {
-    uError("failed to get disk size, logDir:%s errno:%s", tsLogDir, strerror(errno));
-    return false;
-  } else {
-    tsTotalLogDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
-    tsAvailLogDirGB = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
-  }
-
-  if (statvfs("/tmp", &info)) {
-    uError("failed to get disk size, tmpDir:/tmp errno:%s", strerror(errno));
-    return false;
+  if (statvfs(tsDataDir, &info)) {
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
   } else {
-    tsTotalTmpDirGB = (float)((double)info.f_blocks * (double)info.f_frsize / unit);
-    tsAvailTmpDirectorySpace = (float)((double)info.f_bavail * (double)info.f_frsize / unit);
+    diskSize->tsize = info.f_blocks * info.f_frsize;
+    diskSize->avail = info.f_bavail * info.f_frsize;
+    return 0;
   }
-
-  return true;
 }
 
 static bool taosGetCardInfo(int64_t *bytes) {
@@ -510,7 +491,7 @@ void taosGetSystemInfo() {
   float tmp1, tmp2;
   taosGetSysMemory(&tmp1);
   taosGetProcMemory(&tmp2);
-  taosGetDisk();
+  // taosGetDisk();
   taosGetBandSpeed(&tmp1);
   taosGetCpuUsage(&tmp1, &tmp2);
   taosGetProcIO(&tmp1, &tmp2);
@@ -537,7 +518,6 @@ void taosPrintOsInfo() {
   uInfo(" os release:             %s", buf.release);
   uInfo(" os version:             %s", buf.version);
   uInfo(" os machine:             %s", buf.machine);
-  uInfo("==================================");
 }
 
 void taosKillSystem() {

src/os/src/windows/wSysinfo.c

@@ -21,6 +21,7 @@
 #include "ttimer.h"
 #include "tulog.h"
 #include "tutil.h"
+#include "taoserror.h"
 #if (_WIN64)
 #include <iphlpapi.h>
 #include <mswsock.h>
@@ -126,37 +127,22 @@ bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage) {
   return true;
 }
 
-bool taosGetDisk() {
-  const double    unit = 1024 * 1024 * 1024;
-  BOOL            fResult;
+int32_t taosGetDiskSize(char *dataDir, SysDiskSize *diskSize) {
   unsigned _int64 i64FreeBytesToCaller;
   unsigned _int64 i64TotalBytes;
   unsigned _int64 i64FreeBytes;
 
-  if (tscEmbedded) {
-    fResult = GetDiskFreeSpaceExA(tsDataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-                                  (PULARGE_INTEGER)&i64FreeBytes);
-    if (fResult) {
-      tsTotalDataDirGB = (float)(i64TotalBytes / unit);
-      tsAvailDataDirGB = (float)(i64FreeBytes / unit);
-    }
-  }
-
-  fResult = GetDiskFreeSpaceExA(tsLogDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-                                (PULARGE_INTEGER)&i64FreeBytes);
+  BOOL fResult = GetDiskFreeSpaceExA(dataDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
+                                     (PULARGE_INTEGER)&i64FreeBytes);
   if (fResult) {
-    tsTotalLogDirGB = (float)(i64TotalBytes / unit);
-    tsAvailLogDirGB = (float)(i64FreeBytes / unit);
-  }
-
-  fResult = GetDiskFreeSpaceExA(tsTempDir, (PULARGE_INTEGER)&i64FreeBytesToCaller, (PULARGE_INTEGER)&i64TotalBytes,
-                                (PULARGE_INTEGER)&i64FreeBytes);
-  if (fResult) {
-    tsTotalTmpDirGB = (float)(i64TotalBytes / unit);
-    tsAvailTmpDirectorySpace = (float)(i64FreeBytes / unit);
+    diskSize->tsize = (int64_t)(i64TotalBytes);
+    diskSize->avail = (int64_t)(i64FreeBytes);
+    return 0;
+  } else {
+    uError("failed to get disk size, dataDir:%s errno:%s", tsDataDir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
   }
-
-  return true;
 }
 
 bool taosGetBandSpeed(float *bandSpeedKb) {
@@ -207,7 +193,7 @@ void taosGetSystemInfo() {
   tsTotalMemoryMB = taosGetTotalMemory();
 
   float tmp1, tmp2;
-  taosGetDisk();
+  // taosGetDisk();
   taosGetBandSpeed(&tmp1);
   taosGetCpuUsage(&tmp1, &tmp2);
   taosGetProcIO(&tmp1, &tmp2);

src/os/src/windows/wWordexp.c

@@ -21,13 +21,24 @@
 #include "tulog.h"
 #include "tutil.h"
 
-int wordexp(const char *words, wordexp_t *pwordexp, int flags) {
+int wordexp(char *words, wordexp_t *pwordexp, int flags) {
   pwordexp->we_offs = 0;
   pwordexp->we_wordc = 1;
-  pwordexp->we_wordv = (char **)(pwordexp->wordPos);
-  pwordexp->we_wordv[0] = (char *)words;
+  pwordexp->we_wordv[0] = pwordexp->wordPos;
+
+  memset(pwordexp->wordPos, 0, 1025);
+  if (_fullpath(pwordexp->wordPos, words, 1024) == NULL) {
+    pwordexp->we_wordv[0] = words;
+    uError("failed to parse relative path:%s to abs path", words);
+    return -1;
+  }
+
+  uTrace("parse relative path:%s to abs path:%s", words, pwordexp->wordPos);
   return 0;
 }
 
 void wordfree(wordexp_t *pwordexp) {}
 
+char *realpath(char *path, char *resolved_path) {
+  return _fullpath(path, resolved_path, TSDB_FILENAME_LEN - 1);
+}
\ No newline at end of file

src/plugins/http/src/httpRestJson.c

@@ -35,7 +35,7 @@ void restBuildSqlAffectRowsJson(HttpContext *pContext, HttpSqlCmd *cmd, int32_t
   // data row array end
   httpJsonToken(jsonBuf, JsonArrEnd);
 
-  cmd->numOfRows = affect_rows;
+  cmd->numOfRows = 1;
 }
 
 void restStartSqlJson(HttpContext *pContext, HttpSqlCmd *cmd, TAOS_RES *result) {

src/plugins/monitor/src/monMain.c

@@ -20,7 +20,6 @@
 #include "tlog.h"
 #include "ttimer.h"
 #include "tutil.h"
-#include "tsystem.h"
 #include "tscUtil.h"
 #include "tsclient.h"
 #include "dnode.h"

src/query/inc/queryLog.h

@@ -30,6 +30,7 @@ extern uint32_t qDebugFlag;
 #define qInfo(...)  do { if (qDebugFlag & DEBUG_INFO)  { taosPrintLog("QRY ", 255, __VA_ARGS__); }}        while(0)
 #define qDebug(...) do { if (qDebugFlag & DEBUG_DEBUG) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
 #define qTrace(...) do { if (qDebugFlag & DEBUG_TRACE) { taosPrintLog("QRY ", qDebugFlag, __VA_ARGS__); }} while(0)
+#define qDump(a, l) do { if (qDebugFlag & DEBUG_DUMP)  { taosDumpData((unsigned char *)a, l); }} while(0)
 
 #ifdef __cplusplus
 }

src/query/src/qExecutor.c

@@ -7160,14 +7160,23 @@ static int32_t doDumpQueryResult(SQInfo *pQInfo, char *data) {
     if (f) {
       off_t s = lseek(fileno(f), 0, SEEK_END);
 
-      qDebug("QInfo:%p ts comp data return, file:%p, size:%"PRId64, pQInfo, f, s);
+      qDebug("QInfo:%p ts comp data return, file:%p, size:%"PRId64, pQInfo, f, (uint64_t)s);
       if (fseek(f, 0, SEEK_SET) >= 0) {
         size_t sz = fread(data, 1, s, f);
         if(sz < s) {  // todo handle error
+          qError("fread(f:%p,%d) failed, rsize:%" PRId64 ", expect size:%" PRId64, f, fileno(f), (uint64_t)sz, (uint64_t)s);
           assert(0);
         }
       } else {
         UNUSED(s);
+        qError("fseek(f:%p,%d) failed, error:%s", f, fileno(f), strerror(errno));
+        assert(0);
+      }
+
+      if (s <= (sizeof(STSBufFileHeader) + sizeof(STSGroupBlockInfo) + 6 * sizeof(int32_t))) {
+        qDump(data, s);
+        
+        assert(0);
       }
 
       fclose(f);

src/rpc/src/rpcUdp.c

@@ -15,7 +15,6 @@
 
 #include "os.h"
 #include "tsocket.h"
-#include "tsystem.h"
 #include "ttimer.h"
 #include "tutil.h"
 #include "taosdef.h"

src/sync/inc/syncInt.h

@@ -108,14 +108,17 @@ typedef struct SSyncNode {
   SSyncFwds *  pSyncFwds;  // saved forward info if quorum >1
   void *       pFwdTimer;
   void *       pRoleTimer;
-  FGetFileInfo      getFileInfo;
-  FGetWalInfo       getWalInfo;
-  FWriteToCache     writeToCache;
+  void *       pTsdb;
+  FGetWalInfo       getWalInfoFp;
+  FWriteToCache     writeToCacheFp;
   FConfirmForward   confirmForward;
-  FNotifyRole       notifyRole;
-  FNotifyFlowCtrl   notifyFlowCtrl;
-  FNotifyFileSynced notifyFileSynced;
-  FGetVersion       getVersion;
+  FNotifyRole       notifyRoleFp;
+  FNotifyFlowCtrl   notifyFlowCtrlFp;
+  FStartSyncFile    startSyncFileFp;
+  FStopSyncFile     stopSyncFileFp;
+  FGetVersion       getVersionFp;
+  FSendFile         sendFileFp;
+  FRecvFile         recvFileFp;
   pthread_mutex_t   mutex;
 } SSyncNode;
 

src/sync/inc/syncMsg.h

@@ -99,16 +99,12 @@ typedef struct {
 
 typedef struct {
   SSyncHead head;
-  char      name[TSDB_FILENAME_LEN];
-  uint32_t  magic;
-  uint32_t  index;
   uint64_t  fversion;
-  int64_t   size;
-} SFileInfo;
+} SFileVersion;
 
 typedef struct {
   SSyncHead head;
-  int8_t    sync;
+  int8_t    ack;
 } SFileAck;
 
 typedef struct {
@@ -136,7 +132,7 @@ void syncBuildPeersStatus(SPeersStatus *pMsg, int32_t vgId);
 void syncBuildSyncTestMsg(SSyncMsg *pMsg, int32_t vgId);
 
 void syncBuildFileAck(SFileAck *pMsg, int32_t vgId);
-void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId);
+void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId);
 
 #ifdef __cplusplus
 }

src/sync/src/syncMain.c

@@ -174,19 +174,22 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
   pthread_mutex_init(&pNode->mutex, NULL);
 
-  pNode->getFileInfo = pInfo->getFileInfo;
-  pNode->getWalInfo = pInfo->getWalInfo;
-  pNode->writeToCache = pInfo->writeToCache;
-  pNode->notifyRole = pInfo->notifyRole;
+  pNode->getWalInfoFp = pInfo->getWalInfoFp;
+  pNode->writeToCacheFp = pInfo->writeToCacheFp;
+  pNode->notifyRoleFp = pInfo->notifyRoleFp;
   pNode->confirmForward = pInfo->confirmForward;
-  pNode->notifyFlowCtrl = pInfo->notifyFlowCtrl;
-  pNode->notifyFileSynced = pInfo->notifyFileSynced;
-  pNode->getVersion = pInfo->getVersion;
+  pNode->notifyFlowCtrlFp = pInfo->notifyFlowCtrlFp;
+  pNode->startSyncFileFp = pInfo->startSyncFileFp;
+  pNode->stopSyncFileFp = pInfo->stopSyncFileFp;
+  pNode->getVersionFp = pInfo->getVersionFp;
+  pNode->sendFileFp = pInfo->sendFileFp;
+  pNode->recvFileFp = pInfo->recvFileFp;
 
   pNode->selfIndex = -1;
   pNode->vgId = pInfo->vgId;
   pNode->replica = pCfg->replica;
   pNode->quorum = pCfg->quorum;
+  pNode->pTsdb = pInfo->pTsdb;
   if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
 
   pNode->refCount = 1;
@@ -248,8 +251,8 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   syncAddArbitrator(pNode);
   taosHashPut(tsVgIdHash, &pNode->vgId, sizeof(int32_t), &pNode, sizeof(SSyncNode *));
 
-  if (pNode->notifyRole) {
-    (*pNode->notifyRole)(pNode->vgId, nodeRole);
+  if (pNode->notifyRoleFp) {
+    (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
   }
 
   syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]);  // arb
@@ -357,7 +360,7 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
   if (pNewCfg->replica <= 1) {
     sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId);
     nodeRole = TAOS_SYNC_ROLE_MASTER;
-    (*pNode->notifyRole)(pNode->vgId, nodeRole);
+    (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
   }
 
   syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]);  // arb
@@ -417,7 +420,7 @@ void syncRecover(int64_t rid) {
   // if take this node to unsync state, the whole system may not work
 
   nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-  (*pNode->notifyRole)(pNode->vgId, nodeRole);
+  (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
   nodeVersion = 0;
 
   pthread_mutex_lock(&pNode->mutex);
@@ -625,8 +628,8 @@ static void syncResetFlowCtrl(SSyncNode *pNode) {
     pNode->peerInfo[index]->numOfRetrieves = 0;
   }
 
-  if (pNode->notifyFlowCtrl) {
-    (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
+  if (pNode->notifyFlowCtrlFp) {
+    (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
   }
 }
 
@@ -694,7 +697,7 @@ static void syncChooseMaster(SSyncNode *pNode) {
       taosMsleep(SYNC_WAIT_AFTER_CHOOSE_MASTER);
 
       syncResetFlowCtrl(pNode);
-      (*pNode->notifyRole)(pNode->vgId, nodeRole);
+      (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
     } else {
       pPeer = pNode->peerInfo[index];
       sInfo("%s, it shall work as master", pPeer->id);
@@ -730,7 +733,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
         nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
         sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
        }
-      (*pNode->notifyRole)(pNode->vgId, nodeRole);
+      (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
     }
   } else {
     for (int32_t index = 0; index < pNode->replica; ++index) {
@@ -742,7 +745,7 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
         if (masterIndex == pNode->selfIndex) {
           sError("%s, peer is master, work as slave instead", pTemp->id);
           nodeRole = TAOS_SYNC_ROLE_SLAVE;
-          (*pNode->notifyRole)(pNode->vgId, nodeRole);
+          (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
         }
       }
     }
@@ -759,7 +762,7 @@ static int32_t syncValidateMaster(SSyncPeer *pPeer) {
   if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) {
     sDebug("%s, peer has higher sver:%" PRIu64 ", restart all peer connections", pPeer->id, pPeer->version);
     nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-    (*pNode->notifyRole)(pNode->vgId, nodeRole);
+    (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
     code = -1;
 
     for (int32_t index = 0; index < pNode->replica; ++index) {
@@ -796,7 +799,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new
       } else {
         sInfo("%s, is master, work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion);
         nodeRole = TAOS_SYNC_ROLE_SLAVE;
-        (*pNode->notifyRole)(pNode->vgId, nodeRole);
+        (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
       }
     } else if (nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) {
       sDebug("%s, is master, continue work as slave, self sver:%" PRIu64, pMaster->id, nodeVersion);
@@ -989,7 +992,7 @@ static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer) {
 
   if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
     // nodeVersion = pHead->version;
-    (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
+    (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
   } else {
     if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
       syncSaveIntoBuffer(pPeer, pHead);

src/sync/src/syncMsg.c

@@ -102,9 +102,9 @@ void syncBuildFileAck(SFileAck *pMsg, int32_t vgId) {
   syncBuildHead(&pMsg->head);
 }
 
-void syncBuildFileInfo(SFileInfo *pMsg, int32_t vgId) {
+void syncBuildFileVersion(SFileVersion *pMsg, int32_t vgId) {
   pMsg->head.type = TAOS_SMSG_SYNC_FILE;
   pMsg->head.vgId = vgId;
-  pMsg->head.len = sizeof(SFileInfo) - sizeof(SSyncHead);
+  pMsg->head.len = sizeof(SFileVersion) - sizeof(SSyncHead);
   syncBuildHead(&pMsg->head);
 }
\ No newline at end of file

src/sync/src/syncRestore.c

@@ -25,139 +25,44 @@
 #include "tsync.h"
 #include "syncInt.h"
 
-static void syncRemoveExtraFile(SSyncPeer *pPeer, int32_t sindex, int32_t eindex) {
-  char       name[TSDB_FILENAME_LEN * 2] = {0};
-  char       fname[TSDB_FILENAME_LEN * 3] = {0};
-  uint32_t   magic;
-  uint64_t   fversion;
-  int64_t    size;
-  uint32_t   index = sindex;
+static int32_t syncRecvFileVersion(SSyncPeer *pPeer, uint64_t *fversion) {
   SSyncNode *pNode = pPeer->pSyncNode;
 
-  if (sindex < 0 || eindex < sindex) return;
-
-  sDebug("%s, extra files will be removed between sindex:%d and eindex:%d", pPeer->id, sindex, eindex);
-
-  while (1) {
-    name[0] = 0;
-    magic = (*pNode->getFileInfo)(pNode->vgId, name, &index, eindex, &size, &fversion);
-    if (magic == 0) break;
-
-    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name);
-    (void)remove(fname);
-    sInfo("%s, %s is removed for its extra", pPeer->id, fname);
+  SFileVersion fileVersion;
+  memset(&fileVersion, 0, sizeof(SFileVersion));
+  int32_t ret = taosReadMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion));
+  if (ret != sizeof(SFileVersion)) {
+    sError("%s, failed to read fver since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
 
-    index++;
-    if (index > eindex) break;
+  SFileAck fileVersionAck;
+  memset(&fileVersionAck, 0, sizeof(SFileAck));
+  syncBuildFileAck(&fileVersionAck, pNode->vgId);
+  ret = taosWriteMsg(pPeer->syncFd, &fileVersionAck, sizeof(SFileAck));
+  if (ret != sizeof(SFileAck)) {
+    sError("%s, failed to write fver ack since %s", pPeer->id, strerror(errno));
+    return -1;
   }
+
+  *fversion = htobe64(fileVersion.fversion);
+  return 0;
 }
 
 static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  SFileInfo  minfo; memset(&minfo, 0, sizeof(SFileInfo)); /* = {0}; */
-  SFileInfo  sinfo; memset(&sinfo, 0, sizeof(SFileInfo)); /* = {0}; */
-  SFileAck   fileAck; memset(&fileAck, 0, sizeof(SFileAck));
-  int32_t    code = -1;
-  char       name[TSDB_FILENAME_LEN * 2] = {0};
-  uint32_t   pindex = 0;  // index in last restore
-  bool       fileChanged = false;
-
-  *fversion = 0;
-  sinfo.index = -1;
-  while (1) {
-    // read file info
-    minfo.index = -1;
-    int32_t ret = taosReadMsg(pPeer->syncFd, &minfo, sizeof(SFileInfo));
-    if (ret != sizeof(SFileInfo) || minfo.index == -1) {
-      sError("%s, failed to read fileinfo while restore file since %s", pPeer->id, strerror(errno));
-      break;
-    }
-
-    assert(ret == sizeof(SFileInfo));
-    ret = syncCheckHead((SSyncHead *)(&minfo));
-    if (ret != 0) {
-      sError("%s, failed to check fileinfo while restore file since %s", pPeer->id, strerror(ret));
-      break;
-    }
-
-    // if no more file from master, break;
-    if (minfo.name[0] == 0 || minfo.magic == 0) {
-      sDebug("%s, no more files to restore", pPeer->id);
 
-      // remove extra files after the current index
-      if (sinfo.index != -1) syncRemoveExtraFile(pPeer, sinfo.index + 1, TAOS_SYNC_MAX_INDEX);
-      code = 0;
-      break;
-    }
-
-    sDebug("%s, file:%s info is received from master, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id,
-           minfo.name, minfo.index, minfo.size, minfo.fversion, minfo.magic);
-
-    // remove extra files on slave between the current and last index
-    syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1);
-    pindex = minfo.index;
-
-    // check the file info
-    sinfo = minfo;
-    sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion);
-    sDebug("%s, local file:%s info, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id, sinfo.name,
-           sinfo.index, sinfo.size, sinfo.fversion, sinfo.magic);
-
-    // if file not there or magic is not the same, file shall be synced
-    memset(&fileAck, 0, sizeof(SFileAck));
-    syncBuildFileAck(&fileAck, pNode->vgId);
-    fileAck.sync = (sinfo.magic != minfo.magic || sinfo.size != minfo.size || sinfo.name[0] == 0) ? 1 : 0;
-
-    // send file ack
-    ret = taosWriteMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
-    if (ret != sizeof(SFileAck)) {
-      sError("%s, failed to write file:%s ack while restore file since %s", pPeer->id, minfo.name, strerror(errno));
-      break;
-    }
-
-    // if sync is not required, continue
-    if (fileAck.sync == 0) {
-      sDebug("%s, %s is the same", pPeer->id, minfo.name);
-      continue;
-    } else {
-      sDebug("%s, %s will be received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
-    }
-
-    // if sync is required, open file, receive from master, and write to file
-    // get the full path to file
-    minfo.name[sizeof(minfo.name) - 1] = 0;
-    snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name);
-
-    int32_t dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, S_IRWXU | S_IRWXG | S_IRWXO);
-    if (dfd < 0) {
-      sError("%s, failed to open file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno));
-      break;
-    }
-
-    ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size);
-    fsync(dfd);
-    close(dfd);
-    if (ret < 0) {
-      sError("%s, failed to copy file:%s while restore file since %s", pPeer->id, minfo.name, strerror(errno));
-      break;
-    }
-
-    fileChanged = true;
-    sDebug("%s, %s is received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
-  }
-
-  if (code == 0 && fileChanged) {
-    // data file is changed, code shall be set to 1
-    *fversion = minfo.fversion;
-    code = 1;
-    sDebug("%s, file changed after restore file, fver:%" PRIu64, pPeer->id, *fversion);
+  if (pNode->recvFileFp && (*pNode->recvFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) {
+    sError("%s, failed to restore file", pPeer->id);
+    return -1;
   }
 
-  if (code < 0) {
-    sError("%s, failed to restore %s since %s", pPeer->id, name, strerror(errno));
+  if (syncRecvFileVersion(pPeer, fversion) < 0) {
+    return -1;
   }
 
-  return code;
+  sInfo("%s, all files are restored, fver:%" PRIu64, pPeer->id, *fversion);
+  return 0;
 }
 
 static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) {
@@ -195,7 +100,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) {
     }
     lastVer = pHead->version;
 
-    ret = (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL);
+    ret = (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_WAL, NULL);
     if (ret != 0) {
       sError("%s, failed to restore record since %s, hver:%" PRIu64, pPeer->id, tstrerror(ret), pHead->version);
       break;
@@ -215,7 +120,7 @@ static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset) {
   SSyncNode *pNode = pPeer->pSyncNode;
   SWalHead * pHead = (SWalHead *)offset;
 
-  (*pNode->writeToCache)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
+  (*pNode->writeToCacheFp)(pNode->vgId, pHead, TAOS_QTYPE_FWD, NULL);
   offset += pHead->len + sizeof(SWalHead);
 
   return offset;
@@ -315,20 +220,16 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
   sDebug("%s, send sync rsp to peer, tranId:%u", pPeer->id, rsp.tranId);
 
   sInfo("%s, start to restore file, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
+  (*pNode->startSyncFileFp)(pNode->vgId);
+
   int32_t code = syncRestoreFile(pPeer, &fversion);
   if (code < 0) {
-    sError("%s, failed to restore file", pPeer->id);
+    (*pNode->stopSyncFileFp)(pNode->vgId, fversion);
+    sError("%s, failed to restore files", pPeer->id);
     return -1;
   }
 
-  // if code > 0, data file is changed, notify app, and pass the version
-  if (code > 0 && pNode->notifyFileSynced) {
-    if ((*pNode->notifyFileSynced)(pNode->vgId, fversion) < 0) {
-      sError("%s, app not in ready state", pPeer->id);
-      return -1;
-    }
-  }
-
+  (*pNode->stopSyncFileFp)(pNode->vgId, fversion);
   nodeVersion = fversion;
 
   sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion);
@@ -368,7 +269,7 @@ void *syncRestoreData(void *param) {
   atomic_add_fetch_32(&tsSyncNum, 1);
   sInfo("%s, start to restore data, sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
 
-  (*pNode->notifyRole)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING);
+  (*pNode->notifyRoleFp)(pNode->vgId, TAOS_SYNC_ROLE_SYNCING);
 
   if (syncOpenRecvBuffer(pNode) < 0) {
     sError("%s, failed to allocate recv buffer, restart connection", pPeer->id);
@@ -385,7 +286,7 @@ void *syncRestoreData(void *param) {
     }
   }
 
-  (*pNode->notifyRole)(pNode->vgId, nodeRole);
+  (*pNode->notifyRoleFp)(pNode->vgId, nodeRole);
 
   nodeSStatus = TAOS_SYNC_STATUS_INIT;
   sInfo("%s, restore data over, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);

src/sync/src/syncRetrieve.c

@@ -27,7 +27,7 @@
 
 static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
     return -1;
@@ -39,7 +39,7 @@ static int32_t syncGetWalVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
 
 static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while retrieve, last wver:%" PRIu64, pPeer->id, pPeer->lastWalVer);
     return true;
@@ -55,7 +55,7 @@ static bool syncIsWalModified(SSyncNode *pNode, SSyncPeer *pPeer) {
 
 static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while get fver for retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
     return -1;
@@ -67,7 +67,7 @@ static int32_t syncGetFileVersion(SSyncNode *pNode, SSyncPeer *pPeer) {
 
 static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) {
   uint64_t fver, wver;
-  int32_t  code = (*pNode->getVersion)(pNode->vgId, &fver, &wver);
+  int32_t  code = (*pNode->getVersionFp)(pNode->vgId, &fver, &wver);
   if (code != 0) {
     sDebug("%s, vnode is commiting while retrieve, last fver:%" PRIu64, pPeer->id, pPeer->lastFileVer);
     pPeer->fileChanged = 1;
@@ -84,104 +84,54 @@ static bool syncAreFilesModified(SSyncNode *pNode, SSyncPeer *pPeer) {
   return false;
 }
 
-static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
+static int32_t syncSendFileVersion(SSyncPeer *pPeer) {
   SSyncNode *pNode = pPeer->pSyncNode;
-  SFileInfo  fileInfo; memset(&fileInfo, 0, sizeof(SFileInfo));
-  SFileAck   fileAck; memset(&fileAck, 0, sizeof(SFileAck));
-  int32_t    code = -1;
-  char       name[TSDB_FILENAME_LEN * 2] = {0};
 
-  if (syncGetFileVersion(pNode, pPeer) < 0) {
-    pPeer->fileChanged = 1;
+  SFileVersion fileVersion;
+  memset(&fileVersion, 0, sizeof(SFileVersion));
+  syncBuildFileVersion(&fileVersion, pNode->vgId);
+
+  uint64_t fver = pPeer->lastFileVer;
+  fileVersion.fversion = htobe64(fver);
+  int32_t ret = taosWriteMsg(pPeer->syncFd, &fileVersion, sizeof(SFileVersion));
+  if (ret != sizeof(SFileVersion)) {
+    sError("%s, failed to write fver:%" PRIu64 " since %s", pPeer->id, fver, strerror(errno));
     return -1;
   }
 
-  while (1) {
-    // retrieve file info
-    fileInfo.name[0] = 0;
-    fileInfo.size = 0;
-    fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
-                                           &fileInfo.size, &fileInfo.fversion);
-    syncBuildFileInfo(&fileInfo, pNode->vgId);
-    sDebug("%s, file:%s info is sent, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%u", pPeer->id, fileInfo.name,
-           fileInfo.index, fileInfo.size, fileInfo.fversion, fileInfo.magic);
-
-    // send the file info
-    int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(SFileInfo));
-    if (ret != sizeof(SFileInfo)) {
-      code = -1;
-      sError("%s, failed to write file:%s info while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
-
-    // if no file anymore, break
-    if (fileInfo.magic == 0 || fileInfo.name[0] == 0) {
-      code = 0;
-      sDebug("%s, no more files to sync", pPeer->id);
-      break;
-    }
-
-    // wait for the ack from peer
-    ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
-    if (ret != sizeof(SFileAck)) {
-      code = -1;
-      sError("%s, failed to read file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
-
-    ret = syncCheckHead((SSyncHead*)(&fileAck));
-    if (ret != 0) {
-      code = -1;
-      sError("%s, failed to check file:%s ack while retrieve file since %s", pPeer->id, fileInfo.name, strerror(ret));
-      break;
-    }
-
-    // set the peer sync version
-    pPeer->sversion = fileInfo.fversion;
-
-    // if sync is not required, continue
-    if (fileAck.sync == 0) {
-      fileInfo.index++;
-      sDebug("%s, %s is the same, fver:%" PRIu64, pPeer->id, fileInfo.name, fileInfo.fversion);
-      continue;
-    } else {
-      sDebug("%s, %s will be sent, fver:%" PRIu64, pPeer->id, fileInfo.name, fileInfo.fversion);
-    }
+  SFileAck fileAck;
+  memset(&fileAck, 0, sizeof(SFileAck));
+  ret = taosReadMsg(pPeer->syncFd, &fileAck, sizeof(SFileAck));
+  if (ret != sizeof(SFileAck)) {
+    sError("%s, failed to read fver ack since %s", pPeer->id, strerror(errno));
+    return -1;
+  }
 
-    // get the full path to file
-    snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
+  // set the peer sync version
+  pPeer->sversion = fver;
 
-    // send the file to peer
-    int32_t sfd = open(name, O_RDONLY | O_BINARY);
-    if (sfd < 0) {
-      code = -1;
-      sError("%s, failed to open file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
+  return 0;
+}
 
-    ret = (int32_t)taosSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size);
-    close(sfd);
-    if (ret < 0) {
-      code = -1;
-      sError("%s, failed to send file:%s while retrieve file since %s", pPeer->id, fileInfo.name, strerror(errno));
-      break;
-    }
+static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
+  SSyncNode *pNode = pPeer->pSyncNode;
 
-    sDebug("%s, file:%s is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
-    fileInfo.index++;
+  if (syncGetFileVersion(pNode, pPeer) < 0) {
+    pPeer->fileChanged = 1;
+    return -1;
+  }
 
-    // check if processed files are modified
-    if (syncAreFilesModified(pNode, pPeer)) {
-      code = -1;
-      break;
-    }
+  if (pNode->sendFileFp && (*pNode->sendFileFp)(pNode->pTsdb, pPeer->syncFd) != 0) {
+    sError("%s, failed to retrieve file", pPeer->id);
+    return -1;
   }
 
-  if (code != TSDB_CODE_SUCCESS) {
-    sError("%s, failed to retrieve file, code:0x%x", pPeer->id, code);
+  if (syncSendFileVersion(pPeer) < 0) {
+    return -1;
   }
 
-  return code;
+  sInfo("%s, all files are retrieved", pPeer->id);
+  return 0;
 }
 
 // if only a partial record is read out, upper layer will reload the file to get a complete record
@@ -345,7 +295,7 @@ static int32_t syncRetrieveWal(SSyncPeer *pPeer) {
   while (1) {
     // retrieve wal info
     wname[0] = 0;
-    code = (*pNode->getWalInfo)(pNode->vgId, wname, &index);
+    code = (*pNode->getWalInfoFp)(pNode->vgId, wname, &index);
     if (code < 0) {
       sError("%s, failed to get wal info since:%s, code:0x%x", pPeer->id, strerror(errno), code);
       break;
@@ -477,7 +427,7 @@ void *syncRetrieveData(void *param) {
   sInfo("%s, start to retrieve data, sstatus:%s, numOfRetrieves:%d", pPeer->id, syncStatus[pPeer->sstatus],
         pPeer->numOfRetrieves);
 
-  if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, pPeer->numOfRetrieves);
+  if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, pPeer->numOfRetrieves);
 
   pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
   if (pPeer->syncFd < 0) {
@@ -497,10 +447,10 @@ void *syncRetrieveData(void *param) {
     pPeer->numOfRetrieves++;
   } else {
     pPeer->numOfRetrieves = 0;
-    // if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
+    // if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
   }
 
-  if (pNode->notifyFlowCtrl) (*pNode->notifyFlowCtrl)(pNode->vgId, 0);
+  if (pNode->notifyFlowCtrlFp) (*pNode->notifyFlowCtrlFp)(pNode->vgId, 0);
 
   pPeer->fileChanged = 0;
   taosCloseSocket(pPeer->syncFd);

src/sync/test/syncServer.c

@@ -296,11 +296,10 @@ void initSync() {
   pCfg->replica = 1;
   pCfg->quorum = 1;
   syncInfo.vgId = 1;
-  syncInfo.getFileInfo = getFileInfo;
-  syncInfo.getWalInfo = getWalInfo;
-  syncInfo.writeToCache = writeToCache;
+  syncInfo.getWalInfoFp = getWalInfo;
+  syncInfo.writeToCacheFp = writeToCache;
   syncInfo.confirmForward = confirmForward;
-  syncInfo.notifyRole = notifyRole;
+  syncInfo.notifyRoleFp = notifyRole;
 
   pCfg->nodeInfo[0].nodeId = 1;
   pCfg->nodeInfo[0].nodePort = 7010;

src/tfs/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(inc)
+AUX_SOURCE_DIRECTORY(src SRC)
+ADD_LIBRARY(tfs ${SRC})
+TARGET_LINK_LIBRARIES(tfs tutil)
+
+IF (TD_LINUX)
+  # Someone has no gtest directory, so comment it
+  # ADD_SUBDIRECTORY(tests)
+ENDIF ()

src/tfs/inc/tfsint.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TD_TFSINT_H
+#define TD_TFSINT_H
+
+#include "tlog.h"
+#include "tglobal.h"
+#include "tfs.h"
+#include "tcoding.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern int fsDebugFlag;
+
+// For debug purpose
+#define fFatal(...) { if (fsDebugFlag & DEBUG_FATAL) { taosPrintLog("TFS FATAL ", 255, __VA_ARGS__); }}
+#define fError(...) { if (fsDebugFlag & DEBUG_ERROR) { taosPrintLog("TFS ERROR ", 255, __VA_ARGS__); }}
+#define fWarn(...)  { if (fsDebugFlag & DEBUG_WARN)  { taosPrintLog("TFS WARN ", 255, __VA_ARGS__); }}
+#define fInfo(...)  { if (fsDebugFlag & DEBUG_INFO)  { taosPrintLog("TFS ", 255, __VA_ARGS__); }}
+#define fDebug(...) { if (fsDebugFlag & DEBUG_DEBUG) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }}
+#define fTrace(...) { if (fsDebugFlag & DEBUG_TRACE) { taosPrintLog("TFS ", cqDebugFlag, __VA_ARGS__); }}
+
+// Global Definitions
+#define TFS_MIN_DISK_FREE_SIZE 50 * 1024 * 1024
+
+// tdisk.c ======================================================
+typedef struct {
+  int64_t size;
+  int64_t free;
+} SDiskMeta;
+
+typedef struct SDisk {
+  int       level;
+  int       id;
+  char      dir[TSDB_FILENAME_LEN];
+  SDiskMeta dmeta;
+} SDisk;
+
+#define DISK_LEVEL(pd) ((pd)->level)
+#define DISK_ID(pd) ((pd)->id)
+#define DISK_DIR(pd) ((pd)->dir)
+#define DISK_META(pd) ((pd)->dmeta)
+#define DISK_SIZE(pd) ((pd)->dmeta.size)
+#define DISK_FREE_SIZE(pd) ((pd)->dmeta.free)
+
+SDisk *tfsNewDisk(int level, int id, const char *dir);
+SDisk *tfsFreeDisk(SDisk *pDisk);
+int    tfsUpdateDiskInfo(SDisk *pDisk);
+
+// ttier.c ======================================================
+typedef struct {
+  int64_t size;
+  int64_t free;
+  int16_t nAvailDisks;  // # of Available disks
+} STierMeta;
+typedef struct STier {
+  pthread_spinlock_t lock;
+  int                level;
+  int16_t            ndisk;   // # of disks mounted to this tier
+  int16_t            nextid;  // next disk id to allocate
+  STierMeta          tmeta;
+  SDisk *            disks[TSDB_MAX_DISKS_PER_TIER];
+} STier;
+
+#define TIER_LEVEL(pt) ((pt)->level)
+#define TIER_NDISKS(pt) ((pt)->ndisk)
+#define TIER_SIZE(pt) ((pt)->tmeta.size)
+#define TIER_FREE_SIZE(pt) ((pt)->tmeta.free)
+#define TIER_AVAIL_DISKS(pt) ((pt)->tmeta.nAvailDisks)
+#define DISK_AT_TIER(pt, id) ((pt)->disks[id])
+
+int    tfsInitTier(STier *pTier, int level);
+void   tfsDestroyTier(STier *pTier);
+SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg);
+void   tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta);
+int    tfsAllocDiskOnTier(STier *pTier);
+void   tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta);
+void   tfsPosNextId(STier *pTier);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
\ No newline at end of file

src/tfs/src/tdisk.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "os.h"
+
+#include "taoserror.h"
+#include "tfsint.h"
+
+// PROTECTED ====================================
+SDisk *tfsNewDisk(int level, int id, const char *dir) {
+  SDisk *pDisk = (SDisk *)calloc(1, sizeof(*pDisk));
+  if (pDisk == NULL) {
+    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  pDisk->level = level;
+  pDisk->id = id;
+  strncpy(pDisk->dir, dir, TSDB_FILENAME_LEN);
+
+  return pDisk;
+}
+
+SDisk *tfsFreeDisk(SDisk *pDisk) {
+  if (pDisk) {
+    free(pDisk);
+  }
+  return NULL;
+}
+
+int tfsUpdateDiskInfo(SDisk *pDisk) {
+  ASSERT(pDisk != NULL);
+
+  SysDiskSize diskSize = {0};
+
+  int code = taosGetDiskSize(pDisk->dir, &diskSize);
+  if (code != 0) {
+    fError("failed to update disk information at level %d id %d dir %s since %s", pDisk->level, pDisk->id, pDisk->dir,
+           strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+  }
+
+  pDisk->dmeta.size = diskSize.tsize;
+  pDisk->dmeta.free = diskSize.tsize - diskSize.avail;
+
+  return code;
+}
\ No newline at end of file

src/tfs/src/tfs.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+
+#include "hash.h"
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tfs.h"
+#include "tfsint.h"
+
+#define TMPNAME_LEN (TSDB_FILENAME_LEN * 2 + 32)
+
+typedef struct {
+  pthread_spinlock_t lock;
+  SFSMeta            meta;
+  int                nlevel;
+  STier              tiers[TSDB_MAX_TIERS];
+  SHashObj *         map;  // name to did map
+} SFS;
+
+typedef struct {
+  SDisk *pDisk;
+} SDiskIter;
+
+#define TFS_META() (pfs->meta)
+#define TFS_NLEVEL() (pfs->nlevel)
+#define TFS_TIERS() (pfs->tiers)
+#define TFS_TIER_AT(level) (TFS_TIERS() + (level))
+#define TFS_DISK_AT(level, id) DISK_AT_TIER(TFS_TIER_AT(level), id)
+#define TFS_PRIMARY_DISK() TFS_DISK_AT(TFS_PRIMARY_LEVEL, TFS_PRIMARY_ID)
+#define TFS_IS_VALID_LEVEL(level) (((level) >= 0) && ((level) < TFS_NLEVEL()))
+#define TFS_IS_VALID_ID(level, id) (((id) >= 0) && ((id) < TIER_NDISKS(TFS_TIER_AT(level))))
+#define TFS_IS_VALID_DISK(level, id) (TFS_IS_VALID_LEVEL(level) && TFS_IS_VALID_ID(level, id))
+
+#define tfsLock() pthread_spin_lock(&(pfs->lock))
+#define tfsUnLock() pthread_spin_unlock(&(pfs->lock))
+
+static SFS  tfs = {0};
+static SFS *pfs = &tfs;
+
+// STATIC DECLARATION
+static int    tfsMount(SDiskCfg *pCfg);
+static int    tfsCheck();
+static int    tfsCheckAndFormatCfg(SDiskCfg *pCfg);
+static int    tfsFormatDir(char *idir, char *odir);
+static SDisk *tfsGetDiskByID(SDiskID did);
+static SDisk *tfsGetDiskByName(const char *dir);
+static int    tfsOpendirImpl(TDIR *tdir);
+static void   tfsInitDiskIter(SDiskIter *pIter);
+static SDisk *tfsNextDisk(SDiskIter *pIter);
+
+// FS APIs ====================================
+int tfsInit(SDiskCfg *pDiskCfg, int ndisk) {
+  ASSERT(ndisk > 0);
+
+  for (int level = 0; level < TSDB_MAX_TIERS; level++) {
+    if (tfsInitTier(TFS_TIER_AT(level), level) < 0) {
+      while (true) {
+        level--;
+        if (level < 0) break;
+
+        tfsDestroyTier(TFS_TIER_AT(level));
+      }
+
+      return -1;
+    }
+  }
+
+  pthread_spin_init(&(pfs->lock), 0);
+
+  pfs->map = taosHashInit(TSDB_MAX_TIERS * TSDB_MAX_DISKS_PER_TIER * 2,
+                          taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK);
+  if (pfs->map == NULL) {
+    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    tfsDestroy();
+    return -1;
+  }
+
+  for (int idisk = 0; idisk < ndisk; idisk++) {
+    if (tfsMount(pDiskCfg + idisk) < 0) {
+      tfsDestroy();
+      return -1;
+    }
+  }
+
+  if (tfsCheck() < 0) {
+    tfsDestroy();
+    return -1;
+  }
+
+  tfsUpdateInfo(NULL);
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    tfsPosNextId(TFS_TIER_AT(level));
+  }
+
+  return 0;
+}
+
+void tfsDestroy() {
+  taosHashCleanup(pfs->map);
+  pfs->map = NULL;
+
+  pthread_spin_destroy(&(pfs->lock));
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    tfsDestroyTier(TFS_TIER_AT(level));
+  }
+}
+
+void tfsUpdateInfo(SFSMeta *pFSMeta) {
+  SFSMeta   fsMeta;
+  STierMeta tierMeta;
+
+  if (pFSMeta == NULL) {
+    pFSMeta = &fsMeta;
+  }
+
+  memset(pFSMeta, 0, sizeof(*pFSMeta));
+
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    tfsUpdateTierInfo(pTier, &tierMeta);
+    pFSMeta->tsize += tierMeta.size;
+    pFSMeta->avail += tierMeta.free;
+  }
+
+  tfsLock();
+  pfs->meta = *pFSMeta;
+  tfsUnLock();
+}
+
+void tfsGetMeta(SFSMeta *pMeta) {
+  ASSERT(pMeta);
+
+  tfsLock();
+  *pMeta = pfs->meta;
+  tfsUnLock();
+}
+
+/* Allocate an existing available tier level
+ */
+void tfsAllocDisk(int expLevel, int *level, int *id) {
+  ASSERT(expLevel >= 0);
+
+  *level = expLevel;
+  *id = TFS_UNDECIDED_ID;
+
+  if (*level >= TFS_NLEVEL()) {
+    *level = TFS_NLEVEL() - 1;
+  }
+
+  while (*level >= 0) {
+    *id = tfsAllocDiskOnTier(TFS_TIER_AT(*level));
+    if (*id == TFS_UNDECIDED_ID) {
+      (*level)--;
+      continue;
+    }
+
+    return;
+  }
+
+  *level = TFS_UNDECIDED_LEVEL;
+  *id = TFS_UNDECIDED_ID;
+}
+
+const char *TFS_PRIMARY_PATH() { return DISK_DIR(TFS_PRIMARY_DISK()); }
+const char *TFS_DISK_PATH(int level, int id) { return DISK_DIR(TFS_DISK_AT(level, id)); }
+
+// TFILE APIs ====================================
+void tfsInitFile(TFILE *pf, int level, int id, const char *bname) {
+  ASSERT(TFS_IS_VALID_DISK(level, id));
+
+  SDisk *pDisk = TFS_DISK_AT(level, id);
+
+  pf->level = level;
+  pf->id = id;
+  strncpy(pf->rname, bname, TSDB_FILENAME_LEN);
+
+  char tmpName[TMPNAME_LEN] = {0};
+  snprintf(tmpName, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), bname);
+  tstrncpy(pf->aname, tmpName, TSDB_FILENAME_LEN);
+}
+
+bool tfsIsSameFile(const TFILE *pf1, const TFILE *pf2) {
+  ASSERT(pf1 != NULL || pf2 != NULL);
+  if (pf1 == NULL || pf2 == NULL) return false;
+  if (pf1->level != pf2->level) return false;
+  if (pf1->id != pf2->id) return false;
+  if (strncmp(pf1->rname, pf2->rname, TSDB_FILENAME_LEN) != 0) return false;
+  return true;
+}
+
+int tfsEncodeFile(void **buf, TFILE *pf) {
+  int tlen = 0;
+
+  tlen += taosEncodeVariantI32(buf, pf->level);
+  tlen += taosEncodeVariantI32(buf, pf->id);
+  tlen += taosEncodeString(buf, pf->rname);
+
+  return tlen;
+}
+
+void *tfsDecodeFile(void *buf, TFILE *pf) {
+  int32_t level, id;
+  char *  rname;
+
+  buf = taosDecodeVariantI32(buf, &(level));
+  buf = taosDecodeVariantI32(buf, &(id));
+  buf = taosDecodeString(buf, &rname);
+
+  tfsInitFile(pf, level, id, rname);
+
+  tfree(rname);
+  return buf;
+}
+
+void tfsbasename(const TFILE *pf, char *dest) {
+  char tname[TSDB_FILENAME_LEN] = "\0";
+
+  strncpy(tname, pf->aname, TSDB_FILENAME_LEN);
+  strncpy(dest, basename(tname), TSDB_FILENAME_LEN);
+}
+
+void tfsdirname(const TFILE *pf, char *dest) {
+  char tname[TSDB_FILENAME_LEN] = "\0";
+
+  strncpy(tname, pf->aname, TSDB_FILENAME_LEN);
+  strncpy(dest, dirname(tname), TSDB_FILENAME_LEN);
+}
+
+// DIR APIs ====================================
+int tfsMkdirAt(const char *rname, int level, int id) {
+  SDisk *pDisk = TFS_DISK_AT(level, id);
+  char   aname[TMPNAME_LEN];
+
+  snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname);
+  if (taosMkDir(aname, 0755) != 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+int tfsMkdirRecurAt(const char *rname, int level, int id) {
+  if (tfsMkdirAt(rname, level, id) < 0) {
+    if (errno == ENOENT) {
+      // Try to create upper
+      char *s = strdup(rname);
+
+      if (tfsMkdirRecurAt(dirname(s), level, id) < 0) {
+        tfree(s);
+        return -1;
+      }
+      tfree(s);
+
+      if (tfsMkdirAt(rname, level, id) < 0) {
+        return -1;
+      }
+    } else {
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+int tfsMkdir(const char *rname) {
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+      if (tfsMkdirAt(rname, level, id) < 0) {
+        return -1;
+      }
+    }
+  }
+
+  return 0;
+}
+
+int tfsRmdir(const char *rname) {
+  char aname[TMPNAME_LEN] = "\0";
+
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+      SDisk *pDisk = DISK_AT_TIER(pTier, id);
+
+      snprintf(aname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), rname);
+
+      taosRemoveDir(aname);
+    }
+  }
+
+  return 0;
+}
+
+int tfsRename(char *orname, char *nrname) {
+  char oaname[TMPNAME_LEN] = "\0";
+  char naname[TMPNAME_LEN] = "\0";
+
+  for (int level = 0; level < pfs->nlevel; level++) {
+    STier *pTier = TFS_TIER_AT(level);
+    for (int id = 0; id < TIER_NDISKS(pTier); id++) {
+      SDisk *pDisk = DISK_AT_TIER(pTier, id);
+
+      snprintf(oaname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), orname);
+      snprintf(naname, TMPNAME_LEN, "%s/%s", DISK_DIR(pDisk), nrname);
+
+      taosRename(oaname, naname);
+    }
+  }
+
+  return 0;
+}
+
+struct TDIR {
+  SDiskIter iter;
+  int       level;
+  int       id;
+  char      dirname[TSDB_FILENAME_LEN];
+  TFILE     tfile;
+  DIR *     dir;
+};
+
+TDIR *tfsOpendir(const char *rname) {
+  TDIR *tdir = (TDIR *)calloc(1, sizeof(*tdir));
+  if (tdir == NULL) {
+    terrno = TSDB_CODE_FS_OUT_OF_MEMORY;
+    return NULL;
+  }
+
+  tfsInitDiskIter(&(tdir->iter));
+  strncpy(tdir->dirname, rname, TSDB_FILENAME_LEN);
+
+  if (tfsOpendirImpl(tdir) < 0) {
+    free(tdir);
+    return NULL;
+  }
+
+  return tdir;
+}
+
+const TFILE *tfsReaddir(TDIR *tdir) {
+  if (tdir == NULL || tdir->dir == NULL) return NULL;
+  char bname[TMPNAME_LEN * 2] = "\0";
+
+  while (true) {
+    struct dirent *dp = NULL;
+    dp = readdir(tdir->dir);
+    if (dp != NULL) {
+      // Skip . and ..
+      if (strcmp(dp->d_name, ".") == 0 || strcmp(dp->d_name, "..") == 0) continue;
+
+      snprintf(bname, TMPNAME_LEN * 2, "%s/%s", tdir->dirname, dp->d_name);
+      tfsInitFile(&(tdir->tfile), tdir->level, tdir->id, bname);
+      return &(tdir->tfile);
+    }
+
+    if (tfsOpendirImpl(tdir) < 0) {
+      return NULL;
+    }
+
+    if (tdir->dir == NULL) {
+      terrno = TSDB_CODE_SUCCESS;
+      return NULL;
+    }
+  }
+}
+
+void tfsClosedir(TDIR *tdir) {
+  if (tdir) {
+    if (tdir->dir != NULL) {
+      closedir(tdir->dir);
+      tdir->dir = NULL;
+    }
+    free(tdir);
+  }
+}
+
+// private
+static int tfsMount(SDiskCfg *pCfg) {
+  SDiskID did;
+  SDisk * pDisk = NULL;
+
+  if (tfsCheckAndFormatCfg(pCfg) < 0) return -1;
+
+  did.level = pCfg->level;
+  pDisk = tfsMountDiskToTier(TFS_TIER_AT(did.level), pCfg);
+  if (pDisk == NULL) {
+    fError("failed to mount disk %s to level %d since %s", pCfg->dir, pCfg->level, tstrerror(terrno));
+    return -1;
+  }
+  did.id = DISK_ID(pDisk);
+
+  taosHashPut(pfs->map, (void *)(pCfg->dir), strnlen(pCfg->dir, TSDB_FILENAME_LEN), (void *)(&did), sizeof(did));
+  if (pfs->nlevel < pCfg->level + 1) pfs->nlevel = pCfg->level + 1;
+
+  return 0;
+}
+
+static int tfsCheckAndFormatCfg(SDiskCfg *pCfg) {
+  char        dirName[TSDB_FILENAME_LEN] = "\0";
+  struct stat pstat;
+
+  if (pCfg->level < 0 || pCfg->level >= TSDB_MAX_TIERS) {
+    fError("failed to mount %s to FS since invalid level %d", pCfg->dir, pCfg->level);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (pCfg->primary) {
+    if (pCfg->level != 0) {
+      fError("failed to mount %s to FS since disk is primary but level %d not 0", pCfg->dir, pCfg->level);
+      terrno = TSDB_CODE_FS_INVLD_CFG;
+      return -1;
+    }
+
+    if (TFS_PRIMARY_DISK() != NULL) {
+      fError("failed to mount %s to FS since duplicate primary mount", pCfg->dir);
+      terrno = TSDB_CODE_FS_DUP_PRIMARY;
+      return -1;
+    }
+  }
+
+  if (tfsFormatDir(pCfg->dir, dirName) < 0) {
+    fError("failed to mount %s to FS since invalid dir format", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (tfsGetDiskByName(dirName) != NULL) {
+    fError("failed to mount %s to FS since duplicate mount", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (access(dirName, W_OK | R_OK | F_OK) != 0) {
+    fError("failed to mount %s to FS since no R/W access rights", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  if (stat(dirName, &pstat) < 0) {
+    fError("failed to mount %s to FS since %s", pCfg->dir, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  if (!S_ISDIR(pstat.st_mode)) {
+    fError("failed to mount %s to FS since not a directory", pCfg->dir);
+    terrno = TSDB_CODE_FS_INVLD_CFG;
+    return -1;
+  }
+
+  strncpy(pCfg->dir, dirName, TSDB_FILENAME_LEN);
+
+  return 0;
+}
+
+static int tfsFormatDir(char *idir, char *odir) {
+  wordexp_t wep = {0};
+
+  int code = wordexp(idir, &wep, 0);
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+
+  if (realpath(wep.we_wordv[0], odir) == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    wordfree(&wep);
+    return -1;
+  }
+
+  wordfree(&wep);
+  return 0;
+
+}
+
+static int tfsCheck() {
+  if (TFS_PRIMARY_DISK() == NULL) {
+    fError("no primary disk is set");
+    terrno = TSDB_CODE_FS_NO_PRIMARY_DISK;
+    return -1;
+  }
+
+  for (int level = 0; level < TFS_NLEVEL(); level++) {
+    if (TIER_NDISKS(TFS_TIER_AT(level)) == 0) {
+      fError("no disk at level %d", level);
+      terrno = TSDB_CODE_FS_NO_MOUNT_AT_TIER;
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static SDisk *tfsGetDiskByID(SDiskID did) { return TFS_DISK_AT(did.level, did.id); }
+static SDisk *tfsGetDiskByName(const char *dir) {
+  SDiskID did;
+  SDisk * pDisk = NULL;
+  void *  pr = NULL;
+
+  pr = taosHashGet(pfs->map, (void *)dir, strnlen(dir, TSDB_FILENAME_LEN));
+  if (pr == NULL) return NULL;
+
+  did = *(SDiskID *)pr;
+  pDisk = tfsGetDiskByID(did);
+  ASSERT(pDisk != NULL);
+
+  return pDisk;
+}
+
+static int tfsOpendirImpl(TDIR *tdir) {
+  SDisk *pDisk = NULL;
+  char   adir[TMPNAME_LEN * 2] = "\0";
+
+  if (tdir->dir != NULL) {
+    closedir(tdir->dir);
+    tdir->dir = NULL;
+  }
+
+  while (true) {
+    pDisk = tfsNextDisk(&(tdir->iter));
+    if (pDisk == NULL) return 0;
+
+    tdir->level = DISK_LEVEL(pDisk);
+    tdir->id = DISK_ID(pDisk);
+
+    snprintf(adir, TMPNAME_LEN * 2, "%s/%s", DISK_DIR(pDisk), tdir->dirname);
+    tdir->dir = opendir(adir);
+    if (tdir->dir != NULL) break;
+  }
+
+  return 0;
+}
+
+static void tfsInitDiskIter(SDiskIter *pIter) { pIter->pDisk = TFS_DISK_AT(0, 0); }
+
+static SDisk *tfsNextDisk(SDiskIter *pIter) {
+  SDisk *pDisk = pIter->pDisk;
+
+  if (pDisk == NULL) return NULL;
+
+  int level = DISK_LEVEL(pDisk);
+  int id = DISK_ID(pDisk);
+
+  id++;
+  if (id < TIER_NDISKS(TFS_TIER_AT(level))) {
+    pIter->pDisk = TFS_DISK_AT(level, id);
+    ASSERT(pIter->pDisk != NULL);
+  } else {
+    level++;
+    id = 0;
+    if (level < TFS_NLEVEL()) {
+      pIter->pDisk = TFS_DISK_AT(level, id);
+      ASSERT(pIter->pDisk != NULL);
+    } else {
+      pIter->pDisk = NULL;
+    }
+  }
+
+  return pDisk;
+}
+
+// OTHER FUNCTIONS ===================================
+void taosGetDisk() {
+  const double unit = 1024 * 1024 * 1024;
+  SysDiskSize  diskSize;
+  SFSMeta      fsMeta;
+
+  if (tscEmbedded) {
+    tfsUpdateInfo(&fsMeta);
+    tsTotalDataDirGB = (float)(fsMeta.tsize / unit);
+    tsAvailDataDirGB = (float)(fsMeta.avail / unit);
+  }
+
+  if (taosGetDiskSize(tsLogDir, &diskSize) == 0) {
+    tsTotalLogDirGB = (float)(diskSize.tsize / unit);
+    tsAvailLogDirGB = (float)(diskSize.avail / unit);
+  }
+
+  if (taosGetDiskSize("/tmp", &diskSize) == 0) {
+    tsTotalTmpDirGB = (float)(diskSize.tsize / unit);
+    tsAvailTmpDirectorySpace = (float)(diskSize.avail / unit);
+  }
+}

src/tfs/src/ttier.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+#include "os.h"
+
+#include "taosdef.h"
+#include "taoserror.h"
+#include "tfsint.h"
+
+#define tfsLockTier(pTier) pthread_spin_lock(&((pTier)->lock))
+#define tfsUnLockTier(pTier) pthread_spin_unlock(&((pTier)->lock))
+
+// PROTECTED ==========================================
+int tfsInitTier(STier *pTier, int level) {
+  memset((void *)pTier, 0, sizeof(*pTier));
+
+  int code = pthread_spin_init(&(pTier->lock), 0);
+  if (code) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+
+  pTier->level = level;
+
+  return 0;
+}
+
+void tfsDestroyTier(STier *pTier) {
+  for (int id = 0; id < TSDB_MAX_DISKS_PER_TIER; id++) {
+    DISK_AT_TIER(pTier, id) = tfsFreeDisk(DISK_AT_TIER(pTier, id));
+  }
+
+  pTier->ndisk = 0;
+
+  pthread_spin_destroy(&(pTier->lock));
+}
+
+SDisk *tfsMountDiskToTier(STier *pTier, SDiskCfg *pCfg) {
+  ASSERT(pTier->level == pCfg->level);
+
+  int    id = 0;
+  SDisk *pDisk;
+
+  if (TIER_NDISKS(pTier) >= TSDB_MAX_DISKS_PER_TIER) {
+    terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
+    return NULL;
+  }
+
+  if (pTier->level == 0) {
+    if (DISK_AT_TIER(pTier, 0) != NULL) {
+      id = pTier->ndisk;
+    } else {
+      if (pCfg->primary) {
+        id = 0;
+      } else {
+        id = pTier->ndisk + 1;
+      }
+      if (id >= TSDB_MAX_DISKS_PER_TIER) {
+        terrno = TSDB_CODE_FS_TOO_MANY_MOUNT;
+        return NULL;
+      }
+    }
+  } else {
+    id = pTier->ndisk;
+  }
+
+  pDisk = tfsNewDisk(pCfg->level, id, pCfg->dir);
+  if (pDisk == NULL) return NULL;
+  DISK_AT_TIER(pTier, id) = pDisk;
+  pTier->ndisk++;
+
+  fInfo("disk %s is mounted to tier level %d id %d", pCfg->dir, pCfg->level, id);
+
+  return DISK_AT_TIER(pTier, id);
+}
+
+void tfsUpdateTierInfo(STier *pTier, STierMeta *pTierMeta) {
+  STierMeta tmeta;
+
+  if (pTierMeta == NULL) {
+    pTierMeta = &tmeta;
+  }
+  memset(pTierMeta, 0, sizeof(*pTierMeta));
+
+  tfsLockTier(pTier);
+
+  for (int id = 0; id < pTier->ndisk; id++) {
+    if (tfsUpdateDiskInfo(DISK_AT_TIER(pTier, id)) < 0) {
+      continue;
+    }
+    pTierMeta->size += DISK_SIZE(DISK_AT_TIER(pTier, id));
+    pTierMeta->free += DISK_FREE_SIZE(DISK_AT_TIER(pTier, id));
+    pTierMeta->nAvailDisks++;
+  }
+
+  pTier->tmeta = *pTierMeta;
+
+  tfsUnLockTier(pTier);
+}
+
+// Round-Robin to allocate disk on a tier
+int tfsAllocDiskOnTier(STier *pTier) {
+  ASSERT(pTier->ndisk > 0);
+  int    id = TFS_UNDECIDED_ID;
+  SDisk *pDisk;
+
+  tfsLockTier(pTier);
+
+  if (TIER_AVAIL_DISKS(pTier) <= 0) {
+    tfsUnLockTier(pTier);
+    return id;
+  }
+
+  id = pTier->nextid;
+  while (true) {
+    pDisk = DISK_AT_TIER(pTier, id);
+    ASSERT(pDisk != NULL);
+
+    if (DISK_FREE_SIZE(pDisk) < TFS_MIN_DISK_FREE_SIZE) {
+      id = (id + 1) % pTier->ndisk;
+      if (id == pTier->nextid) {
+        tfsUnLockTier(pTier);
+        return TFS_UNDECIDED_ID;
+      } else {
+        continue;
+      }
+    } else {
+      pTier->nextid = (id + 1) % pTier->ndisk;
+      break;
+    }
+  }
+
+  tfsUnLockTier(pTier);
+  return id;
+}
+
+void tfsGetTierMeta(STier *pTier, STierMeta *pTierMeta) {
+  ASSERT(pTierMeta != NULL);
+
+  tfsLockTier(pTier);
+  *pTierMeta = pTier->tmeta;
+  tfsUnLockTier(pTier);
+}
+
+void tfsPosNextId(STier *pTier) {
+  ASSERT(pTier->ndisk > 0);
+  int nextid = 0;
+
+  for (int id = 1; id < pTier->ndisk; id++) {
+    SDisk *pLDisk = DISK_AT_TIER(pTier, nextid);
+    SDisk *pDisk = DISK_AT_TIER(pTier, id);
+    if (DISK_FREE_SIZE(pDisk) > TFS_MIN_DISK_FREE_SIZE && DISK_FREE_SIZE(pDisk) > DISK_FREE_SIZE(pLDisk)) {
+      nextid = id;
+    }
+  }
+
+  pTier->nextid = nextid;
+}
\ No newline at end of file

src/tsdb/CMakeLists.txt

@@ -4,7 +4,7 @@ PROJECT(TDengine)
 INCLUDE_DIRECTORIES(inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 ADD_LIBRARY(tsdb ${SRC})
-TARGET_LINK_LIBRARIES(tsdb common tutil)
+TARGET_LINK_LIBRARIES(tsdb tfs common tutil)
 
 IF (TD_LINUX)
   # Someone has no gtest directory, so comment it

src/util/inc/tkvstore.h → src/tsdb/inc/tsdbBuffer.h

@@ -12,56 +12,32 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#ifndef _TD_KVSTORE_H_
-#define _TD_KVSTORE_H_
 
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdint.h>
-
-#define KVSTORE_FILE_VERSION ((uint32_t)0)
-
-typedef int (*iterFunc)(void *, void *cont, int contLen);
-typedef void (*afterFunc)(void *);
+#ifndef _TD_TSDB_BUFFER_H_
+#define _TD_TSDB_BUFFER_H_
 
 typedef struct {
-  int64_t  size;  // including 512 bytes of header size
-  int64_t  tombSize;
-  int64_t  nRecords;
-  int64_t  nDels;
-  uint32_t magic;
-} SStoreInfo;
+  int64_t blockId;
+  int     offset;
+  int     remain;
+  char    data[];
+} STsdbBufBlock;
 
 typedef struct {
-  char *     fname;
-  int        fd;
-  char *     fsnap;
-  int        sfd;
-  char *     fnew;
-  int        nfd;
-  SHashObj * map;
-  iterFunc   iFunc;
-  afterFunc  aFunc;
-  void *     appH;
-  SStoreInfo info;
-} SKVStore;
-
-#define KVSTORE_MAGIC(s) (s)->info.magic
-
-int       tdCreateKVStore(char *fname);
-int       tdDestroyKVStore(char *fname);
-SKVStore *tdOpenKVStore(char *fname, iterFunc iFunc, afterFunc aFunc, void *appH);
+  pthread_cond_t poolNotEmpty;
+  int            bufBlockSize;
+  int            tBufBlocks;
+  int            nBufBlocks;
+  int64_t        index;
+  SList*         bufBlockList;
+} STsdbBufPool;
+
+#define TSDB_BUFFER_RESERVE 1024  // Reseve 1K as commit threshold
+
+STsdbBufPool* tsdbNewBufPool();
+void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
+int           tsdbOpenBufPool(STsdbRepo* pRepo);
+void          tsdbCloseBufPool(STsdbRepo* pRepo);
+SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
+
+#endif /* _TD_TSDB_BUFFER_H_ */
\ No newline at end of file
-void      tdCloseKVStore(SKVStore *pStore);
-int       tdKVStoreStartCommit(SKVStore *pStore);
-int       tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLen);
-int       tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid);
-int       tdKVStoreEndCommit(SKVStore *pStore);
-void      tsdbGetStoreInfo(char *fname, uint32_t *magic, int64_t *size);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif
\ No newline at end of file

src/dnode/inc/dnodeMgmt.h → src/tsdb/inc/tsdbCommit.h

@@ -13,35 +13,37 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_DNODE_MGMT_H
-#define TDENGINE_DNODE_MGMT_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include "trpc.h"
-
-int32_t dnodeInitMgmt();
-void    dnodeCleanupMgmt();
-int32_t dnodeInitMgmtTimer();
-void    dnodeCleanupMgmtTimer();
-void    dnodeDispatchToMgmtQueue(SRpcMsg *rpcMsg);
-
-void*   dnodeGetVnode(int32_t vgId);
-int32_t dnodeGetVnodeStatus(void *pVnode);
-void*   dnodeGetVnodeRworker(void *pVnode);
-void*   dnodeGetVnodeWworker(void *pVnode);
-void*   dnodeGetVnodeWal(void *pVnode);
-void*   dnodeGetVnodeTsdb(void *pVnode);
-void    dnodeReleaseVnode(void *pVnode);
-
-void    dnodeSendRedirectMsg(SRpcMsg *rpcMsg, bool forShell);
-void    dnodeGetEpSetForPeer(SRpcEpSet *epSet);
-void    dnodeGetEpSetForShell(SRpcEpSet *epSet);
-
-#ifdef __cplusplus
+#ifndef _TD_TSDB_COMMIT_H_
+#define _TD_TSDB_COMMIT_H_
+
+typedef struct {
+  int   minFid;
+  int   midFid;
+  int   maxFid;
+  TSKEY minKey;
+} SRtn;
+
+typedef struct {
+  uint64_t uid;
+  int64_t  offset;
+  int64_t  size;
+} SKVRecord;
+
+void  tsdbGetRtnSnap(STsdbRepo *pRepo, SRtn *pRtn);
+int   tsdbEncodeKVRecord(void **buf, SKVRecord *pRecord);
+void *tsdbDecodeKVRecord(void *buf, SKVRecord *pRecord);
+void *tsdbCommitData(STsdbRepo *pRepo);
+
+static FORCE_INLINE int tsdbGetFidLevel(int fid, SRtn *pRtn) {
+  if (fid >= pRtn->maxFid) {
+    return 0;
+  } else if (fid >= pRtn->midFid) {
+    return 1;
+  } else if (fid >= pRtn->minFid) {
+    return 2;
+  } else {
+    return -1;
+  }
 }
-#endif
 
-#endif
+#endif /* _TD_TSDB_COMMIT_H_ */
\ No newline at end of file

src/common/inc/tsystem.h → src/tsdb/inc/tsdbCommitQueue.h

@@ -13,26 +13,9 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TSYSTEM_H
-#define TDENGINE_TSYSTEM_H
+#ifndef _TD_TSDB_COMMIT_QUEUE_H_
+#define _TD_TSDB_COMMIT_QUEUE_H_
 
-#ifdef __cplusplus
-extern "C" {
-#endif
+int tsdbScheduleCommit(STsdbRepo *pRepo);
 
-bool taosGetSysMemory(float *memoryUsedMB);
-bool taosGetProcMemory(float *memoryUsedMB);
+#endif /* _TD_TSDB_COMMIT_QUEUE_H_ */
\ No newline at end of file
-bool taosGetDisk();
-bool taosGetCpuUsage(float *sysCpuUsage, float *procCpuUsage);
-bool taosGetBandSpeed(float *bandSpeedKb);
-bool taosGetProcIO(float *readKB, float *writeKB);
-void taosGetSystemInfo();
-void taosPrintOsInfo();
-void taosKillSystem();
-void taosSetCoreDump();
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/tsdb/inc/tsdbFS.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_FS_H_
+#define _TD_TSDB_FS_H_
+
+#define TSDB_FS_VERSION 0
+
+// ================== CURRENT file header info
+typedef struct {
+  uint32_t version;  // Current file system version (relating to code)
+  uint32_t len;      // Encode content length (including checksum)
+} SFSHeader;
+
+// ================== TSDB File System Meta
+typedef struct {
+  uint32_t version;       // Commit version from 0 to increase
+  int64_t  totalPoints;   // total points
+  int64_t  totalStorage;  // Uncompressed total storage
+} STsdbFSMeta;
+
+// ==================
+typedef struct {
+  STsdbFSMeta meta;  // FS meta
+  SMFile*     pmf;   // meta file pointer
+  SMFile      mf;    // meta file
+  SArray*     df;    // data file array
+} SFSStatus;
+
+typedef struct {
+  pthread_rwlock_t lock;
+
+  SFSStatus* cstatus;    // current status
+  SHashObj*  metaCache;  // meta cache
+  bool       intxn;
+  SFSStatus* nstatus;  // new status
+} STsdbFS;
+
+#define FS_CURRENT_STATUS(pfs) ((pfs)->cstatus)
+#define FS_NEW_STATUS(pfs) ((pfs)->nstatus)
+#define FS_IN_TXN(pfs) (pfs)->intxn
+#define FS_VERSION(pfs) ((pfs)->cstatus->meta.version)
+#define FS_TXN_VERSION(pfs) ((pfs)->nstatus->meta.version)
+
+typedef struct {
+  int        direction;
+  uint64_t   version;  // current FS version
+  STsdbFS*   pfs;
+  int        index;  // used to position next fset when version the same
+  int        fid;    // used to seek when version is changed
+  SDFileSet* pSet;
+} SFSIter;
+
+#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC
+#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC
+
+STsdbFS *tsdbNewFS(STsdbCfg *pCfg);
+void *   tsdbFreeFS(STsdbFS *pfs);
+int      tsdbOpenFS(STsdbRepo *pRepo);
+void     tsdbCloseFS(STsdbRepo *pRepo);
+void     tsdbStartFSTxn(STsdbRepo *pRepo, int64_t pointsAdd, int64_t storageAdd);
+int      tsdbEndFSTxn(STsdbRepo *pRepo);
+int      tsdbEndFSTxnWithError(STsdbFS *pfs);
+void     tsdbUpdateFSTxnMeta(STsdbFS *pfs, STsdbFSMeta *pMeta);
+void     tsdbUpdateMFile(STsdbFS *pfs, const SMFile *pMFile);
+int      tsdbUpdateDFileSet(STsdbFS *pfs, const SDFileSet *pSet);
+
+void       tsdbFSIterInit(SFSIter *pIter, STsdbFS *pfs, int direction);
+void       tsdbFSIterSeek(SFSIter *pIter, int fid);
+SDFileSet *tsdbFSIterNext(SFSIter *pIter);
+int        tsdbLoadMetaCache(STsdbRepo *pRepo, bool recoverMeta);
+
+static FORCE_INLINE int tsdbRLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_rdlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbWLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_wrlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+static FORCE_INLINE int tsdbUnLockFS(STsdbFS* pFs) {
+  int code = pthread_rwlock_unlock(&(pFs->lock));
+  if (code != 0) {
+    terrno = TAOS_SYSTEM_ERROR(code);
+    return -1;
+  }
+  return 0;
+}
+
+#endif /* _TD_TSDB_FS_H_ */
\ No newline at end of file

src/tsdb/inc/tsdbFile.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TS_TSDB_FILE_H_
+#define _TS_TSDB_FILE_H_
+
+#define TSDB_FILE_HEAD_SIZE 512
+#define TSDB_FILE_DELIMITER 0xF00AFA0F
+#define TSDB_FILE_INIT_MAGIC 0xFFFFFFFF
+#define TSDB_IVLD_FID INT_MIN
+
+#define TSDB_FILE_INFO(tf) (&((tf)->info))
+#define TSDB_FILE_F(tf) (&((tf)->f))
+#define TSDB_FILE_FD(tf) ((tf)->fd)
+#define TSDB_FILE_FULL_NAME(tf) TFILE_NAME(TSDB_FILE_F(tf))
+#define TSDB_FILE_OPENED(tf) (TSDB_FILE_FD(tf) >= 0)
+#define TSDB_FILE_CLOSED(tf) (!TSDB_FILE_OPENED(tf))
+#define TSDB_FILE_SET_CLOSED(f) (TSDB_FILE_FD(f) = -1)
+#define TSDB_FILE_LEVEL(tf) TFILE_LEVEL(TSDB_FILE_F(tf))
+#define TSDB_FILE_ID(tf) TFILE_ID(TSDB_FILE_F(tf))
+#define TSDB_FILE_FSYNC(tf) fsync(TSDB_FILE_FD(tf))
+
+typedef enum { TSDB_FILE_HEAD = 0, TSDB_FILE_DATA, TSDB_FILE_LAST, TSDB_FILE_MAX, TSDB_FILE_META } TSDB_FILE_T;
+
+// =============== SMFile
+typedef struct {
+  int64_t  size;
+  int64_t  tombSize;
+  int64_t  nRecords;
+  int64_t  nDels;
+  uint32_t magic;
+} SMFInfo;
+
+typedef struct {
+  SMFInfo info;
+  TFILE   f;
+  int     fd;
+} SMFile;
+
+void  tsdbInitMFile(SMFile* pMFile, SDiskID did, int vid, uint32_t ver);
+void  tsdbInitMFileEx(SMFile* pMFile, SMFile* pOMFile);
+int   tsdbEncodeSMFile(void** buf, SMFile* pMFile);
+void* tsdbDecodeSMFile(void* buf, SMFile* pMFile);
+int   tsdbEncodeSMFileEx(void** buf, SMFile* pMFile);
+void* tsdbDecodeSMFileEx(void* buf, SMFile* pMFile);
+int   tsdbApplyMFileChange(SMFile* from, SMFile* to);
+int   tsdbCreateMFile(SMFile* pMFile, bool updateHeader);
+int   tsdbUpdateMFileHeader(SMFile* pMFile);
+int   tsdbLoadMFileHeader(SMFile* pMFile, SMFInfo* pInfo);
+int   tsdbScanAndTryFixMFile(STsdbRepo* pRepo);
+int   tsdbEncodeMFInfo(void** buf, SMFInfo* pInfo);
+void* tsdbDecodeMFInfo(void* buf, SMFInfo* pInfo);
+
+static FORCE_INLINE void tsdbSetMFileInfo(SMFile* pMFile, SMFInfo* pInfo) { pMFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenMFile(SMFile* pMFile, int flags) {
+  ASSERT(TSDB_FILE_CLOSED(pMFile));
+
+  pMFile->fd = open(TSDB_FILE_FULL_NAME(pMFile), flags);
+  if (pMFile->fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseMFile(SMFile* pMFile) {
+  if (TSDB_FILE_OPENED(pMFile)) {
+    close(pMFile->fd);
+    TSDB_FILE_SET_CLOSED(pMFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekMFile(SMFile* pMFile, int64_t offset, int whence) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t loffset = taosLSeek(TSDB_FILE_FD(pMFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t nwrite = taosWrite(pMFile->fd, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateMFileMagic(SMFile* pMFile, void* pCksum) {
+  pMFile->info.magic = taosCalcChecksum(pMFile->info.magic, (uint8_t*)(pCksum), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendMFile(SMFile* pMFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekMFile(pMFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pMFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteMFile(pMFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pMFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveMFile(SMFile* pMFile) { return tfsremove(TSDB_FILE_F(pMFile)); }
+
+static FORCE_INLINE int64_t tsdbReadMFile(SMFile* pMFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pMFile));
+
+  int64_t nread = taosRead(pMFile->fd, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+// =============== SDFile
+typedef struct {
+  uint32_t magic;
+  uint32_t len;
+  uint32_t totalBlocks;
+  uint32_t totalSubBlocks;
+  uint32_t offset;
+  uint64_t size;
+  uint64_t tombSize;
+} SDFInfo;
+
+typedef struct {
+  SDFInfo info;
+  TFILE   f;
+  int     fd;
+} SDFile;
+
+void  tsdbInitDFile(SDFile* pDFile, SDiskID did, int vid, int fid, uint32_t ver, TSDB_FILE_T ftype);
+void  tsdbInitDFileEx(SDFile* pDFile, SDFile* pODFile);
+int   tsdbEncodeSDFile(void** buf, SDFile* pDFile);
+void* tsdbDecodeSDFile(void* buf, SDFile* pDFile);
+int   tsdbCreateDFile(SDFile* pDFile, bool updateHeader);
+int   tsdbUpdateDFileHeader(SDFile* pDFile);
+int   tsdbLoadDFileHeader(SDFile* pDFile, SDFInfo* pInfo);
+int   tsdbParseDFilename(const char* fname, int* vid, int* fid, TSDB_FILE_T* ftype, uint32_t* version);
+
+static FORCE_INLINE void tsdbSetDFileInfo(SDFile* pDFile, SDFInfo* pInfo) { pDFile->info = *pInfo; }
+
+static FORCE_INLINE int tsdbOpenDFile(SDFile* pDFile, int flags) {
+  ASSERT(!TSDB_FILE_OPENED(pDFile));
+
+  pDFile->fd = open(TSDB_FILE_FULL_NAME(pDFile), flags);
+  if (pDFile->fd < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbCloseDFile(SDFile* pDFile) {
+  if (TSDB_FILE_OPENED(pDFile)) {
+    close(pDFile->fd);
+    TSDB_FILE_SET_CLOSED(pDFile);
+  }
+}
+
+static FORCE_INLINE int64_t tsdbSeekDFile(SDFile* pDFile, int64_t offset, int whence) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t loffset = taosLSeek(TSDB_FILE_FD(pDFile), offset, whence);
+  if (loffset < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return loffset;
+}
+
+static FORCE_INLINE int64_t tsdbWriteDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nwrite = taosWrite(pDFile->fd, buf, nbyte);
+  if (nwrite < nbyte) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nwrite;
+}
+
+static FORCE_INLINE void tsdbUpdateDFileMagic(SDFile* pDFile, void* pCksm) {
+  pDFile->info.magic = taosCalcChecksum(pDFile->info.magic, (uint8_t*)(pCksm), sizeof(TSCKSUM));
+}
+
+static FORCE_INLINE int tsdbAppendDFile(SDFile* pDFile, void* buf, int64_t nbyte, int64_t* offset) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t toffset;
+
+  if ((toffset = tsdbSeekDFile(pDFile, 0, SEEK_END)) < 0) {
+    return -1;
+  }
+
+  ASSERT(pDFile->info.size == toffset);
+
+  if (offset) {
+    *offset = toffset;
+  }
+
+  if (tsdbWriteDFile(pDFile, buf, nbyte) < 0) {
+    return -1;
+  }
+
+  pDFile->info.size += nbyte;
+
+  return (int)nbyte;
+}
+
+static FORCE_INLINE int tsdbRemoveDFile(SDFile* pDFile) { return tfsremove(TSDB_FILE_F(pDFile)); }
+
+static FORCE_INLINE int64_t tsdbReadDFile(SDFile* pDFile, void* buf, int64_t nbyte) {
+  ASSERT(TSDB_FILE_OPENED(pDFile));
+
+  int64_t nread = taosRead(pDFile->fd, buf, nbyte);
+  if (nread < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  return nread;
+}
+
+static FORCE_INLINE int tsdbCopyDFile(SDFile* pSrc, SDFile* pDest) {
+  if (tfscopy(TSDB_FILE_F(pSrc), TSDB_FILE_F(pDest)) < 0) {
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return -1;
+  }
+
+  tsdbSetDFileInfo(pDest, TSDB_FILE_INFO(pSrc));
+  return 0;
+}
+
+// =============== SDFileSet
+typedef struct {
+  int    fid;
+  int    state;
+  SDFile files[TSDB_FILE_MAX];
+} SDFileSet;
+
+#define TSDB_FSET_FID(s) ((s)->fid)
+#define TSDB_DFILE_IN_SET(s, t) ((s)->files + (t))
+#define TSDB_FSET_LEVEL(s) TSDB_FILE_LEVEL(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_ID(s) TSDB_FILE_ID(TSDB_DFILE_IN_SET(s, 0))
+#define TSDB_FSET_SET_CLOSED(s)                                                \
+  do {                                                                         \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
+      TSDB_FILE_SET_CLOSED(TSDB_DFILE_IN_SET(s, ftype));                       \
+    }                                                                          \
+  } while (0);
+#define TSDB_FSET_FSYNC(s)                                                     \
+  do {                                                                         \
+    for (TSDB_FILE_T ftype = TSDB_FILE_HEAD; ftype < TSDB_FILE_MAX; ftype++) { \
+      TSDB_FILE_FSYNC(TSDB_DFILE_IN_SET(s, ftype));                            \
+    }                                                                          \
+  } while (0);
+
+void  tsdbInitDFileSet(SDFileSet* pSet, SDiskID did, int vid, int fid, uint32_t ver);
+void  tsdbInitDFileSetEx(SDFileSet* pSet, SDFileSet* pOSet);
+int   tsdbEncodeDFileSet(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSet(void* buf, SDFileSet* pSet);
+int   tsdbEncodeDFileSetEx(void** buf, SDFileSet* pSet);
+void* tsdbDecodeDFileSetEx(void* buf, SDFileSet* pSet);
+int   tsdbApplyDFileSetChange(SDFileSet* from, SDFileSet* to);
+int   tsdbCreateDFileSet(SDFileSet* pSet, bool updateHeader);
+int   tsdbUpdateDFileSetHeader(SDFileSet* pSet);
+int   tsdbScanAndTryFixDFileSet(STsdbRepo *pRepo, SDFileSet* pSet);
+
+static FORCE_INLINE void tsdbCloseDFileSet(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tsdbCloseDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbOpenDFileSet(SDFileSet* pSet, int flags) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbOpenDFile(TSDB_DFILE_IN_SET(pSet, ftype), flags) < 0) {
+      tsdbCloseDFileSet(pSet);
+      return -1;
+    }
+  }
+  return 0;
+}
+
+static FORCE_INLINE void tsdbRemoveDFileSet(SDFileSet* pSet) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    tsdbRemoveDFile(TSDB_DFILE_IN_SET(pSet, ftype));
+  }
+}
+
+static FORCE_INLINE int tsdbCopyDFileSet(SDFileSet* pSrc, SDFileSet* pDest) {
+  for (TSDB_FILE_T ftype = 0; ftype < TSDB_FILE_MAX; ftype++) {
+    if (tsdbCopyDFile(TSDB_DFILE_IN_SET(pSrc, ftype), TSDB_DFILE_IN_SET(pDest, ftype)) < 0) {
+      tsdbRemoveDFileSet(pDest);
+      return -1;
+    }
+  }
+
+  return 0;
+}
+
+static FORCE_INLINE void tsdbGetFidKeyRange(int days, int8_t precision, int fid, TSKEY* minKey, TSKEY* maxKey) {
+  *minKey = fid * days * tsMsPerDay[precision];
+  *maxKey = *minKey + days * tsMsPerDay[precision] - 1;
+}
+
+#endif /* _TS_TSDB_FILE_H_ */
\ No newline at end of file

src/tsdb/inc/tsdbLog.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef _TD_TSDB_LOG_H_
+#define _TD_TSDB_LOG_H_
+
+extern int32_t tsdbDebugFlag;
+
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+
+#endif /* _TD_TSDB_LOG_H_ */
\ No newline at end of file

src/tsdb/inc/tsdbMain.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-#ifndef _TD_TSDB_MAIN_H_
-#define _TD_TSDB_MAIN_H_
-
-#include "os.h"
-#include "hash.h"
-#include "tcoding.h"
-#include "tglobal.h"
-#include "tkvstore.h"
-#include "tlist.h"
-#include "tlog.h"
-#include "tlockfree.h"
-#include "tsdb.h"
-#include "tskiplist.h"
-#include "tutil.h"
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-extern int32_t tsdbDebugFlag;
-
-#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
-#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
-#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
-#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
-#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
-#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
-
-#define TSDB_MAX_TABLE_SCHEMAS 16
-#define TSDB_FILE_HEAD_SIZE    512
-#define TSDB_FILE_DELIMITER    0xF00AFA0F
-#define TSDB_FILE_INIT_MAGIC   0xFFFFFFFF
-
-#define TAOS_IN_RANGE(key, keyMin, keyLast) (((key) >= (keyMin)) && ((key) <= (keyMax)))
-
-// NOTE: Any file format change must increase this version number by 1
-//       Also, implement the convert function
-#define TSDB_FILE_VERSION ((uint32_t)0)
-
-// Definitions
-// ------------------ tsdbMeta.c
-typedef struct STable {
-  STableId       tableId;
-  ETableType     type;
-  tstr*          name;  // NOTE: there a flexible string here
-  uint64_t       suid;
-  struct STable* pSuper;  // super table pointer
-  uint8_t        numOfSchemas;
-  STSchema*      schema[TSDB_MAX_TABLE_SCHEMAS];
-  STSchema*      tagSchema;
-  SKVRow         tagVal;
-  SSkipList*     pIndex;         // For TSDB_SUPER_TABLE, it is the skiplist index
-  void*          eventHandler;   // TODO
-  void*          streamHandler;  // TODO
-  TSKEY          lastKey;
-  SDataRow       lastRow;
-  char*          sql;
-  void*          cqhandle;
-  SRWLatch       latch;  // TODO: implementa latch functions
-  T_REF_DECLARE()
-} STable;
-
-typedef struct {
-  pthread_rwlock_t rwLock;
-
-  int32_t   nTables;
-  int32_t   maxTables;
-  STable**  tables;
-  SList*    superList;
-  SHashObj* uidMap;
-  SKVStore* pStore;
-  int       maxRowBytes;
-  int       maxCols;
-} STsdbMeta;
-
-// ------------------ tsdbBuffer.c
-typedef struct {
-  int64_t blockId;
-  int     offset;
-  int     remain;
-  char    data[];
-} STsdbBufBlock;
-
-typedef struct {
-  pthread_cond_t poolNotEmpty;
-  int            bufBlockSize;
-  int            tBufBlocks;
-  int            nBufBlocks;
-  int64_t        index;
-  SList*         bufBlockList;
-} STsdbBufPool;
-
-// ------------------ tsdbMemTable.c
-typedef struct {
-  STable *           pTable;
-  SSkipListIterator *pIter;
-} SCommitIter;
-
-typedef struct {
-  uint64_t   uid;
-  TSKEY      keyFirst;
-  TSKEY      keyLast;
-  int64_t    numOfRows;
-  SSkipList* pData;
-} STableData;
-
-typedef struct {
-  T_REF_DECLARE()
-  SRWLatch     latch;
-  TSKEY        keyFirst;
-  TSKEY        keyLast;
-  int64_t      numOfRows;
-  int32_t      maxTables;
-  STableData** tData;
-  SList*       actList;
-  SList*       extraBuffList;
-  SList*       bufBlockList;
-} SMemTable;
-
-enum { TSDB_UPDATE_META, TSDB_DROP_META };
-
-#ifdef WINDOWS
-#pragma pack(push ,1) 
-typedef struct {
-#else
-typedef struct __attribute__((packed)){
-#endif
-  char     act;
-  uint64_t uid;
-} SActObj;
-#ifdef WINDOWS
-#pragma pack(pop) 
-#endif
-
-typedef struct {
-  int  len;
-  char cont[];
-} SActCont;
-
-// ------------------ tsdbFile.c
-extern const char* tsdbFileSuffix[];
-typedef enum {
-  TSDB_FILE_TYPE_HEAD = 0,
-  TSDB_FILE_TYPE_DATA,
-  TSDB_FILE_TYPE_LAST,
-  TSDB_FILE_TYPE_STAT,
-  TSDB_FILE_TYPE_NHEAD,
-  TSDB_FILE_TYPE_NDATA,
-  TSDB_FILE_TYPE_NLAST,
-  TSDB_FILE_TYPE_NSTAT
-} TSDB_FILE_TYPE;
-
-#ifndef TDINTERNAL
-#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_LAST+1)
-#else
-#define TSDB_FILE_TYPE_MAX (TSDB_FILE_TYPE_STAT+1)
-#endif
-
-typedef struct {
-  uint32_t magic;
-  uint32_t len;
-  uint32_t totalBlocks;
-  uint32_t totalSubBlocks;
-  uint32_t offset;
-  uint64_t size;      // total size of the file
-  uint64_t tombSize;  // unused file size
-} STsdbFileInfo;
-
-typedef struct {
-  char  fname[TSDB_FILENAME_LEN];
-  int   fd;
-
-  STsdbFileInfo info;
-} SFile;
-
-typedef struct {
-  int   fileId;
-  int   state; // 0 for health, 1 for problem
-  SFile files[TSDB_FILE_TYPE_MAX];
-} SFileGroup;
-
-typedef struct {
-  pthread_rwlock_t fhlock;
-
-  int         maxFGroups;
-  int         nFGroups;
-  SFileGroup* pFGroup;
-} STsdbFileH;
-
-typedef struct {
-  int         direction;
-  STsdbFileH* pFileH;
-  int         fileId;
-  int         index;
-} SFileGroupIter;
-
-// ------------------ tsdbMain.c
-typedef struct {
-  int32_t  totalLen;
-  int32_t  len;
-  SDataRow row;
-} SSubmitBlkIter;
-
-typedef struct {
-  int32_t totalLen;
-  int32_t len;
-  void *  pMsg;
-} SSubmitMsgIter;
-
-typedef struct {
-  int8_t state;
-
-  char*           rootDir;
-  STsdbCfg        config;
-  STsdbAppH       appH;
-  STsdbStat       stat;
-  STsdbMeta*      tsdbMeta;
-  STsdbBufPool*   pPool;
-  SMemTable*      mem;
-  SMemTable*      imem;
-  STsdbFileH*     tsdbFileH;
-  tsem_t          readyToCommit;
-  pthread_mutex_t mutex;
-  bool            repoLocked;
-  int32_t         code; // Commit code
-} STsdbRepo;
-
-// ------------------ tsdbRWHelper.c
-typedef struct {
-  int32_t  tid;
-  uint32_t len;
-  uint32_t offset;
-  uint32_t hasLast : 2;
-  uint32_t numOfBlocks : 30;
-  uint64_t uid;
-  TSKEY    maxKey;
-} SCompIdx;
-
-typedef struct {
-  int64_t last : 1;
-  int64_t offset : 63;
-  int32_t algorithm : 8;
-  int32_t numOfRows : 24;
-  int32_t len;
-  int32_t keyLen;     // key column length, keyOffset = offset+sizeof(SCompData)+sizeof(SCompCol)*numOfCols
-  int16_t numOfSubBlocks;
-  int16_t numOfCols; // not including timestamp column
-  TSKEY   keyFirst;
-  TSKEY   keyLast;
-} SCompBlock;
-
-typedef struct {
-  int32_t    delimiter;  // For recovery usage
-  int32_t    tid;
-  uint64_t   uid;
-  SCompBlock blocks[];
-} SCompInfo;
-
-typedef struct {
-  int16_t colId;
-  int32_t len;
-  int32_t type : 8;
-  int32_t offset : 24;
-  int64_t sum;
-  int64_t max;
-  int64_t min;
-  int16_t maxIndex;
-  int16_t minIndex;
-  int16_t numOfNull;
-  char    padding[2];
-} SCompCol;
-
-typedef struct {
-  int32_t  delimiter;  // For recovery usage
-  int32_t  numOfCols;  // For recovery usage
-  uint64_t uid;        // For recovery usage
-  SCompCol cols[];
-} SCompData;
-
-typedef enum { TSDB_WRITE_HELPER, TSDB_READ_HELPER } tsdb_rw_helper_t;
-
-typedef struct {
-  TSKEY      minKey;
-  TSKEY      maxKey;
-  SFileGroup fGroup;
-  SFile      nHeadF;
-  SFile      nLastF;
-} SHelperFile;
-
-typedef struct {
-  uint64_t uid;
-  int32_t  tid;
-} SHelperTable;
-
-typedef struct {
-  SCompIdx* pIdxArray;
-  int       numOfIdx;
-  int       curIdx;
-} SIdxH;
-
-typedef struct {
-  tsdb_rw_helper_t type;
-
-  STsdbRepo* pRepo;
-  int8_t     state;
-  // For file set usage
-  SHelperFile files;
-  SIdxH       idxH;
-  SCompIdx    curCompIdx;
-  void*       pWIdx;
-  // For table set usage
-  SHelperTable tableInfo;
-  SCompInfo*   pCompInfo;
-  bool         hasOldLastBlock;
-  // For block set usage
-  SCompData* pCompData;
-  SDataCols* pDataCols[2];
-  void*      pBuffer;     // Buffer to hold the whole data block
-  void*      compBuffer;  // Buffer for temperary compress/decompress purpose
-} SRWHelper;
-
-typedef struct {
-  int   rowsInserted;
-  int   rowsUpdated;
-  int   rowsDeleteSucceed;
-  int   rowsDeleteFailed;
-  int   nOperations;
-  TSKEY keyFirst;
-  TSKEY keyLast;
-} SMergeInfo;
-// ------------------ tsdbScan.c
-typedef struct {
-  SFileGroup fGroup;
-  int        numOfIdx;
-  SCompIdx*  pCompIdx;
-  SCompInfo* pCompInfo;
-  void*      pBuf;
-  FILE*      tLogStream;
-} STsdbScanHandle;
-
-// Operations
-// ------------------ tsdbMeta.c
-#define TSDB_INIT_NTABLES 1024
-#define TABLE_TYPE(t) (t)->type
-#define TABLE_NAME(t) (t)->name
-#define TABLE_CHAR_NAME(t) TABLE_NAME(t)->data
-#define TABLE_UID(t) (t)->tableId.uid
-#define TABLE_TID(t) (t)->tableId.tid
-#define TABLE_SUID(t) (t)->suid
-#define TSDB_META_FILE_MAGIC(m) KVSTORE_MAGIC((m)->pStore)
-#define TSDB_RLOCK_TABLE(t) taosRLockLatch(&((t)->latch))
-#define TSDB_RUNLOCK_TABLE(t) taosRUnLockLatch(&((t)->latch))
-#define TSDB_WLOCK_TABLE(t) taosWLockLatch(&((t)->latch))
-#define TSDB_WUNLOCK_TABLE(t) taosWUnLockLatch(&((t)->latch))
-
-STsdbMeta* tsdbNewMeta(STsdbCfg* pCfg);
-void       tsdbFreeMeta(STsdbMeta* pMeta);
-int        tsdbOpenMeta(STsdbRepo* pRepo);
-int        tsdbCloseMeta(STsdbRepo* pRepo);
-STable*    tsdbGetTableByUid(STsdbMeta* pMeta, uint64_t uid);
-STSchema*  tsdbGetTableSchemaByVersion(STable* pTable, int16_t version);
-int        tsdbWLockRepoMeta(STsdbRepo* pRepo);
-int        tsdbRLockRepoMeta(STsdbRepo* pRepo);
-int        tsdbUnlockRepoMeta(STsdbRepo* pRepo);
-void       tsdbRefTable(STable* pTable);
-void       tsdbUnRefTable(STable* pTable);
-void       tsdbUpdateTableSchema(STsdbRepo* pRepo, STable* pTable, STSchema* pSchema, bool insertAct);
-
-static FORCE_INLINE int tsdbCompareSchemaVersion(const void *key1, const void *key2) {
-  if (*(int16_t *)key1 < schemaVersion(*(STSchema **)key2)) {
-    return -1;
-  } else if (*(int16_t *)key1 > schemaVersion(*(STSchema **)key2)) {
-    return 1;
-  } else {
-    return 0;
-  }
-}
-
-static FORCE_INLINE STSchema* tsdbGetTableSchemaImpl(STable* pTable, bool lock, bool copy, int16_t version) {
-  STable*   pDTable = (TABLE_TYPE(pTable) == TSDB_CHILD_TABLE) ? pTable->pSuper : pTable;
-  STSchema* pSchema = NULL;
-  STSchema* pTSchema = NULL;
-
-  if (lock) TSDB_RLOCK_TABLE(pDTable);
-  if (version < 0) {  // get the latest version of schema
-    pTSchema = pDTable->schema[pDTable->numOfSchemas - 1];
-  } else {  // get the schema with version
-    void* ptr = taosbsearch(&version, pDTable->schema, pDTable->numOfSchemas, sizeof(STSchema*),
-                            tsdbCompareSchemaVersion, TD_EQ);
-    if (ptr == NULL) {
-      terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION;
-      goto _exit;
-    }
-    pTSchema = *(STSchema**)ptr;
-  }
-
-  ASSERT(pTSchema != NULL);
-
-  if (copy) {
-    if ((pSchema = tdDupSchema(pTSchema)) == NULL) terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-  } else {
-    pSchema = pTSchema;
-  }
-
-_exit:
-  if (lock) TSDB_RUNLOCK_TABLE(pDTable);
-  return pSchema;
-}
-
-static FORCE_INLINE STSchema* tsdbGetTableSchema(STable* pTable) {
-  return tsdbGetTableSchemaImpl(pTable, false, false, -1);
-}
-
-static FORCE_INLINE STSchema *tsdbGetTableTagSchema(STable *pTable) {
-  if (pTable->type == TSDB_CHILD_TABLE) {  // check child table first
-    STable *pSuper = pTable->pSuper;
-    if (pSuper == NULL) return NULL;
-    return pSuper->tagSchema;
-  } else if (pTable->type == TSDB_SUPER_TABLE) {
-    return pTable->tagSchema;
-  } else {
-    return NULL;
-  }
-}
-
-static FORCE_INLINE TSKEY tsdbGetTableLastKeyImpl(STable* pTable) {
-  ASSERT(pTable->lastRow == NULL || pTable->lastKey == dataRowKey(pTable->lastRow));
-  return pTable->lastKey;
-}
-
-// ------------------ tsdbBuffer.c
-#define TSDB_BUFFER_RESERVE 1024  // Reseve 1K as commit threshold
-
-STsdbBufPool* tsdbNewBufPool();
-void          tsdbFreeBufPool(STsdbBufPool* pBufPool);
-int           tsdbOpenBufPool(STsdbRepo* pRepo);
-void          tsdbCloseBufPool(STsdbRepo* pRepo);
-SListNode*    tsdbAllocBufBlockFromPool(STsdbRepo* pRepo);
-
-// ------------------ tsdbMemTable.c
-int   tsdbRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
-int   tsdbUnRefMemTable(STsdbRepo* pRepo, SMemTable* pMemTable);
-int   tsdbTakeMemSnapshot(STsdbRepo* pRepo, SMemTable** pMem, SMemTable** pIMem);
-void  tsdbUnTakeMemSnapShot(STsdbRepo* pRepo, SMemTable* pMem, SMemTable* pIMem);
-void* tsdbAllocBytes(STsdbRepo* pRepo, int bytes);
-int   tsdbAsyncCommit(STsdbRepo* pRepo);
-int   tsdbLoadDataFromCache(STable* pTable, SSkipListIterator* pIter, TSKEY maxKey, int maxRowsToRead, SDataCols* pCols,
-                            TKEY* filterKeys, int nFilterKeys, bool keepDup, SMergeInfo* pMergeInfo);
-void* tsdbCommitData(STsdbRepo* pRepo);
-
-static FORCE_INLINE SDataRow tsdbNextIterRow(SSkipListIterator* pIter) {
-  if (pIter == NULL) return NULL;
-
-  SSkipListNode* node = tSkipListIterGet(pIter);
-  if (node == NULL) return NULL;
-
-  return (SDataRow)SL_GET_NODE_DATA(node);
-}
-
-static FORCE_INLINE TSKEY tsdbNextIterKey(SSkipListIterator* pIter) {
-  SDataRow row = tsdbNextIterRow(pIter);
-  if (row == NULL) return TSDB_DATA_TIMESTAMP_NULL;
-
-  return dataRowKey(row);
-}
-
-static FORCE_INLINE TKEY tsdbNextIterTKey(SSkipListIterator* pIter) {
-  SDataRow row = tsdbNextIterRow(pIter);
-  if (row == NULL) return TKEY_NULL;
-
-  return dataRowTKey(row);
-}
-
-static FORCE_INLINE STsdbBufBlock* tsdbGetCurrBufBlock(STsdbRepo* pRepo) {
-  ASSERT(pRepo != NULL);
-  if (pRepo->mem == NULL) return NULL;
-
-  SListNode* pNode = listTail(pRepo->mem->bufBlockList);
-  if (pNode == NULL) return NULL;
-
-  STsdbBufBlock* pBufBlock = NULL;
-  tdListNodeGetData(pRepo->mem->bufBlockList, pNode, (void*)(&pBufBlock));
-
-  return pBufBlock;
-}
-
-// ------------------ tsdbFile.c
-#define TSDB_KEY_FILEID(key, daysPerFile, precision) ((key) / tsMsPerDay[(precision)] / (daysPerFile))
-#define TSDB_MAX_FILE(keep, daysPerFile) ((keep) / (daysPerFile) + 3)
-#define TSDB_MIN_FILE_ID(fh) (fh)->pFGroup[0].fileId
-#define TSDB_MAX_FILE_ID(fh) (fh)->pFGroup[(fh)->nFGroups - 1].fileId
-#define TSDB_IS_FILE_OPENED(f) ((f)->fd > 0)
-#define TSDB_FGROUP_ITER_FORWARD TSDB_ORDER_ASC
-#define TSDB_FGROUP_ITER_BACKWARD TSDB_ORDER_DESC
-
-STsdbFileH* tsdbNewFileH(STsdbCfg* pCfg);
-void        tsdbFreeFileH(STsdbFileH* pFileH);
-int         tsdbOpenFileH(STsdbRepo* pRepo);
-void        tsdbCloseFileH(STsdbRepo* pRepo);
-SFileGroup* tsdbCreateFGroupIfNeed(STsdbRepo* pRepo, char* dataDir, int fid);
-void        tsdbInitFileGroupIter(STsdbFileH* pFileH, SFileGroupIter* pIter, int direction);
-void        tsdbSeekFileGroupIter(SFileGroupIter* pIter, int fid);
-SFileGroup* tsdbGetFileGroupNext(SFileGroupIter* pIter);
-int         tsdbOpenFile(SFile* pFile, int oflag);
-void        tsdbCloseFile(SFile* pFile);
-int         tsdbCreateFile(SFile* pFile, STsdbRepo* pRepo, int fid, int type);
-SFileGroup* tsdbSearchFGroup(STsdbFileH* pFileH, int fid, int flags);
-void        tsdbFitRetention(STsdbRepo* pRepo);
-int         tsdbUpdateFileHeader(SFile* pFile);
-int         tsdbEncodeSFileInfo(void** buf, const STsdbFileInfo* pInfo);
-void*       tsdbDecodeSFileInfo(void* buf, STsdbFileInfo* pInfo);
-void        tsdbRemoveFileGroup(STsdbRepo* pRepo, SFileGroup* pFGroup);
-int         tsdbLoadFileHeader(SFile* pFile, uint32_t* version);
-void        tsdbGetFileInfoImpl(char* fname, uint32_t* magic, int64_t* size);
-void        tsdbGetFidKeyRange(int daysPerFile, int8_t precision, int fileId, TSKEY *minKey, TSKEY *maxKey);
-
-// ------------------ tsdbRWHelper.c
-#define TSDB_HELPER_CLEAR_STATE 0x0        // Clear state
-#define TSDB_HELPER_FILE_SET_AND_OPEN 0x1  // File is set
-#define TSDB_HELPER_IDX_LOAD 0x2           // SCompIdx part is loaded
-#define TSDB_HELPER_TABLE_SET 0x4          // Table is set
-#define TSDB_HELPER_INFO_LOAD 0x8          // SCompInfo part is loaded
-#define TSDB_HELPER_FILE_DATA_LOAD 0x10    // SCompData part is loaded
-#define helperSetState(h, s) (((h)->state) |= (s))
-#define helperClearState(h, s) ((h)->state &= (~(s)))
-#define helperHasState(h, s) ((((h)->state) & (s)) == (s))
-#define blockAtIdx(h, idx) ((h)->pCompInfo->blocks + idx)
-#define TSDB_MAX_SUBBLOCKS 8
-#define IS_SUB_BLOCK(pBlock) ((pBlock)->numOfSubBlocks == 0)
-#define helperType(h) (h)->type
-#define helperRepo(h) (h)->pRepo
-#define helperState(h) (h)->state
-#define TSDB_NLAST_FILE_OPENED(h) ((h)->files.nLastF.fd > 0)
-#define helperFileId(h) ((h)->files.fGroup.fileId)
-#define helperHeadF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_HEAD]))
-#define helperDataF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_DATA]))
-#define helperLastF(h) (&((h)->files.fGroup.files[TSDB_FILE_TYPE_LAST]))
-#define helperNewHeadF(h) (&((h)->files.nHeadF))
-#define helperNewLastF(h) (&((h)->files.nLastF))
-
-int  tsdbInitReadHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
-int  tsdbInitWriteHelper(SRWHelper* pHelper, STsdbRepo* pRepo);
-void tsdbDestroyHelper(SRWHelper* pHelper);
-void tsdbResetHelper(SRWHelper* pHelper);
-int  tsdbSetAndOpenHelperFile(SRWHelper* pHelper, SFileGroup* pGroup);
-int  tsdbCloseHelperFile(SRWHelper* pHelper, bool hasError, SFileGroup* pGroup);
-int  tsdbSetHelperTable(SRWHelper* pHelper, STable* pTable, STsdbRepo* pRepo);
-int  tsdbCommitTableData(SRWHelper* pHelper, SCommitIter* pCommitIter, SDataCols* pDataCols, TSKEY maxKey);
-int  tsdbMoveLastBlockIfNeccessary(SRWHelper* pHelper);
-int  tsdbWriteCompInfo(SRWHelper* pHelper);
-int  tsdbWriteCompIdx(SRWHelper* pHelper);
-int  tsdbLoadCompIdxImpl(SFile* pFile, uint32_t offset, uint32_t len, void* buffer);
-int  tsdbDecodeSCompIdxImpl(void* buffer, uint32_t len, SCompIdx** ppCompIdx, int* numOfIdx);
-int  tsdbLoadCompIdx(SRWHelper* pHelper, void* target);
-int  tsdbLoadCompInfoImpl(SFile* pFile, SCompIdx* pIdx, SCompInfo** ppCompInfo);
-int  tsdbLoadCompInfo(SRWHelper* pHelper, void* target);
-int  tsdbLoadCompData(SRWHelper* phelper, SCompBlock* pcompblock, void* target);
-void tsdbGetDataStatis(SRWHelper* pHelper, SDataStatis* pStatis, int numOfCols);
-int  tsdbLoadBlockDataCols(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo, int16_t* colIds,
-                           int numOfColIds);
-int  tsdbLoadBlockData(SRWHelper* pHelper, SCompBlock* pCompBlock, SCompInfo* pCompInfo);
-
-static FORCE_INLINE int compTSKEY(const void* key1, const void* key2) {
-  if (*(TSKEY*)key1 > *(TSKEY*)key2) {
-    return 1;
-  } else if (*(TSKEY*)key1 == *(TSKEY*)key2) {
-    return 0;
-  } else {
-    return -1;
-  }
-}
-
-// ------------------ tsdbMain.c
-#define REPO_ID(r) (r)->config.tsdbId
-#define IS_REPO_LOCKED(r) (r)->repoLocked
-#define TSDB_SUBMIT_MSG_HEAD_SIZE sizeof(SSubmitMsg)
-
-char*       tsdbGetMetaFileName(char* rootDir);
-void        tsdbGetDataFileName(char* rootDir, int vid, int fid, int type, char* fname);
-int         tsdbLockRepo(STsdbRepo* pRepo);
-int         tsdbUnlockRepo(STsdbRepo* pRepo);
-char*       tsdbGetDataDirName(char* rootDir);
-int         tsdbGetNextMaxTables(int tid);
-STsdbMeta*  tsdbGetMeta(TSDB_REPO_T* pRepo);
-STsdbFileH* tsdbGetFile(TSDB_REPO_T* pRepo);
-int         tsdbCheckCommit(STsdbRepo* pRepo);
-
-// ------------------ tsdbScan.c
-int              tsdbScanFGroup(STsdbScanHandle* pScanHandle, char* rootDir, int fid);
-STsdbScanHandle* tsdbNewScanHandle();
-void             tsdbSetScanLogStream(STsdbScanHandle* pScanHandle, FILE* fLogStream);
-int              tsdbSetAndOpenScanFile(STsdbScanHandle* pScanHandle, char* rootDir, int fid);
-int              tsdbScanSCompIdx(STsdbScanHandle* pScanHandle);
-int              tsdbScanSCompBlock(STsdbScanHandle* pScanHandle, int idx);
-int              tsdbCloseScanFile(STsdbScanHandle* pScanHandle);
-void             tsdbFreeScanHandle(STsdbScanHandle* pScanHandle);
-
-// ------------------ tsdbCommitQueue.c
-int tsdbScheduleCommit(STsdbRepo *pRepo);
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/tsdb/src/tsdbBuffer.c

@@ -13,8 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "tsdb.h"
-#include "tsdbMain.h"
+#include "tsdbint.h"
 
 #define POOL_IS_EMPTY(b) (listNEles((b)->bufBlockList) == 0)
 

src/tsdb/src/tsdbCommitQueue.c

@@ -13,11 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include "os.h"
-#include "tglobal.h"
-#include "tlist.h"
-#include "tref.h"
-#include "tsdbMain.h"
+#include "tsdbint.h"
 
 typedef struct {
   bool            stop;
@@ -35,7 +31,7 @@ typedef struct {
 
 static void *tsdbLoopCommit(void *arg);
 
-SCommitQueue tsCommitQueue = {0};
+static SCommitQueue tsCommitQueue = {0};
 
 int tsdbInitCommitQueue() {
   int nthreads = tsNumOfCommitThreads;

src/util/CMakeLists.txt

@@ -33,3 +33,7 @@ ELSEIF(TD_DARWIN)
   TARGET_LINK_LIBRARIES(tutil m)
   TARGET_LINK_LIBRARIES(tutil iconv)
 ENDIF()
+
+IF (TD_STORAGE)
+  TARGET_LINK_LIBRARIES(tutil storage)
+ENDIF ()
\ No newline at end of file