Merge branch 'develop' of https://github.com/taosdata/TDengine into develop

CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-SET(TD_SYNC FALSE)
 SET(TD_ACCOUNT FALSE)
 SET(TD_ADMIN FALSE)
 SET(TD_GRANT FALSE)

cmake/define.inc

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF (TD_SYNC)
-  ADD_DEFINITIONS(-D_SYNC)
-ENDIF ()
-
 IF (TD_ACCOUNT)
   ADD_DEFINITIONS(-D_ACCT)
 ENDIF ()

cmake/env.inc

@@ -9,7 +9,6 @@ SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/lib)
 SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/build/bin)
 SET(TD_TESTS_OUTPUT_DIR ${PROJECT_BINARY_DIR}/test)
 
-MESSAGE(STATUS "Operating system dependency directory: " ${TD_OS_DIR})
 MESSAGE(STATUS "Project source directory: " ${PROJECT_SOURCE_DIR})
 MESSAGE(STATUS "Project binary files output path: " ${PROJECT_BINARY_DIR})
 MESSAGE(STATUS "Project executable files output path: " ${EXECUTABLE_OUTPUT_PATH})

cmake/platform.inc

@@ -33,20 +33,23 @@ IF (${CPUTYPE} MATCHES "aarch32")
   SET(TD_PAGMODE_LITE TRUE)
   ADD_DEFINITIONS(-D_TD_ARM_)
   ADD_DEFINITIONS(-D_TD_ARM_32_)
+  MESSAGE(STATUS "input cpuType: aarch32")
 ELSEIF (${CPUTYPE} MATCHES "aarch64")
   SET(TD_ARM TRUE)
   SET(TD_ARM_64 TRUE)
   ADD_DEFINITIONS(-D_TD_ARM_)
   ADD_DEFINITIONS(-D_TD_ARM_64_)
+   MESSAGE(STATUS "input cpuType: aarch64")
 ELSEIF (${CPUTYPE} MATCHES "mips64")
   SET(TD_MIPS TRUE)
   SET(TD_MIPS_64 TRUE)
   ADD_DEFINITIONS(-D_TD_MIPS_)
   ADD_DEFINITIONS(-D_TD_MIPS_64_)
+  MESSAGE(STATUS "input cpuType: mips64")
 ELSEIF (${CPUTYPE} MATCHES "x64")
-  MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
+  MESSAGE(STATUS "input cpuType: x64")
 ELSEIF (${CPUTYPE} MATCHES "x86")
-  MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
+  MESSAGE(STATUS "input cpuType: x86")
 ELSE ()
   MESSAGE(STATUS "input cpuType: " ${CPUTYPE})
 ENDIF ()
@@ -61,13 +64,13 @@ MESSAGE(STATUS "The current os is " ${TD_OS_INFO})
 IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_LINUX_64 TRUE)
-    SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
     ADD_DEFINITIONS(-D_M_X64)
+    ADD_DEFINITIONS(-D_TD_LINUX_64)
     MESSAGE(STATUS "The current platform is Linux 64-bit")
   ELSEIF (${CMAKE_SIZEOF_VOID_P} MATCHES 4)
+    SET(TD_LINUX_32 TRUE)
+    ADD_DEFINITIONS(-D_TD_LINUX_32)
     IF (TD_ARM)
-      SET(TD_LINUX_32 TRUE)
-      SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/linux)
       #ADD_DEFINITIONS(-D_M_IX86)
       MESSAGE(STATUS "The current platform is Linux 32-bit")
     ELSE ()
@@ -81,7 +84,7 @@ IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
 ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_DARWIN_64 TRUE)
-    SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/darwin)
+    ADD_DEFINITIONS(-D_TD_DARWIN_64)
     MESSAGE(STATUS "The current platform is Darwin 64-bit")
   ELSE ()
     MESSAGE(FATAL_ERROR "The current platform is Darwin 32-bit, not supported yet")
@@ -90,10 +93,12 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
 ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows")
   IF (${CMAKE_SIZEOF_VOID_P} MATCHES 8)
     SET(TD_WINDOWS_64 TRUE)
-    SET(TD_OS_DIR ${TD_COMMUNITY_DIR}/src/os/windows)
     ADD_DEFINITIONS(-D_M_X64)
+    ADD_DEFINITIONS(-D_TD_WINDOWS_64)
     MESSAGE(STATUS "The current platform is Windows 64-bit")
   ELSE ()
+    SET(TD_WINDOWS_32 TRUE)
+    ADD_DEFINITIONS(-D_TD_WINDOWS_32)
     MESSAGE(FATAL_ERROR "The current platform is Windows 32-bit, not supported yet")
     EXIT ()
   ENDIF ()

documentation/webdocs/markdowndocs/TAOS SQL-ch.md

@@ -556,7 +556,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
     功能说明：统计表中某列的值百分比分位数。  
     返回结果数据类型： 双精度浮点数Double。  
     应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
-    说明：*k*值取值范围0≤*k*≤100，为0的时候等同于MIN，为100的时候等同于MAX。
+    说明：*k*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。
 
 - **APERCENTILE**
     ```mysql
@@ -565,7 +565,7 @@ TDengine支持针对数据的聚合查询。提供支持的聚合和选择函数
     功能说明：统计表中某列的值百分比分位数，与PERCENTILE函数相似，但是返回近似结果。  
     返回结果数据类型： 双精度浮点数Double。  
     应用字段：不能应用在timestamp、binary、nchar、bool类型字段。  
-    说明：*k*值取值范围0≤*k*≤100，为0的时候等同于MIN，为100的时候等同于MAX。推荐使用```APERCENTILE```函数，该函数性能远胜于```PERCENTILE```函数
+    说明：*k*值取值范围0≤*P*≤100，为0的时候等同于MIN，为100的时候等同于MAX。推荐使用```APERCENTILE```函数，该函数性能远胜于```PERCENTILE```函数
     
 - **LAST_ROW**
     ```mysql

documentation/webdocs/markdowndocs/TAOS SQL.md

@@ -289,10 +289,10 @@ TDengine supports aggregations over numerical values, they are listed below:
     Applied to: table/STable. 
 
 
-- **WAVG**
+- **TWA**
   
     ```mysql
-    SELECT WAVG(field_name) FROM tb_name WHERE clause
+    SELECT TWA(field_name) FROM tb_name WHERE clause
     ```
     Function: return the time-weighted average value of a specific column  
     Return Data Type: `double`  
@@ -324,7 +324,7 @@ TDengine supports aggregations over numerical values, they are listed below:
 
 - **LEASTSQUARES**
     ```mysql
-    SELECT LEASTSQUARES(field_name) FROM tb_name [WHERE clause]
+    SELECT LEASTSQUARES(field_name, start_val, step_val) FROM tb_name [WHERE clause]
     ```
     Function: performs a linear fit to the primary timestamp and the specified column. 
     Return Data Type: return a string of the coefficient and the interception of the fitted line.  
@@ -417,6 +417,15 @@ TDengine supports aggregations over numerical values, they are listed below:
     Applied to: table/STable.  
     Note: The range of `P` is `[0, 100]`. When `P=0` , `PERCENTILE` returns the equal value as `MIN`; when `P=100`, `PERCENTILE` returns the equal value as `MAX`. 
 
+- **APERCENTILE**
+    ```mysql
+    SELECT APERCENTILE(field_name, P) FROM { tb_name | stb_name } [WHERE clause]
+    ```
+    Function: the value of the specified column below which `P` percent of the data points fall, it returns approximate value of percentile.
+    Return Data Type: double.  
+    Applicable Data Types: all types except `timestamp`, `binary`, `nchar`, `bool`. 
+    Applied to: table/STable.  
+    Note: The range of `P` is `[0, 100]`. When `P=0` , `APERCENTILE` returns the equal value as `MIN`; when `P=100`, `APERCENTILE` returns the equal value as `MAX`. `APERCENTILE` has a much better performance than `PERCENTILE`.
 
 - **LAST_ROW**
     ```mysql

src/CMakeLists.txt

@@ -10,6 +10,8 @@ ADD_SUBDIRECTORY(client)
 ADD_SUBDIRECTORY(query)
 ADD_SUBDIRECTORY(kit)
 ADD_SUBDIRECTORY(plugins)
+ADD_SUBDIRECTORY(sync)
+ADD_SUBDIRECTORY(balance)
 ADD_SUBDIRECTORY(mnode)
 ADD_SUBDIRECTORY(vnode)
 ADD_SUBDIRECTORY(tsdb)

src/balance/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/dnode/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/sdb/inc)
+  INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(inc)
+  AUX_SOURCE_DIRECTORY(src SRC)
+  ADD_LIBRARY(balance ${SRC})
+ENDIF ()

src/balance/src/balance.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "tutil.h"
+#include "tbalance.h"
+#include "tsync.h"
+#include "ttime.h"
+#include "ttimer.h"
+#include "tglobal.h"
+#include "tdataformat.h"
+#include "dnode.h"
+#include "mnode.h"
+#include "mnodeDef.h"
+#include "mnodeInt.h"
+#include "mnodeDnode.h"
+#include "mnodeDb.h"
+#include "mnodeMnode.h"
+#include "mnodeSdb.h"
+#include "mnodeShow.h"
+#include "mnodeUser.h"
+#include "mnodeVgroup.h"
+
+/*
+ * once sdb work as mater, then tsAccessSquence reset to zero
+ * increase tsAccessSquence every balance interval
+ */
+extern void *       tsMnodeTmr;
+static void *       tsBalanceTimer = NULL;
+static int32_t      tsBalanceDnodeListSize = 0;
+static SDnodeObj ** tsBalanceDnodeList = NULL;
+static int32_t      tsBalanceDnodeListMallocSize = 16;
+static pthread_mutex_t tsBalanceMutex;
+
+static void  balanceStartTimer(int64_t mseconds);
+static void  balanceInitDnodeList();
+static void  balanceCleanupDnodeList();
+static void  balanceAccquireDnodeList();
+static void  balanceReleaseDnodeList();
+static void  balanceMonitorDnodeModule();
+static float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extraVnode);
+static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn);
+static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn);
+
+static void balanceLock() {
+  pthread_mutex_lock(&tsBalanceMutex);
+}
+
+static void balanceUnLock() {
+  pthread_mutex_unlock(&tsBalanceMutex);
+}
+
+static bool balanceCheckFree(SDnodeObj *pDnode) {
+  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+    mError("dnode:%d, status:%s not available", pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status));
+    return false;
+  }
+  
+  if (pDnode->openVnodes >= TSDB_MAX_VNODES) {
+    mError("dnode:%d, openVnodes:%d maxVnodes:%d not available", pDnode->dnodeId, pDnode->openVnodes, TSDB_MAX_VNODES);
+    return false;
+  }
+
+  if (pDnode->diskAvailable <= tsMinimalDataDirGB) {
+    mError("dnode:%d, disk space:%fGB, not available", pDnode->dnodeId, pDnode->diskAvailable);
+    return false;
+  }
+
+  if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_MNODE) {
+    mDebug("dnode:%d, alternative role is master, can't alloc vnodes in this dnode", pDnode->dnodeId);
+    return false;
+  }
+
+  return true;
+}
+
+static void balanceDiscardVnode(SVgObj *pVgroup, SVnodeGid *pVnodeGid) {
+  mDebug("vgId:%d, dnode:%d is dropping", pVgroup->vgId, pVnodeGid->dnodeId);
+
+  SDnodeObj *pDnode = mnodeGetDnode(pVnodeGid->dnodeId);
+  if (pDnode != NULL) {
+    atomic_sub_fetch_32(&pDnode->openVnodes, 1);
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; memset(vnodeGid, 0, sizeof(vnodeGid)); /* = {0}; */
+  int32_t   numOfVnodes = 0;
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pTmpVodeGid = pVgroup->vnodeGid + i;
+    if (pTmpVodeGid == pVnodeGid) {
+      continue;
+    }
+    vnodeGid[numOfVnodes] = *pTmpVodeGid;
+    ++numOfVnodes;
+  }
+  memcpy(pVgroup->vnodeGid, vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid));
+  pVgroup->numOfVnodes = numOfVnodes;
+
+  mnodeUpdateVgroup(pVgroup);
+}
+
+static void balanceSwapVnodeGid(SVnodeGid *pVnodeGid1, SVnodeGid *pVnodeGid2) {
+  // SVnodeGid tmp = *pVnodeGid1;
+  // *pVnodeGid1 = *pVnodeGid2;
+  // *pVnodeGid2 = tmp;
+}
+
+int32_t balanceAllocVnodes(SVgObj *pVgroup) {
+  int32_t dnode = 0;
+  int32_t vnodes = 0;
+
+  balanceLock();
+
+  balanceAccquireDnodeList();
+
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    for (; dnode < tsBalanceDnodeListSize; ++dnode) {
+      SDnodeObj *pDnode = tsBalanceDnodeList[dnode];
+      if (balanceCheckFree(pDnode)) {
+        SVnodeGid *pVnodeGid = pVgroup->vnodeGid + i;
+        pVnodeGid->dnodeId = pDnode->dnodeId;
+        pVnodeGid->pDnode = pDnode;
+        dnode++;
+        vnodes++;
+        break;
+      }
+    }
+  }
+
+  if (vnodes != pVgroup->numOfVnodes) {
+    mDebug("vgId:%d, db:%s need vnodes:%d, but alloc:%d, free them", pVgroup->vgId, pVgroup->dbName,
+           pVgroup->numOfVnodes, vnodes);
+    balanceReleaseDnodeList();       
+    balanceUnLock();
+    return -1;
+  }
+
+  /*
+   * make the choice more random.
+   * replica 1: no choice
+   * replica 2: there are 2 combinations
+   * replica 3 or larger: there are 6 combinations
+   */
+  if (pVgroup->numOfVnodes == 1) {
+  } else if (pVgroup->numOfVnodes == 2) {
+    if (rand() % 2 == 0) {
+      balanceSwapVnodeGid(pVgroup->vnodeGid, pVgroup->vnodeGid + 1);
+    }
+  } else {
+    int32_t randVal = rand() % 6;
+    if (randVal == 1) {  // 1, 0, 2
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+    } else if (randVal == 2) {  // 1, 2, 0
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 1);
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+    } else if (randVal == 3) {  // 2, 1, 0
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+    } else if (randVal == 4) {  // 2, 0, 1
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 0, pVgroup->vnodeGid + 2);
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+    }
+    if (randVal == 5) {  // 0, 2, 1
+      balanceSwapVnodeGid(pVgroup->vnodeGid + 1, pVgroup->vnodeGid + 2);
+    } else {
+    }  // 0, 1, 2
+  }
+
+  balanceReleaseDnodeList();
+  balanceUnLock();
+  return 0;
+}
+
+static bool balanceCheckVgroupReady(SVgObj *pVgroup, SVnodeGid *pRmVnode) {
+  if (pVgroup->lbTime + 5 * tsStatusInterval > tsAccessSquence) {
+    return false;
+  }
+
+  bool isReady = false;
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pVnode = pVgroup->vnodeGid + i;
+    if (pVnode == pRmVnode) continue;
+
+    mTrace("vgId:%d, change vgroup status, dnode:%d status:%d", pVgroup->vgId, pVnode->pDnode->dnodeId,
+           pVnode->pDnode->status);
+    if (pVnode->pDnode->status == TAOS_DN_STATUS_DROPPING) continue;
+    if (pVnode->pDnode->status == TAOS_DN_STATUS_OFFLINE) continue;
+
+    if (pVnode->role == TAOS_SYNC_ROLE_SLAVE || pVnode->role == TAOS_SYNC_ROLE_MASTER) {
+      isReady = true;
+    }
+  }
+
+  return isReady;
+}
+
+/**
+ * desc: remove one vnode from vgroup
+ * all vnodes in vgroup should in ready state, except the balancing one
+ **/
+static void balanceRemoveVnode(SVgObj *pVgroup) {
+  if (pVgroup->numOfVnodes <= 1) return;
+
+  SVnodeGid *pRmVnode = NULL;
+  SVnodeGid *pSelVnode = NULL;
+  int32_t    maxScore = 0;
+
+  for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pVnode = &(pVgroup->vnodeGid[i]);
+    SDnodeObj *pDnode = mnodeGetDnode(pVnode->dnodeId);
+
+    if (pDnode == NULL) {
+      mError("vgId:%d, dnode:%d not exist, remove it", pVgroup->vgId, pVnode->dnodeId);
+      pRmVnode = pVnode;
+      break;
+    }
+
+    if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
+      mDebug("vgId:%d, dnode:%d in dropping state", pVgroup->vgId, pVnode->dnodeId);
+      pRmVnode = pVnode;
+    } else if (pVnode->dnodeId == pVgroup->lbDnodeId) {
+      mDebug("vgId:%d, dnode:%d in updating state", pVgroup->vgId, pVnode->dnodeId);
+      pRmVnode = pVnode;
+    } else {
+      if (pSelVnode == NULL) {
+        pSelVnode = pVnode;
+        maxScore = pDnode->score;
+      } else {
+        if (maxScore < pDnode->score) {
+          pSelVnode = pVnode;
+          maxScore = pDnode->score;
+        }
+      }
+    }
+
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  if (pRmVnode != NULL) {
+    pSelVnode = pRmVnode;
+  }
+
+  if (!balanceCheckVgroupReady(pVgroup, pSelVnode)) {
+    mDebug("vgId:%d, is not ready", pVgroup->vgId);
+  } else {
+    mDebug("vgId:%d, is ready, discard dnode:%d", pVgroup->vgId, pSelVnode->dnodeId);
+    balanceDiscardVnode(pVgroup, pSelVnode);
+  }
+}
+
+static bool balanceCheckDnodeInVgroup(SDnodeObj *pDnode, SVgObj *pVgroup) {
+ for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+    SVnodeGid *pGid = &pVgroup->vnodeGid[i];
+    if (pGid->dnodeId == 0) break;
+    if (pGid->dnodeId == pDnode->dnodeId) {
+      return true;
+    }
+  }
+
+  return false;
+}
+
+/**
+ * desc: add vnode to vgroup, find a new one if dest dnode is null
+ **/
+static int32_t balanceAddVnode(SVgObj *pVgroup, SDnodeObj *pSrcDnode, SDnodeObj *pDestDnode) {
+  if (pDestDnode == NULL) {
+    for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+      SDnodeObj *pDnode = tsBalanceDnodeList[i];
+      if (pDnode == pSrcDnode) continue;
+      if (balanceCheckDnodeInVgroup(pDnode, pVgroup)) continue;
+      if (!balanceCheckFree(pDnode)) continue;
+      
+      pDestDnode = pDnode;
+      mDebug("vgId:%d, add vnode to dnode:%d", pVgroup->vgId, pDnode->dnodeId);
+      break;
+    }
+  }
+
+  if (pDestDnode == NULL) {
+    return TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+
+  SVnodeGid *pVnodeGid = pVgroup->vnodeGid + pVgroup->numOfVnodes;
+  pVnodeGid->dnodeId = pDestDnode->dnodeId;
+  pVnodeGid->pDnode = pDestDnode;
+  pVgroup->numOfVnodes++;
+
+  if (pSrcDnode != NULL) {
+    pVgroup->lbDnodeId = pSrcDnode->dnodeId;
+  }
+
+  atomic_add_fetch_32(&pDestDnode->openVnodes, 1);
+
+  mnodeUpdateVgroup(pVgroup);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static bool balanceMonitorBalance() {
+  if (tsBalanceDnodeListSize < 2) return false;
+
+  for (int32_t src = tsBalanceDnodeListSize - 1; src >= 0; --src) {
+    SDnodeObj *pDnode = tsBalanceDnodeList[src];
+    mDebug("%d-dnode:%d, state:%s, score:%.1f, numOfCores:%d, openVnodes:%d", tsBalanceDnodeListSize - src - 1,
+           pDnode->dnodeId, mnodeGetDnodeStatusStr(pDnode->status), pDnode->score, pDnode->numOfCores,
+           pDnode->openVnodes);
+  }
+
+  float scoresDiff = tsBalanceDnodeList[tsBalanceDnodeListSize - 1]->score - tsBalanceDnodeList[0]->score;
+  if (scoresDiff < 0.01) {
+    mDebug("all dnodes:%d is already balanced, scoresDiff:%f", tsBalanceDnodeListSize, scoresDiff);
+    return false;
+  }
+
+  for (int32_t src = tsBalanceDnodeListSize - 1; src > 0; --src) {
+    SDnodeObj *pSrcDnode = tsBalanceDnodeList[src];
+    float srcScore = balanceTryCalcDnodeScore(pSrcDnode, -1);
+    if (tsEnableBalance == 0 && pSrcDnode->status != TAOS_DN_STATUS_DROPPING) {
+      continue;
+    }
+
+    void *pIter = NULL;
+    while (1) {
+      SVgObj *pVgroup;
+      pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+      if (pVgroup == NULL) break;
+
+      if (balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+        for (int32_t dest = 0; dest < src; dest++) {
+          SDnodeObj *pDestDnode = tsBalanceDnodeList[dest];
+          if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) continue;
+
+          float destScore = balanceTryCalcDnodeScore(pDestDnode, 1);
+          if (srcScore + 0.0001 < destScore) continue;
+          if (!balanceCheckFree(pDestDnode)) continue;
+          
+          mDebug("vgId:%d, balance from dnode:%d to dnode:%d, srcScore:%.1f:%.1f, destScore:%.1f:%.1f",
+                 pVgroup->vgId, pSrcDnode->dnodeId, pDestDnode->dnodeId, pSrcDnode->score,
+                 srcScore, pDestDnode->score, destScore);
+          balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+          mnodeDecVgroupRef(pVgroup);
+          sdbFreeIter(pIter);
+          return true;
+        }
+      }
+
+      mnodeDecVgroupRef(pVgroup);
+    }
+
+    sdbFreeIter(pIter);
+  }
+
+  return false;
+}
+
+// if mgmt changed to master
+// 1. reset balanceAccessSquence to zero
+// 2. reset state of dnodes to offline
+// 3. reset lastAccess of dnodes to zero
+void balanceReset() {
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    // while master change, should reset dnode to offline
+    mInfo("dnode:%d set access:%d to 0", pDnode->dnodeId, pDnode->lastAccess);
+    pDnode->lastAccess = 0;
+    if (pDnode->status != TAOS_DN_STATUS_DROPPING) {
+      pDnode->status = TAOS_DN_STATUS_OFFLINE;
+    }
+
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  sdbFreeIter(pIter);
+
+  tsAccessSquence = 0;
+}
+
+static int32_t balanceMonitorVgroups() {
+  void *  pIter = NULL;
+  SVgObj *pVgroup = NULL;
+  bool    hasUpdatingVgroup = false;
+
+  while (1) {
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    int32_t dbReplica = pVgroup->pDb->cfg.replications;
+    int32_t vgReplica = pVgroup->numOfVnodes;
+    
+    if (vgReplica > dbReplica) {
+      mInfo("vgId:%d, replica:%d numOfVnodes:%d, try remove one vnode", pVgroup->vgId, dbReplica, vgReplica);
+      hasUpdatingVgroup = true;
+      balanceRemoveVnode(pVgroup);
+    } else if (vgReplica < dbReplica) {
+      mInfo("vgId:%d, replica:%d numOfVnodes:%d, try add one vnode", pVgroup->vgId, dbReplica, vgReplica);
+      hasUpdatingVgroup = true;
+      balanceAddVnode(pVgroup, NULL, NULL);
+    }
+
+    mnodeDecVgroupRef(pVgroup);
+  }
+
+  sdbFreeIter(pIter);
+
+  return hasUpdatingVgroup;
+}
+
+static bool balanceMonitorDnodeDropping(SDnodeObj *pDnode) {
+  mDebug("dnode:%d, in dropping state", pDnode->dnodeId);
+
+  void *  pIter = NULL;
+  bool    hasThisDnode = false;
+  while (1) {
+    SVgObj *pVgroup = NULL;
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    hasThisDnode = balanceCheckDnodeInVgroup(pDnode, pVgroup);
+    mnodeDecVgroupRef(pVgroup);
+
+    if (hasThisDnode) break;
+  }
+
+  sdbFreeIter(pIter);
+
+  if (!hasThisDnode) {
+    mInfo("dnode:%d, dropped for all vnodes are moving to other dnodes", pDnode->dnodeId);
+    mnodeDropDnode(pDnode, NULL);
+    return true;
+  }
+
+  return false;
+}
+
+static bool balanceMontiorDropping() {
+  void *pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+
+  while (1) {
+    mnodeDecDnodeRef(pDnode);
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      if (pDnode->lastAccess + tsOfflineThreshold > tsAccessSquence) continue;
+      if (strcmp(pDnode->dnodeEp, dnodeGetMnodeMasterEp()) == 0) continue; 
+      if (mnodeGetDnodesNum() <= 1) continue;
+
+      mLInfo("dnode:%d, set to removing state for it offline:%d seconds", pDnode->dnodeId,
+              tsAccessSquence - pDnode->lastAccess);
+
+      pDnode->status = TAOS_DN_STATUS_DROPPING;
+      mnodeUpdateDnode(pDnode);
+      mnodeDecDnodeRef(pDnode);
+      sdbFreeIter(pIter);
+      return true;
+    }
+
+    if (pDnode->status == TAOS_DN_STATUS_DROPPING) {
+      bool ret = balanceMonitorDnodeDropping(pDnode);
+      mnodeDecDnodeRef(pDnode);
+      sdbFreeIter(pIter);
+      return ret;
+    }
+  }
+
+  sdbFreeIter(pIter);
+
+  return false;
+}
+
+static bool balanceStart() {
+  if (!sdbIsMaster()) return false;
+
+  balanceLock();
+
+  balanceAccquireDnodeList();
+
+  balanceMonitorDnodeModule();
+
+  bool updateSoon = balanceMontiorDropping();
+
+  if (!updateSoon) {
+    updateSoon = balanceMonitorVgroups();
+  }
+
+  if (!updateSoon) {
+    updateSoon = balanceMonitorBalance();
+  }
+ 
+  balanceReleaseDnodeList();
+
+  balanceUnLock();
+
+  return updateSoon;
+}
+
+static void balanceSetVgroupOffline(SDnodeObj* pDnode) {
+  void *pIter = NULL;
+  while (1) {
+    SVgObj *pVgroup;
+    pIter = mnodeGetNextVgroup(pIter, &pVgroup);
+    if (pVgroup == NULL) break;
+
+    for (int32_t i = 0; i < pVgroup->numOfVnodes; ++i) {
+      if (pVgroup->vnodeGid[i].pDnode == pDnode) {
+        pVgroup->vnodeGid[i].role = TAOS_SYNC_ROLE_OFFLINE;
+      }
+    }
+    mnodeDecVgroupRef(pVgroup);
+  }
+
+  sdbFreeIter(pIter);
+}
+
+static void balanceCheckDnodeAccess() {
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (tsAccessSquence - pDnode->lastAccess > 3) {
+      if (pDnode->status != TAOS_DN_STATUS_DROPPING && pDnode->status != TAOS_DN_STATUS_OFFLINE) {
+        pDnode->status = TAOS_DN_STATUS_OFFLINE;
+        mInfo("dnode:%d, set to offline state", pDnode->dnodeId);
+        balanceSetVgroupOffline(pDnode);
+      }
+    }
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  sdbFreeIter(pIter);
+}
+
+static void balanceProcessBalanceTimer(void *handle, void *tmrId) {
+  if (!sdbIsMaster()) return;
+
+  tsBalanceTimer = NULL;
+  tsAccessSquence ++;
+
+  balanceCheckDnodeAccess();  
+  bool updateSoon = false;
+
+  if (handle == NULL) {
+    if (tsAccessSquence % tsBalanceInterval == 0) {
+      mDebug("balance function is scheduled by timer");
+      updateSoon = balanceStart();
+    }
+  } else {
+    int64_t mseconds = (int64_t)handle;
+    mDebug("balance function is scheduled by event for %" PRId64 " mseconds arrived", mseconds);
+    updateSoon = balanceStart();
+  }
+
+  if (updateSoon) {
+    balanceStartTimer(1000);
+  } else {
+    taosTmrReset(balanceProcessBalanceTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBalanceTimer);
+  }
+}
+
+static void balanceStartTimer(int64_t mseconds) {
+  taosTmrReset(balanceProcessBalanceTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBalanceTimer);
+}
+
+void balanceSyncNotify() {
+  if (sdbIsMaster()) {
+    balanceLock();
+    balanceAccquireDnodeList();
+    balanceMonitorDnodeModule();
+    balanceReleaseDnodeList();
+    balanceUnLock();
+  }
+}
+
+void balanceAsyncNotify() {
+  balanceStartTimer(500); 
+}
+
+int32_t balanceInit() {
+  mnodeAddShowMetaHandle(TSDB_MGMT_TABLE_SCORES, balanceGetScoresMeta);
+  mnodeAddShowRetrieveHandle(TSDB_MGMT_TABLE_SCORES, balanceRetrieveScores);
+  
+  pthread_mutex_init(&tsBalanceMutex, NULL);
+  balanceInitDnodeList();
+  balanceStartTimer(2000);
+  mDebug("balance start fp:%p initialized", balanceProcessBalanceTimer);
+
+  balanceReset();
+  
+  return 0;
+}
+
+void balanceCleanUp() {
+  if (tsBalanceTimer != NULL) {
+    taosTmrStopA(&tsBalanceTimer);
+    pthread_mutex_destroy(&tsBalanceMutex);
+    tsBalanceTimer = NULL;
+    mDebug("stop balance timer");
+  }
+  balanceCleanupDnodeList();
+}
+
+int32_t balanceDropDnode(SDnodeObj *pDnode) {
+  int32_t    totalFreeVnodes = 0;
+  void *     pIter = NULL;
+  SDnodeObj *pTempDnode = NULL;
+
+  while (1) {
+    pIter = mnodeGetNextDnode(pIter, &pTempDnode);
+    if (pTempDnode == NULL) break;
+
+    if (pTempDnode != pDnode && balanceCheckFree(pTempDnode)) {
+      totalFreeVnodes += (TSDB_MAX_VNODES - pTempDnode->openVnodes);
+    }
+
+    mnodeDecDnodeRef(pTempDnode);
+  }
+
+  sdbFreeIter(pIter);
+
+  if (pDnode->openVnodes > totalFreeVnodes) {
+    mError("dnode:%d, openVnodes:%d totalFreeVnodes:%d no enough dnodes", pDnode->dnodeId, pDnode->openVnodes, totalFreeVnodes);
+    return TSDB_CODE_MND_NO_ENOUGH_DNODES;
+  }
+
+  pDnode->status = TAOS_DN_STATUS_DROPPING;
+  mnodeUpdateDnode(pDnode);
+  
+  balanceStartTimer(1100);
+
+  return TSDB_CODE_SUCCESS;
+}
+
+static int32_t balanceCalcCpuScore(SDnodeObj *pDnode) {
+  if (pDnode->cpuAvgUsage < 80)
+    return 0;
+  else if (pDnode->cpuAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t balanceCalcMemoryScore(SDnodeObj *pDnode) {
+  if (pDnode->memoryAvgUsage < 80)
+    return 0;
+  else if (pDnode->memoryAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t balanceCalcDiskScore(SDnodeObj *pDnode) {
+  if (pDnode->diskAvgUsage < 80)
+    return 0;
+  else if (pDnode->diskAvgUsage < 90)
+    return 10;
+  else
+    return 50;
+}
+
+static int32_t balanceCalcBandwidthScore(SDnodeObj *pDnode) {
+  if (pDnode->bandwidthUsage < 30)
+    return 0;
+  else if (pDnode->bandwidthUsage < 80)
+    return 10;
+  else
+    return 50;
+}
+
+static float balanceCalcModuleScore(SDnodeObj *pDnode) {
+  if (pDnode->numOfCores <= 0) return 0;
+  if (pDnode->isMgmt) {
+     return (float)tsMnodeEqualVnodeNum / pDnode->numOfCores;
+  }
+  return 0;
+}
+
+static float balanceCalcVnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  if (pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) return 100000000;
+  if (pDnode->numOfCores <= 0) return 0;
+  return (float)(pDnode->openVnodes + extra) / pDnode->numOfCores;
+}
+
+/**
+ * calc singe score, such as cpu/memory/disk/bandwitdh/vnode
+ * 1. get the score config
+ * 2. if the value is out of range, use border data
+ * 3. otherwise use interpolation method
+ **/
+void balanceCalcDnodeScore(SDnodeObj *pDnode) {
+  pDnode->score = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
+                  balanceCalcBandwidthScore(pDnode) + balanceCalcModuleScore(pDnode) +
+                  balanceCalcVnodeScore(pDnode, 0) + pDnode->customScore;
+}
+
+float balanceTryCalcDnodeScore(SDnodeObj *pDnode, int32_t extra) {
+  int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
+                        balanceCalcBandwidthScore(pDnode);
+  float moduleScore = balanceCalcModuleScore(pDnode);
+  float vnodeScore = balanceCalcVnodeScore(pDnode, extra);
+
+  float score = systemScore + moduleScore + vnodeScore +  pDnode->customScore;
+  return score;
+}
+
+static void balanceInitDnodeList() {
+  tsBalanceDnodeList = calloc(tsBalanceDnodeListMallocSize, sizeof(SDnodeObj *));
+}
+
+static void balanceCleanupDnodeList() {
+  if (tsBalanceDnodeList != NULL) {
+    free(tsBalanceDnodeList);
+    tsBalanceDnodeList = NULL;
+  }
+}
+
+static void balanceCheckDnodeListSize(int32_t dnodesNum) {
+  if (tsBalanceDnodeListMallocSize <= dnodesNum) {
+    tsBalanceDnodeListMallocSize = dnodesNum * 2;
+    tsBalanceDnodeList = realloc(tsBalanceDnodeList, tsBalanceDnodeListMallocSize * sizeof(SDnodeObj *));
+  }
+}
+
+void balanceAccquireDnodeList() {
+  int32_t dnodesNum = mnodeGetDnodesNum();
+  balanceCheckDnodeListSize(dnodesNum);
+
+  void *     pIter = NULL;
+  SDnodeObj *pDnode = NULL;
+  int32_t    dnodeIndex = 0;
+
+  while (1) {  
+    if (dnodeIndex >= dnodesNum) break;
+    pIter = mnodeGetNextDnode(pIter, &pDnode);
+    if (pDnode == NULL) break;
+    if (pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      mnodeDecDnodeRef(pDnode);
+      continue;
+    }
+
+    balanceCalcDnodeScore(pDnode);
+    
+    int32_t orderIndex = dnodeIndex;
+    for (; orderIndex > 0; --orderIndex) {
+      if (pDnode->score > tsBalanceDnodeList[orderIndex - 1]->score) {
+        break;
+      }
+      tsBalanceDnodeList[orderIndex] = tsBalanceDnodeList[orderIndex - 1];
+    }
+    tsBalanceDnodeList[orderIndex] = pDnode;
+    dnodeIndex++;
+  }
+
+  sdbFreeIter(pIter);
+
+  tsBalanceDnodeListSize = dnodeIndex;
+}
+
+void balanceReleaseDnodeList() {
+  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+    SDnodeObj *pDnode = tsBalanceDnodeList[i];
+    if (pDnode != NULL) {
+      mnodeDecDnodeRef(pDnode);
+    }
+  }
+}
+
+static int32_t balanceGetScoresMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pConn) {
+  SUserObj *pUser = mnodeGetUserFromConn(pConn);
+  if (pUser == NULL) return 0;
+
+  if (strcmp(pUser->pAcct->user, "root") != 0)  {
+    mnodeDecUserRef(pUser);
+    return TSDB_CODE_MND_NO_RIGHTS;
+  }
+
+  int32_t cols = 0;
+  SSchema *pSchema = pMeta->schema;
+
+  pShow->bytes[cols] = 2;
+  pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
+  strcpy(pSchema[cols].name, "id");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "system scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "custom scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "module scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "vnode scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_FLOAT;
+  strcpy(pSchema[cols].name, "total scores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "open vnodes");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 4;
+  pSchema[cols].type = TSDB_DATA_TYPE_INT;
+  strcpy(pSchema[cols].name, "cpu cores");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pShow->bytes[cols] = 18 + VARSTR_HEADER_SIZE;
+  pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
+  strcpy(pSchema[cols].name, "balance state");
+  pSchema[cols].bytes = htons(pShow->bytes[cols]);
+  cols++;
+
+  pMeta->numOfColumns = htons(cols);
+  pShow->numOfColumns = cols;
+
+  pShow->offset[0] = 0;
+  for (int32_t i = 1; i < cols; ++i) {
+    pShow->offset[i] = pShow->offset[i - 1] + pShow->bytes[i - 1];
+  }
+
+  pShow->numOfRows = mnodeGetDnodesNum();
+  pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
+  pShow->pIter = NULL;
+
+  mnodeDecUserRef(pUser);
+
+  return 0;
+}
+
+static int32_t balanceRetrieveScores(SShowObj *pShow, char *data, int32_t rows, void *pConn) {
+  int32_t    numOfRows = 0;
+  SDnodeObj *pDnode = NULL;
+  char *     pWrite;
+  int32_t    cols = 0;
+  
+  while (numOfRows < rows) {
+    pShow->pIter = mnodeGetNextDnode(pShow->pIter, &pDnode);
+    if (pDnode == NULL) break;
+
+    int32_t systemScore = balanceCalcCpuScore(pDnode) + balanceCalcMemoryScore(pDnode) + balanceCalcDiskScore(pDnode) +
+                      balanceCalcBandwidthScore(pDnode);
+    float moduleScore = balanceCalcModuleScore(pDnode);
+    float vnodeScore = balanceCalcVnodeScore(pDnode, 0);
+
+    cols = 0;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = pDnode->dnodeId;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = systemScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = pDnode->customScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)moduleScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)vnodeScore;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(float *)pWrite = (int32_t)(vnodeScore + moduleScore + pDnode->customScore + systemScore);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->openVnodes;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int32_t *)pWrite = pDnode->numOfCores;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, mnodeGetDnodeStatusStr(pDnode->status));
+    cols++;
+
+    numOfRows++;
+    mnodeDecDnodeRef(pDnode);
+  }
+
+  pShow->numOfReads += numOfRows;
+  return numOfRows;
+}
+
+static void balanceMonitorDnodeModule() {
+  int32_t numOfMnodes = mnodeGetMnodesNum();
+  if (numOfMnodes >= tsNumOfMnodes) return;
+
+  for (int32_t i = 0; i < tsBalanceDnodeListSize; ++i) {
+    SDnodeObj *pDnode = tsBalanceDnodeList[i];
+    if (pDnode == NULL) break;
+
+    if (pDnode->isMgmt || pDnode->status == TAOS_DN_STATUS_DROPPING || pDnode->status == TAOS_DN_STATUS_OFFLINE) {
+      continue;
+    }
+
+    if (pDnode->alternativeRole == TAOS_DN_ALTERNATIVE_ROLE_VNODE) {
+      continue;
+    }
+
+    mLInfo("dnode:%d, numOfMnodes:%d expect:%d, add mnode in this dnode", pDnode->dnodeId, numOfMnodes, tsNumOfMnodes);
+    mnodeAddMnode(pDnode->dnodeId);
+    
+    numOfMnodes = mnodeGetMnodesNum();
+    if (numOfMnodes >= tsNumOfMnodes) return;
+  }
+}
+
+int32_t balanceAlterDnode(struct SDnodeObj *pSrcDnode, int32_t vnodeId, int32_t dnodeId) {
+  if (!sdbIsMaster()) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for self not master", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+
+  if (tsEnableBalance != 0) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for balance enabled", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_BALANCE_ENABLED;
+  }
+
+  SVgObj *pVgroup = mnodeGetVgroup(vnodeId);
+  if (pVgroup == NULL) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for vgroup not exist", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_VGROUP_NOT_EXIST;
+  }
+
+  SDnodeObj *pDestDnode = mnodeGetDnode(dnodeId);
+  if (pDestDnode == NULL) {
+    mnodeDecVgroupRef(pVgroup);
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode not exist", pSrcDnode->dnodeId, vnodeId, dnodeId);
+    return TSDB_CODE_MND_DNODE_NOT_EXIST;
+  }
+
+  balanceLock();
+  balanceAccquireDnodeList();
+
+  int32_t code = TSDB_CODE_SUCCESS;
+  if (!balanceCheckDnodeInVgroup(pSrcDnode, pVgroup)) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup not in dnode:%d", pSrcDnode->dnodeId, vnodeId,
+           dnodeId, pSrcDnode->dnodeId);
+    code = TSDB_CODE_MND_VGROUP_NOT_IN_DNODE;
+  } else if (balanceCheckDnodeInVgroup(pDestDnode, pVgroup)) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, vgroup already in dnode:%d", pSrcDnode->dnodeId, vnodeId,
+           dnodeId, dnodeId);
+    code = TSDB_CODE_MND_VGROUP_ALREADY_IN_DNODE;
+  } else if (!balanceCheckFree(pDestDnode)) {
+    mError("dnode:%d, failed to alter vgId:%d to dnode:%d, for dnode:%d not free", pSrcDnode->dnodeId, vnodeId, dnodeId,
+           dnodeId);
+    code = TSDB_CODE_MND_DNODE_NOT_FREE;
+  } else {
+    code = balanceAddVnode(pVgroup, pSrcDnode, pDestDnode);
+    mInfo("dnode:%d, alter vgId:%d to dnode:%d, result:%s", pSrcDnode->dnodeId, vnodeId, dnodeId, tstrerror(code));
+  }
+
+  balanceReleaseDnodeList();
+  balanceUnLock();
+
+  mnodeDecVgroupRef(pVgroup);
+  mnodeDecDnodeRef(pDestDnode);
+
+  return code;
+}
\ No newline at end of file

src/client/CMakeLists.txt

@@ -4,11 +4,11 @@ PROJECT(TDengine)
 INCLUDE_DIRECTORIES(inc)
 INCLUDE_DIRECTORIES(jni)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 AUX_SOURCE_DIRECTORY(src SRC)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))

src/client/src/tscParseInsert.c

@@ -104,7 +104,7 @@ int tsParseTime(SSQLToken *pToken, int64_t *time, char **next, char *error, int1
   } else if (strncmp(pToken->z, "0", 1) == 0 && pToken->n == 1) {
     // do nothing
   } else if (pToken->type == TK_INTEGER) {
-    useconds = str2int64(pToken->z);
+    useconds = tsosStr2int64(pToken->z);
   } else {
     // strptime("2001-11-12 18:31:01", "%Y-%m-%d %H:%M:%S", &tm);
     if (taosParseTime(pToken->z, time, pToken->n, timePrec, tsDaylight) != TSDB_CODE_SUCCESS) {

src/client/src/tscSystem.c

@@ -220,7 +220,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
 
         if (strlen(tsLocale) == 0) { // locale does not set yet
           char* defaultLocale = setlocale(LC_CTYPE, "");
-          tstrncpy(tsLocale, defaultLocale, sizeof(tsLocale));
+          tstrncpy(tsLocale, defaultLocale, TSDB_LOCALE_LEN);
         }
 
         // set the user specified locale
@@ -234,7 +234,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
           tscInfo("failed to set locale:%s, current locale:%s", pStr, tsLocale);
         }
 
-        tstrncpy(tsLocale, locale, sizeof(tsLocale));
+        tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
 
         char *charset = strrchr(tsLocale, sep);
         if (charset != NULL) {
@@ -249,7 +249,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
               tscInfo("charset changed from %s to %s", tsCharset, charset);
             }
 
-            tstrncpy(tsCharset, charset, sizeof(tsCharset));
+            tstrncpy(tsCharset, charset, TSDB_LOCALE_LEN);
             cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
 
           } else {
@@ -286,7 +286,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
             tscInfo("charset changed from %s to %s", tsCharset, pStr);
           }
 
-          tstrncpy(tsCharset, pStr, sizeof(tsCharset));
+          tstrncpy(tsCharset, pStr, TSDB_LOCALE_LEN);
           cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         } else {
           tscInfo("charset:%s not valid", pStr);
@@ -304,7 +304,7 @@ static int taos_options_imp(TSDB_OPTION option, const char *pStr) {
       assert(cfg != NULL);
     
       if (cfg->cfgStatus <= TAOS_CFG_CSTATUS_OPTION) {
-        tstrncpy(tsTimezone, pStr, sizeof(tsTimezone));
+        tstrncpy(tsTimezone, pStr, TSDB_TIMEZONE_LEN);
         tsSetTimeZone();
         cfg->cfgStatus = TAOS_CFG_CSTATUS_OPTION;
         tscDebug("timezone set:%s, input:%s by taos_options", tsTimezone, pStr);

src/common/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-  INCLUDE_DIRECTORIES(inc)
-  AUX_SOURCE_DIRECTORY(src SRC)
-  ADD_LIBRARY(common ${SRC})
-  TARGET_LINK_LIBRARIES(common tutil)
-ENDIF ()
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(inc)
+
+AUX_SOURCE_DIRECTORY(src SRC)
+ADD_LIBRARY(common ${SRC})
+TARGET_LINK_LIBRARIES(common tutil)

src/common/inc/tglobal.h

@@ -43,9 +43,9 @@ extern uint32_t tsMaxTmrCtrl;
 extern float    tsNumOfThreadsPerCore;
 extern float    tsRatioOfQueryThreads;
 extern int8_t   tsDaylight;
-extern char     tsTimezone[64];
-extern char     tsLocale[64];
-extern char     tsCharset[64];  // default encode string
+extern char     tsTimezone[];
+extern char     tsLocale[];
+extern char     tsCharset[];  // default encode string
 extern int32_t  tsEnableCoreFile;
 extern int32_t  tsCompressMsgSize;
 

src/common/src/tglobal.c

@@ -50,7 +50,7 @@ int32_t tsShellActivityTimer = 3;  // second
 float   tsNumOfThreadsPerCore = 1.0;
 float   tsRatioOfQueryThreads = 0.5;
 int8_t  tsDaylight = 0;
-char    tsTimezone[64] = {0};
+char    tsTimezone[TSDB_TIMEZONE_LEN] = {0};
 char    tsLocale[TSDB_LOCALE_LEN] = {0};
 char    tsCharset[TSDB_LOCALE_LEN] = {0};  // default encode string
 int32_t tsEnableCoreFile = 0;

src/cq/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(inc)

src/cq/test/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-  INCLUDE_DIRECTORIES(../inc)
-
-  LIST(APPEND CQTEST_SRC ./cqtest.c)
-  ADD_EXECUTABLE(cqtest ${CQTEST_SRC})
-  TARGET_LINK_LIBRARIES(cqtest tcq)
-
-ENDIF ()
-
-
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/cq/inc)
+
+LIST(APPEND CQTEST_SRC ./cqtest.c)
+ADD_EXECUTABLE(cqtest ${CQTEST_SRC})
+TARGET_LINK_LIBRARIES(cqtest tcq)

src/dnode/CMakeLists.txt

@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc)
@@ -16,7 +16,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(src SRC)
 
   ADD_EXECUTABLE(taosd ${SRC})
-  TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4)
+  TARGET_LINK_LIBRARIES(taosd mnode taos_static monitor http mqtt tsdb twal vnode cJson lz4 balance sync)
 
   IF (TD_ACCOUNT)
     TARGET_LINK_LIBRARIES(taosd account)
@@ -26,10 +26,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
     TARGET_LINK_LIBRARIES(taosd grant)
   ENDIF ()
 
-  IF (TD_SYNC)
-    TARGET_LINK_LIBRARIES(taosd balance sync)
-  ENDIF ()
-
   SET(PREPARE_ENV_CMD "prepare_env_cmd")
   SET(PREPARE_ENV_TARGET "prepare_env_target")
   ADD_CUSTOM_COMMAND(OUTPUT ${PREPARE_ENV_CMD}

src/dnode/src/dnodeSystem.c

@@ -20,7 +20,6 @@
 #include "tglobal.h"
 #include "dnodeInt.h"
 #include "dnodeMain.h"
-#include "tfile.h"
 
 static void signal_handler(int32_t signum, siginfo_t *sigInfo, void *context);
 static sem_t exitSem;
@@ -40,7 +39,7 @@ int32_t main(int32_t argc, char *argv[]) {
         exit(EXIT_FAILURE);
       }
     } else if (strcmp(argv[i], "-V") == 0) {
-#ifdef _SYNC
+#ifdef _ACCT
       char *versionStr = "enterprise";
 #else
       char *versionStr = "community";

src/inc/taosdef.h

@@ -250,7 +250,7 @@ void tsDataSwap(void *pLeft, void *pRight, int32_t type, int32_t size);
 #define TSDB_STATE_LEN            20
 #define TSDB_COUNTRY_LEN          20
 #define TSDB_LOCALE_LEN           64
-#define TSDB_TIMEZONE_LEN         64
+#define TSDB_TIMEZONE_LEN         96
 #define TSDB_LABEL_LEN            8 
 
 #define TSDB_FQDN_LEN             128

src/kit/shell/CMakeLists.txt

@@ -2,11 +2,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))

src/kit/taosdemo/CMakeLists.txt

@@ -2,9 +2,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))

src/kit/taosdump/CMakeLists.txt

@@ -2,11 +2,11 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))

src/kit/taosdump/taosdump.c

@@ -545,7 +545,7 @@ int32_t taosSaveAllNormalTableToTempFile(TAOS *taosCon, char*meter, char* metric
   strcpy(tableRecord.name, meter);
   strcpy(tableRecord.metric, metric);
 
-  twrite(*fd, &tableRecord, sizeof(STableRecord));
+  taosTWrite(*fd, &tableRecord, sizeof(STableRecord));
   return 0;
 }
 
@@ -600,7 +600,7 @@ int32_t taosSaveTableOfMetricToTempFile(TAOS *taosCon, char* metric, struct argu
     strncpy(tableRecord.name, (char *)row[0], fields[0].bytes);
     strcpy(tableRecord.metric, metric);
 
-    twrite(fd, &tableRecord, sizeof(STableRecord));
+    taosTWrite(fd, &tableRecord, sizeof(STableRecord));
 
     numOfTable++;
 
@@ -1196,7 +1196,7 @@ int32_t taosDumpCreateSuperTableClause(TAOS* taosCon, char* dbName, FILE *fp)
   while ((row = taos_fetch_row(tmpResult)) != NULL) {  
     memset(&tableRecord, 0, sizeof(STableRecord));
     strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
-    twrite(fd, &tableRecord, sizeof(STableRecord));
+    taosTWrite(fd, &tableRecord, sizeof(STableRecord));
   }  
   
   taos_free_result(tmpResult);
@@ -1282,7 +1282,7 @@ int taosDumpDb(SDbInfo *dbInfo, struct arguments *arguments, FILE *fp, TAOS *tao
     strncpy(tableRecord.name, (char *)row[TSDB_SHOW_TABLES_NAME_INDEX], fields[TSDB_SHOW_TABLES_NAME_INDEX].bytes);
     strncpy(tableRecord.metric, (char *)row[TSDB_SHOW_TABLES_METRIC_INDEX], fields[TSDB_SHOW_TABLES_METRIC_INDEX].bytes);
 
-    twrite(fd, &tableRecord, sizeof(STableRecord));
+    taosTWrite(fd, &tableRecord, sizeof(STableRecord));
 
     numOfTable++;
 

src/kit/taosmigrate/CMakeLists.txt

@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/mnode/inc) 
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/inc)

src/mnode/CMakeLists.txt

@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)

src/mnode/src/mnodeDb.c

@@ -301,13 +301,6 @@ static int32_t mnodeCheckDbCfg(SDbCfg *pCfg) {
     return TSDB_CODE_MND_INVALID_DB_OPTION;
   }
 
-#ifndef _SYNC
-  if (pCfg->replications != 1) {
-    mError("invalid db option replications:%d can only be 1 in this version", pCfg->replications);
-    return TSDB_CODE_MND_INVALID_DB_OPTION;
-  }
-#endif
-
   return TSDB_CODE_SUCCESS;
 }
 

src/mnode/src/mnodeDnode.c

@@ -79,9 +79,6 @@ static int32_t mnodeDnodeActionInsert(SSdbOper *pOper) {
 static int32_t mnodeDnodeActionDelete(SSdbOper *pOper) {
   SDnodeObj *pDnode = pOper->pObj;
  
-#ifndef _SYNC 
-  mnodeDropAllDnodeVgroups(pDnode);
-#endif  
   mnodeDropMnodeLocal(pDnode->dnodeId);
   balanceAsyncNotify();
 
@@ -552,12 +549,7 @@ static int32_t mnodeDropDnodeByEp(char *ep, SMnodeMsg *pMsg) {
 
   mInfo("dnode:%d, start to drop it", pDnode->dnodeId);
 
-#ifndef _SYNC
-  int32_t code = mnodeDropDnode(pDnode, pMsg);
-#else
   int32_t code = balanceDropDnode(pDnode);
-#endif
-
   mnodeDecDnodeRef(pDnode);
   return code;
 }

src/os/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-ADD_SUBDIRECTORY(linux)
-ADD_SUBDIRECTORY(windows)
-ADD_SUBDIRECTORY(darwin)
+IF (TD_LINUX_64)
+  ADD_SUBDIRECTORY(src/linux64)
+ELSEIF (TD_LINUX_32)
+  ADD_SUBDIRECTORY(src/linux32)
+ELSEIF (TD_DARWIN_64)
+  ADD_SUBDIRECTORY(src/darwin64)  
+ELSEIF (TD_WINDOWS_64)
+  ADD_SUBDIRECTORY(src/windows64)
+ELSEIF (TD_WINDOWS_32)
+  ADD_SUBDIRECTORY(src/windows32)
+ENDIF ()
+
+ADD_SUBDIRECTORY(src/detail)

src/os/darwin/CMakeLists.txt

-CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
-PROJECT(TDengine)
-
-IF (TD_DARWIN_64)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(inc)
-  AUX_SOURCE_DIRECTORY(src SRC)
-  ADD_LIBRARY(os ${SRC})
-ENDIF ()

src/os/inc/os.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_H
+#define TDENGINE_OS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef _TD_DARWIN_64
+#include "osDarwin64.h"
+#endif
+
+#ifdef _TD_LINUX_64
+#include "osLinux64.h"
+#endif
+
+#ifdef _TD_LINUX_32
+#include "osLinux32.h"
+#endif
+
+#ifdef _TD_ALPINE
+#include "osAlpine.h"
+#endif
+
+#ifdef _TD_WINDOWS_64
+#include "osWindows64.h"
+#endif
+
+#ifdef _TD_WINDOWS_32
+#include "osWindows32.h"
+#endif
+
+#include "osSpec.h"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/os/inc/osAlpine.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_LINUX64_H
+#define TDENGINE_OS_LINUX64_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <argp.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <endian.h>
+#include <errno.h>
+#include <float.h>
+#include <ifaddrs.h>
+#include <libgen.h>
+#include <limits.h>
+#include <locale.h>
+#include <math.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <regex.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/sendfile.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <syslog.h>
+#include <termios.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wordexp.h>
+#include <wctype.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+#include <linux/sysctl.h>
+
+typedef int(*__compar_fn_t)(const void *, const void *);
+void  error (int, int, const char *);
+#ifndef PTHREAD_MUTEX_RECURSIVE_NP
+  #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/os/darwin/inc/os.h → src/os/inc/osDarwin64.h

@@ -201,6 +201,8 @@ int tsem_destroy(dispatch_semaphore_t *sem);
 
 void osInit();
 
+ssize_t tread(int fd, void *buf, size_t count);
+
 ssize_t twrite(int fd, void *buf, size_t n);
 
 char *taosCharsetReplace(char *charsetstr);
@@ -233,6 +235,8 @@ int fsendfile(FILE* out_file, FILE* in_file, int64_t* offset, int32_t count);
 
 void taosSetCoreDump();
 
+int tSystem(const char * cmd);
+
 typedef int(*__compar_fn_t)(const void *, const void *);
 
 // for send function in tsocket.c
@@ -255,6 +259,17 @@ typedef int(*__compar_fn_t)(const void *, const void *);
 #define BUILDIN_CLZ(val)  __builtin_clz(val)
 #define BUILDIN_CTZ(val)  __builtin_ctz(val)
 
+#undef threadlocal
+#ifdef _ISOC11_SOURCE
+  #define threadlocal _Thread_local
+#elif defined(__APPLE__)
+  #define threadlocal
+#elif defined(__GNUC__) && !defined(threadlocal)
+  #define threadlocal __thread
+#else
+  #define threadlocal
+#endif
+
 #ifdef __cplusplus
 }
 #endif

src/os/inc/osLinux32.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_LINUX64_H
+#define TDENGINE_OS_LINUX64_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <argp.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <endian.h>
+#include <errno.h>
+#include <float.h>
+#include <ifaddrs.h>
+#include <libgen.h>
+#include <limits.h>
+#include <locale.h>
+#include <math.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <regex.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/sendfile.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <syslog.h>
+#include <termios.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wordexp.h>
+#include <wctype.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+#include <error.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/os/inc/osLinux64.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_LINUX64_H
+#define TDENGINE_OS_LINUX64_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <argp.h>
+#include <arpa/inet.h>
+#include <assert.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <endian.h>
+#include <errno.h>
+#include <float.h>
+#include <ifaddrs.h>
+#include <libgen.h>
+#include <limits.h>
+#include <locale.h>
+#include <math.h>
+#include <netdb.h>
+#include <netinet/in.h>
+#include <netinet/ip.h>
+#include <netinet/tcp.h>
+#include <netinet/udp.h>
+#include <pthread.h>
+#include <pwd.h>
+#include <regex.h>
+#include <semaphore.h>
+#include <signal.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+#include <strings.h>
+#include <sys/epoll.h>
+#include <sys/eventfd.h>
+#include <sys/file.h>
+#include <sys/ioctl.h>
+#include <sys/mman.h>
+#include <sys/sendfile.h>
+#include <sys/socket.h>
+#include <sys/stat.h>
+#include <sys/syscall.h>
+#include <sys/statvfs.h>
+#include <sys/time.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/un.h>
+#include <syslog.h>
+#include <termios.h>
+#include <unistd.h>
+#include <wchar.h>
+#include <wordexp.h>
+#include <wctype.h>
+#include <inttypes.h>
+#include <fcntl.h>
+#include <sys/utsname.h>
+#include <sys/resource.h>
+#include <error.h>
+#include <linux/sysctl.h>
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/os/inc/osSpec.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_OS_SPEC_H
+#define TDENGINE_OS_SPEC_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef TAOS_OS_FUNC_MATH
+  #define SWAP(a, b, c)        \
+    do {                       \
+        typeof(a) __tmp = (a); \
+        (a) = (b);             \
+        (b) = __tmp;           \
+    } while (0)
+
+  #define MAX(a, b)              \
+    ({                           \
+        typeof(a) __a = (a);     \
+        typeof(b) __b = (b);     \
+        (__a > __b) ? __a : __b; \
+    })
+
+  #define MIN(a, b)              \
+    ({                           \
+        typeof(a) __a = (a);     \
+        typeof(b) __b = (b);     \
+        (__a < __b) ? __a : __b; \
+    })
+#endif
+
+#ifndef TAOS_OS_DEF_TIME
+  #define MILLISECOND_PER_SECOND ((int64_t)1000L)
+#endif
+
+#ifndef TAOS_OS_FUNC_SEMPHONE
+  #define tsem_t sem_t
+  #define tsem_init sem_init
+  #define tsem_wait sem_wait
+  #define tsem_post sem_post
+  #define tsem_destroy sem_destroy
+#endif
+
+#ifndef TAOS_OS_FUNC_ATOMIC
+  #define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+  #define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
+
+  #define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_16 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_32 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_64 __sync_val_compare_and_swap
+  #define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap
+
+  #define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
+
+  #define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+  #define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
+#endif
+
+ssize_t taosTReadImp(int fd, void *buf, size_t count);
+ssize_t taosTWriteImp(int fd, void *buf, size_t count);
+ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size);
+#ifndef TAOS_OS_FUNC_FILE
+  #define taosTRead(fd, buf, count) taosTReadImp(fd, buf, count)
+  #define taosTWrite(fd, buf, count) taosTWriteImp(fd, buf, count)
+  #define taosLSeek(fd, offset, whence) lseek(fd, offset, whence)
+  #define taosTSendFile(dfd, sfd, offset, size) taosTSendFileImp(dfd, sfd, offset, size)
+#endif
+
+#ifdef TAOS_RANDOM_FILE_FAIL
+  void taosSetRandomFileFailFactor(int factor);
+  void taosSetRandomFileFailOutput(const char *path);
+  ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
+  ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line);
+  off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line);
+  #define taosTRead(fd, buf, count) taosReadFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+  #define taosTWrite(fd, buf, count) taosWriteFileRandomFail(fd, buf, count, __FILE__, __LINE__)
+  #define taosLSeek(fd, offset, whence) taosLSeekRandomFail(fd, offset, whence, __FILE__, __LINE__)
+#endif
+
+#ifndef TAOS_OS_FUNC_NETWORK
+  #define taosSend(sockfd, buf, len, flags) send(sockfd, buf, len, flags)
+  #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) sendto(sockfd, buf, len, flags, dest_addr, addrlen)
+  #define taosReadSocket(fd, buf, len) read(fd, buf, len)
+  #define taosWriteSocket(fd, buf, len) write(fd, buf, len)
+  #define taosCloseSocket(x) \
+    {                        \
+      if (FD_VALID(x)) {     \
+        close(x);            \
+        x = FD_INITIALIZER;  \
+      }                      \
+    }
+#endif
+
+#ifdef TAOS_RANDOM_NETWORK_FAIL
+  ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags);
+  ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr, socklen_t addrlen);
+  ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count);
+  ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count);
+  #define taosSend(sockfd, buf, len, flags) taosSendRandomFail(sockfd, buf, len, flags)
+  #define taosSendto(sockfd, buf, len, flags, dest_addr, addrlen) taosSendToRandomFail(sockfd, buf, len, flags, dest_addr, addrlen)
+  #define taosReadSocket(fd, buf, len) taosReadSocketRandomFail(fd, buf, len)
+  #define taosWriteSocket(fd, buf, len) taosWriteSocketRandomFail(fd, buf, len)
+#endif
+
+#ifndef TAOS_OS_FUNC_LZ4
+  #define BUILDIN_CLZL(val) __builtin_clzl(val)
+  #define BUILDIN_CTZL(val) __builtin_ctzl(val)
+  #define BUILDIN_CLZ(val) __builtin_clz(val)
+  #define BUILDIN_CTZ(val) __builtin_ctz(val)
+#endif
+
+#undef threadlocal
+#ifdef _ISOC11_SOURCE
+  #define threadlocal _Thread_local
+#elif defined(__APPLE__)
+  #define threadlocal
+#elif defined(__GNUC__) && !defined(threadlocal)
+  #define threadlocal __thread
+#else
+  #define threadlocal
+#endif
+
+void osInit();
+
+// TAOS_OS_FUNC_PTHREAD
+bool taosCheckPthreadValid(pthread_t thread);
+int64_t taosGetPthreadId();
+
+// TAOS_OS_FUNC_SOCKET
+int taosSetNonblocking(int sock, int on);
+int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen);
+void taosBlockSIGPIPE();
+
+// TAOS_OS_FUNC_SYSINFO
+void taosGetSystemInfo();
+void taosPrintOsInfo();
+void taosKillSystem();
+int tSystem(const char * cmd) ;
+
+// TAOS_OS_FUNC_CORE
+void taosSetCoreDump();
+
+// TAOS_OS_FUNC_UTIL
+int64_t tsosStr2int64(char *str);
+
+// TAOS_OS_FUNC_TIMER
+void taosMsleep(int mseconds);
+int taosInitTimer(void (*callback)(int), int ms);
+void taosUninitTimer();
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/os/inc/osWindows32.h

+/*
+* Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+*
+* This program is free software: you can use, redistribute, and/or modify
+* it under the terms of the GNU Affero General Public License, version 3
+* or later ("AGPL"), as published by the Free Software Foundation.
+*
+* This program is distributed in the hope that it will be useful, but WITHOUT
+* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+* FITNESS FOR A PARTICULAR PURPOSE.
+*
+* You should have received a copy of the GNU Affero General Public License
+* along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#ifndef TDENGINE_PLATFORM_WINDOWS32_H
+#define TDENGINE_PLATFORM_WINDOWS32_H
+
+#include <assert.h>
+#include <ctype.h>
+#include <direct.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <float.h>
+#include <locale.h>
+#include <intrin.h>
+#include <io.h>
+#include <math.h>
+#include <pthread.h>
+#include <semaphore.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <signal.h>
+#include <sys/stat.h>
+#include <sys/types.h>
+#include <time.h>
+#include <inttypes.h>
+#include "winsock2.h"
+#include <WS2tcpip.h>
+#include <winbase.h>
+#include <Winsock2.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+#endif
\ No newline at end of file

src/os/windows/inc/os.h → src/os/inc/osWindows64.h

@@ -78,10 +78,10 @@ extern "C" {
 #define wcsncasecmp _wcsnicmp
 #define strtok_r strtok_s
 #ifdef _TD_GO_DLL_
-  int64_t str2int64(char *str);
+  int64_t tsosStr2int64(char *str);
   uint64_t htonll(uint64_t val);
 #else
-  #define str2int64 _atoi64
+  #define tsosStr2int64 _atoi64
 #endif
 
 #define snprintf _snprintf

src/os/linux/inc/os.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_PLATFORM_LINUX_H
-#define TDENGINE_PLATFORM_LINUX_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <argp.h>
-#include <arpa/inet.h>
-#include <assert.h>
-#include <ctype.h>
-#include <dirent.h>
-#include <endian.h>
-#include <errno.h>
-#include <float.h>
-#include <ifaddrs.h>
-#include <libgen.h>
-#include <limits.h>
-#include <locale.h>
-#include <math.h>
-#include <netdb.h>
-#include <netinet/in.h>
-#include <netinet/ip.h>
-#include <netinet/tcp.h>
-#include <netinet/udp.h>
-#include <pthread.h>
-#include <pwd.h>
-#include <regex.h>
-#include <semaphore.h>
-#include <signal.h>
-#include <stdarg.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
-#include <string.h>
-#include <strings.h>
-#include <sys/epoll.h>
-#include <sys/eventfd.h>
-#include <sys/file.h>
-#include <sys/ioctl.h>
-#include <sys/mman.h>
-#include <sys/sendfile.h>
-#include <sys/socket.h>
-#include <sys/stat.h>
-#include <sys/syscall.h>
-#include <sys/statvfs.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <sys/uio.h>
-#include <sys/un.h>
-#include <syslog.h>
-#include <termios.h>
-#include <unistd.h>
-#include <wchar.h>
-#include <wordexp.h>
-#include <wctype.h>
-#include <inttypes.h>
-#include <fcntl.h>
-#include <sys/utsname.h>
-#include <sys/resource.h>
-#include <error.h>
-
-#define taosCloseSocket(x) \
-  {                        \
-    if (FD_VALID(x)) {     \
-      close(x);            \
-      x = FD_INITIALIZER;  \
-    }                      \
-  }
-  
-#ifdef TAOS_RANDOM_NETWORK_FAIL
-
-ssize_t taos_send_random_fail(int sockfd, const void *buf, size_t len, int flags);
-
-ssize_t taos_sendto_random_fail(int sockfd, const void *buf, size_t len, int flags,
-                      const struct sockaddr *dest_addr, socklen_t addrlen);
-ssize_t taos_read_random_fail(int fd, void *buf, size_t count);
-ssize_t taos_write_random_fail(int fd, const void *buf, size_t count);
-
-#define send(sockfd, buf, len, flags) taos_send_random_fail(sockfd, buf, len, flags)
-#define sendto(sockfd, buf, len, flags, dest_addr, addrlen) \
-          taos_sendto_random_fail(sockfd, buf, len, flags, dest_addr, addrlen)
-#define taosWriteSocket(fd, buf, len) taos_write_random_fail(fd, buf, len)
-#define taosReadSocket(fd, buf, len) taos_read_random_fail(fd, buf, len)
-
-#else
-
-#define taosWriteSocket(fd, buf, len) write(fd, buf, len)
-#define taosReadSocket(fd, buf, len) read(fd, buf, len)
-
-#endif  /* TAOS_RANDOM_NETWORK_FAIL */
-
-#define atomic_load_8(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_16(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_32(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_64(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-#define atomic_load_ptr(ptr) __atomic_load_n((ptr), __ATOMIC_SEQ_CST)
-
-#define atomic_store_8(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_16(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_32(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_64(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_store_ptr(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_exchange_8(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_16(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_32(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_64(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_exchange_ptr(ptr, val) __atomic_exchange_n((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_val_compare_exchange_8 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_16 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_32 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_64 __sync_val_compare_and_swap
-#define atomic_val_compare_exchange_ptr __sync_val_compare_and_swap
-
-#define atomic_add_fetch_8(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_16(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_32(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_64(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_add_fetch_ptr(ptr, val) __atomic_add_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_add_8(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_16(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_32(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_64(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_add_ptr(ptr, val) __atomic_fetch_add((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_sub_fetch_8(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_16(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_32(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_64(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_sub_fetch_ptr(ptr, val) __atomic_sub_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_sub_8(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_16(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_32(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_64(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_sub_ptr(ptr, val) __atomic_fetch_sub((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_and_fetch_8(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_16(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_32(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_64(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_and_fetch_ptr(ptr, val) __atomic_and_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_and_8(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_16(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_32(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_64(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_and_ptr(ptr, val) __atomic_fetch_and((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_or_fetch_8(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_16(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_32(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_64(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_or_fetch_ptr(ptr, val) __atomic_or_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_or_8(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_16(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_32(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_64(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_or_ptr(ptr, val) __atomic_fetch_or((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_xor_fetch_8(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_16(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_32(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_64(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_xor_fetch_ptr(ptr, val) __atomic_xor_fetch((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define atomic_fetch_xor_8(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_16(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_32(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_64(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-#define atomic_fetch_xor_ptr(ptr, val) __atomic_fetch_xor((ptr), (val), __ATOMIC_SEQ_CST)
-
-#define SWAP(a, b, c)      \
-  do {                     \
-    typeof(a) __tmp = (a); \
-    (a) = (b);             \
-    (b) = __tmp;           \
-  } while (0)
-
-#define MAX(a, b)            \
-  ({                         \
-    typeof(a) __a = (a);     \
-    typeof(b) __b = (b);     \
-    (__a > __b) ? __a : __b; \
-  })
-
-#define MIN(a, b)            \
-  ({                         \
-    typeof(a) __a = (a);     \
-    typeof(b) __b = (b);     \
-    (__a < __b) ? __a : __b; \
-  })
-
-#define MILLISECOND_PER_SECOND ((int64_t)1000L)
-
-#define tsem_t sem_t
-#define tsem_init sem_init
-#define tsem_wait sem_wait
-#define tsem_post sem_post
-#define tsem_destroy sem_destroy
-
-void osInit();
-
-ssize_t tsendfile(int dfd, int sfd, off_t *offset, size_t size);
-
-ssize_t twrite(int fd, void *buf, size_t n);
-
-ssize_t tread(int fd, void *buf, size_t count);
-
-bool taosCheckPthreadValid(pthread_t thread);
-
-void taosResetPthread(pthread_t *thread);
-
-int64_t taosGetPthreadId();
-
-int taosSetNonblocking(int sock, int on);
-
-int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen);
-
-void taosPrintOsInfo();
-
-char *taosCharsetReplace(char *charsetstr);
-
-void taosGetSystemInfo();
-
-void taosKillSystem();
-
-bool taosSkipSocketCheck();
-
-int64_t str2int64(char *str);
-
-void taosSetCoreDump();
-
-void taosBlockSIGPIPE();
-
-int tSystem(const char * cmd) ;
-
-
-#ifdef _ALPINE
-  typedef int(*__compar_fn_t)(const void *, const void *);
-  void  error (int, int, const char *);
-  #ifndef PTHREAD_MUTEX_RECURSIVE_NP
-    #define  PTHREAD_MUTEX_RECURSIVE_NP PTHREAD_MUTEX_RECURSIVE
-  #endif
-#endif
-
-#ifndef _TD_ARM_32_
-#define BUILDIN_CLZL(val) __builtin_clzl(val)
-#define BUILDIN_CTZL(val) __builtin_ctzl(val)
-#else
-#define BUILDIN_CLZL(val) __builtin_clzll(val)
-#define BUILDIN_CTZL(val) __builtin_ctzll(val)
-#endif
-#define BUILDIN_CLZ(val) __builtin_clz(val)
-#define BUILDIN_CTZ(val) __builtin_ctz(val)
-
-#undef threadlocal
-#ifdef _ISOC11_SOURCE
-  #define threadlocal _Thread_local
-#elif defined(__APPLE__)
-  #define threadlocal
-#elif defined(__GNUC__) && !defined(threadlocal)
-  #define threadlocal __thread
-#else
-  #define threadlocal
-#endif
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif

src/os/src/alpine/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+AUX_SOURCE_DIRECTORY(. SRC)
+
+ADD_LIBRARY(os ${SRC})
+TARGET_LINK_LIBRARIES(os m rt)
+

src/os/linux/CMakeLists.txt → src/os/src/darwin64/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
+IF (TD_DARWIN_64)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
-
   ADD_LIBRARY(os ${SRC})
-  TARGET_LINK_LIBRARIES(os m rt)
 ENDIF ()
 
-SET_SOURCE_FILES_PROPERTIES(src/linuxSysPara.c PROPERTIES COMPILE_FLAGS -w)

src/os/darwin/src/darwinPlatform.c → src/os/src/darwin64/darwinPlatform.c

@@ -15,12 +15,15 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tglobal.h"
+#include "os.h"
 #include "taosdef.h"
-#include "tutil.h"
+#include "tglobal.h"
+#include "tconfig.h"
+#include "ttimer.h"
 #include "tulog.h"
+#include "tutil.h"
 
-int64_t str2int64(char *str) {
+int64_t tsosStr2int64(char *str) {
   char *endptr = NULL;
   return strtoll(str, &endptr, 10);
 }
@@ -159,12 +162,14 @@ int taosInitTimer(void (*callback)(int), int ms) {
   tv.it_interval.tv_usec = 1000 * ms;  // resolution is in msecond
   tv.it_value = tv.it_interval;
 
-  return setitimer(ITIMER_REAL, &tv, NULL);
+  setitimer(ITIMER_REAL, &tv, NULL);
+
+  return 0;
 }
 
 void taosUninitTimer() {
   struct itimerval tv = { 0 };
-  return setitimer(ITIMER_REAL, &tv, NULL);
+  setitimer(ITIMER_REAL, &tv, NULL);
 }
 
 void taosGetSystemTimezone() {
@@ -207,7 +212,7 @@ void taosGetSystemLocale() {
 void taosPrintOsInfo() {}
 
 void taosKillSystem() {
-  tError("function taosKillSystem, exit!");
+  uError("function taosKillSystem, exit!");
   exit(0);
 }
 
@@ -221,95 +226,75 @@ void taosGetSystemInfo() {
 }
 
 void *taosInitTcpClient(char *ip, uint16_t port, char *flabel, int num, void *fp, void *shandle) {
-  tError("function taosInitTcpClient is not implemented in darwin system, exit!");
+  uError("function taosInitTcpClient is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCloseTcpClientConnection(void *chandle) {
-  tError("function taosCloseTcpClientConnection is not implemented in darwin system, exit!");
+  uError("function taosCloseTcpClientConnection is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void *taosOpenTcpClientConnection(void *shandle, void *thandle, char *ip, uint16_t port) {
-  tError("function taosOpenTcpClientConnection is not implemented in darwin system, exit!");
+  uError("function taosOpenTcpClientConnection is not implemented in darwin system, exit!");
   exit(0);
 }
 
 int taosSendTcpClientData(unsigned int ip, uint16_t port, char *data, int len, void *chandle) {
-  tError("function taosSendTcpClientData is not implemented in darwin system, exit!");
+  uError("function taosSendTcpClientData is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCleanUpTcpClient(void *chandle) {
-  tError("function taosCleanUpTcpClient is not implemented in darwin system, exit!");
+  uError("function taosCleanUpTcpClient is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCloseTcpServerConnection(void *chandle) {
-  tError("function taosCloseTcpServerConnection is not implemented in darwin system, exit!");
+  uError("function taosCloseTcpServerConnection is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosCleanUpTcpServer(void *handle) {
-  tError("function taosCleanUpTcpServer is not implemented in darwin system, exit!");
+  uError("function taosCleanUpTcpServer is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void *taosInitTcpServer(char *ip, uint16_t port, char *label, int numOfThreads, void *fp, void *shandle) {
-  tError("function taosInitTcpServer is not implemented in darwin system, exit!");
+  uError("function taosInitTcpServer is not implemented in darwin system, exit!");
   exit(0);
 }
 
 int taosSendTcpServerData(unsigned int ip, uint16_t port, char *data, int len, void *chandle) {
-  tError("function taosSendTcpServerData is not implemented in darwin system, exit!");
+  uError("function taosSendTcpServerData is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosFreeMsgHdr(void *hdr) {
-  tError("function taosFreeMsgHdr is not implemented in darwin system, exit!");
+  uError("function taosFreeMsgHdr is not implemented in darwin system, exit!");
   exit(0);
 }
 
 int taosMsgHdrSize(void *hdr) {
-  tError("function taosMsgHdrSize is not implemented in darwin system, exit!");
+  uError("function taosMsgHdrSize is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosSendMsgHdr(void *hdr, int fd) {
-  tError("function taosSendMsgHdr is not implemented in darwin system, exit!");
+  uError("function taosSendMsgHdr is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosInitMsgHdr(void **hdr, void *dest, int maxPkts) {
-  tError("function taosInitMsgHdr is not implemented in darwin system, exit!");
+  uError("function taosInitMsgHdr is not implemented in darwin system, exit!");
   exit(0);
 }
 
 void taosSetMsgHdrData(void *hdr, char *data, int dataLen) {
-  tError("function taosSetMsgHdrData is not implemented in darwin system, exit!");
+  uError("function taosSetMsgHdrData is not implemented in darwin system, exit!");
   exit(0);
 }
 
-ssize_t twrite(int fd, void *buf, size_t n) {
-  size_t nleft = n; 
-  ssize_t nwritten = 0;
-  char *tbuf = (char *)buf;
-
-  while (nleft > 0) {
-    nwritten = write(fd, (void *)tbuf, nleft);
-    if (nwritten < 0) {
-      if (errno == EINTR) {
-        continue;
-      }
-      return -1;
-    }
-    nleft -= nwritten;
-    tbuf += nwritten;
-  }
-
-  return n;
-}
-
 bool taosSkipSocketCheck() {
   return true;
 }

src/os/src/detail/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(.)
+AUX_SOURCE_DIRECTORY(. SRC)
+SET_SOURCE_FILES_PROPERTIES(osSysinfo.c PROPERTIES COMPILE_FLAGS -w)
+SET_SOURCE_FILES_PROPERTIES(osCoredump.c PROPERTIES COMPILE_FLAGS -w)
+
+ADD_LIBRARY(osdetail ${SRC})
+TARGET_LINK_LIBRARIES(osdetail os)

src/os/src/detail/osCoredump.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tconfig.h"
+#include "tglobal.h"
+#include "tulog.h"
+#include "tutil.h"
+#include "tsystem.h"
+
+#ifndef TAOS_OS_FUNC_CORE
+
+int _sysctl(struct __sysctl_args *args );
+
+void taosSetCoreDump() {
+  if (0 == tsEnableCoreFile) {
+    return;
+  }
+  
+  // 1. set ulimit -c unlimited
+  struct rlimit rlim;
+  struct rlimit rlim_new;
+  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
+    uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
+    rlim_new.rlim_cur = RLIM_INFINITY;
+    rlim_new.rlim_max = RLIM_INFINITY;
+    if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) {
+      uInfo("set unlimited fail, error: %s", strerror(errno));
+      rlim_new.rlim_cur = rlim.rlim_max;
+      rlim_new.rlim_max = rlim.rlim_max;
+      (void)setrlimit(RLIMIT_CORE, &rlim_new);
+    }
+  }
+
+  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
+    uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
+  }
+
+#ifndef _TD_ARM_
+  // 2. set the path for saving core file
+  struct __sysctl_args args;
+  int     old_usespid = 0;
+  size_t  old_len     = 0;
+  int     new_usespid = 1;
+  size_t  new_len     = sizeof(new_usespid);
+  
+  int name[] = {CTL_KERN, KERN_CORE_USES_PID};
+  
+  memset(&args, 0, sizeof(struct __sysctl_args));
+  args.name    = name;
+  args.nlen    = sizeof(name)/sizeof(name[0]);
+  args.oldval  = &old_usespid;
+  args.oldlenp = &old_len;
+  args.newval  = &new_usespid;
+  args.newlen  = new_len;
+  
+  old_len = sizeof(old_usespid);
+  
+  if (syscall(SYS__sysctl, &args) == -1) {
+      uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno));
+  }
+  
+  uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
+
+
+  old_usespid = 0;
+  old_len     = 0;
+  memset(&args, 0, sizeof(struct __sysctl_args));
+  args.name    = name;
+  args.nlen    = sizeof(name)/sizeof(name[0]);
+  args.oldval  = &old_usespid;
+  args.oldlenp = &old_len;
+  
+  old_len = sizeof(old_usespid);
+  
+  if (syscall(SYS__sysctl, &args) == -1) {
+      uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno));
+  }
+  
+  uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
+#endif
+
+}
+
+#endif
\ No newline at end of file

src/util/src/tfile.c → src/os/src/detail/osFail.c

@@ -13,31 +13,62 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdio.h>
-#include <stdlib.h>
-#include <error.h>
-#include <errno.h>
-#include <stdarg.h>
-#include <unistd.h>
+#define _DEFAULT_SOURCE
+#include "os.h"
 
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
+#define RANDOM_NETWORK_FAIL_FACTOR  20
 
-#include "os.h"
+#ifdef TAOS_RANDOM_NETWORK_FAIL
+
+ssize_t taosSendRandomFail(int sockfd, const void *buf, size_t len, int flags) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = ECONNRESET;
+    return -1;
+  }
+
+  return send(sockfd, buf, len, flags);
+}
+
+ssize_t taosSendToRandomFail(int sockfd, const void *buf, size_t len, int flags, const struct sockaddr *dest_addr,
+                           socklen_t addrlen) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = ECONNRESET;
+    return -1;
+  }
+
+  return sendto(sockfd, buf, len, flags, dest_addr, addrlen);
+}
+
+ssize_t taosReadSocketRandomFail(int fd, void *buf, size_t count) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = ECONNRESET;
+    return -1;
+  }
+
+  return read(fd, buf, count);
+}
+
+ssize_t taosWriteSocketRandomFail(int fd, const void *buf, size_t count) {
+  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
+    errno = EINTR;
+    return -1;
+  }
+
+  return write(fd, buf, count);
+}
+
+#endif //TAOS_RANDOM_NETWORK_FAIL
 
 #ifdef TAOS_RANDOM_FILE_FAIL
 
 static int random_file_fail_factor = 20;
 static FILE *fpRandomFileFailOutput = NULL;
 
-void taosSetRandomFileFailFactor(int factor)
-{
+void taosSetRandomFileFailFactor(int factor) { 
   random_file_fail_factor = factor;
 }
 
-static void close_random_file_fail_output()
-{
+static void close_random_file_fail_output() {
   if (fpRandomFileFailOutput != NULL) {
     if (fpRandomFileFailOutput != stdout) {
       fclose(fpRandomFileFailOutput);
@@ -46,8 +77,7 @@ static void close_random_file_fail_output()
   }
 }
 
-static void random_file_fail_output_sig(int sig)
-{
+static void random_file_fail_output_sig(int sig) {
   fprintf(fpRandomFileFailOutput, "signal %d received.\n", sig);
 
   struct sigaction act = {0};
@@ -58,8 +88,7 @@ static void random_file_fail_output_sig(int sig)
   exit(EXIT_FAILURE);
 }
 
-void taosSetRandomFileFailOutput(const char *path)
-{
+void taosSetRandomFileFailOutput(const char *path) {
   if (path == NULL) {
     fpRandomFileFailOutput = stdout;
   } else if ((fpRandomFileFailOutput = fopen(path, "w")) != NULL) {
@@ -75,43 +104,38 @@ void taosSetRandomFileFailOutput(const char *path)
   sigaction(SIGSEGV, &act, NULL);
   sigaction(SIGILL, &act, NULL);
 }
-#endif
 
-ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line)
-{
-#ifdef TAOS_RANDOM_FILE_FAIL
+ssize_t taosReadFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
   if (random_file_fail_factor > 0) {
     if (rand() % random_file_fail_factor == 0) {
       errno = EIO;
       return -1;
     }
   }
-#endif
-  return tread(fd, buf, count);
+
+  return taosTReadImp(fd, buf, count);
 }
 
-ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line)
-{
-#ifdef TAOS_RANDOM_FILE_FAIL
+ssize_t taosWriteFileRandomFail(int fd, void *buf, size_t count, const char *file, uint32_t line) {
   if (random_file_fail_factor > 0) {
     if (rand() % random_file_fail_factor == 0) {
       errno = EIO;
       return -1;
     }
   }
-#endif
-  return twrite(fd, buf, count);
+
+  return taosTWriteImp(fd, buf, count);
 }
 
-off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line)
-{
-#ifdef TAOS_RANDOM_FILE_FAIL
+off_t taosLSeekRandomFail(int fd, off_t offset, int whence, const char *file, uint32_t line) {
   if (random_file_fail_factor > 0) {
     if (rand() % random_file_fail_factor == 0) {
       errno = EIO;
       return -1;
     }
   }
-#endif
+
   return lseek(fd, offset, whence);
 }
+
+#endif //TAOS_RANDOM_FILE_FAIL

src/mnode/src/mnodeBalance.c → src/os/src/detail/osFile.c

@@ -15,56 +15,74 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tglobal.h"
-#include "mnodeDef.h"
-#include "mnodeInt.h"
-#include "mnodeDnode.h"
-#include "mnodeSdb.h"
 
-#ifndef _SYNC
+ssize_t taosTReadImp(int fd, void *buf, size_t count) {
+  size_t  leftbytes = count;
+  ssize_t readbytes;
+  char *  tbuf = (char *)buf;
 
-int32_t balanceInit() { return TSDB_CODE_SUCCESS; }
-void    balanceCleanUp() {}
-void    balanceAsyncNotify() {}
-void    balanceSyncNotify() {}
-void    balanceReset() {}
-int32_t balanceAlterDnode(struct SDnodeObj *pDnode, int32_t vnodeId, int32_t dnodeId) { return TSDB_CODE_SYN_NOT_ENABLED; }
+  while (leftbytes > 0) {
+    readbytes = read(fd, (void *)tbuf, leftbytes);
+    if (readbytes < 0) {
+      if (errno == EINTR) {
+        continue;
+      } else {
+        return -1;
+      }
+    } else if (readbytes == 0) {
+      return (ssize_t)(count - leftbytes);
+    }
 
-int32_t balanceAllocVnodes(SVgObj *pVgroup) {
-  void *     pIter = NULL;
-  SDnodeObj *pDnode = NULL;
-  SDnodeObj *pSelDnode = NULL;
-  float      vnodeUsage = 1000.0;
+    leftbytes -= readbytes;
+    tbuf += readbytes;
+  }
 
-  while (1) {
-    pIter = mnodeGetNextDnode(pIter, &pDnode);
-    if (pDnode == NULL) break;
+  return (ssize_t)count;
+}
 
-    if (pDnode->numOfCores > 0 && pDnode->openVnodes < TSDB_MAX_VNODES) {
-      float openVnodes = pDnode->openVnodes;
-      if (pDnode->isMgmt) openVnodes += tsMnodeEqualVnodeNum;
+ssize_t taosTSendFileImp(int dfd, int sfd, off_t *offset, size_t size) {
+  size_t  leftbytes = size;
+  ssize_t sentbytes;
 
-      float usage = openVnodes / pDnode->numOfCores;
-      if (usage <= vnodeUsage) {
-        pSelDnode = pDnode;
-        vnodeUsage = usage;
+  while (leftbytes > 0) {
+    /*
+     * TODO : Think to check if file is larger than 1GB
+     */
+    //if (leftbytes > 1000000000) leftbytes = 1000000000;
+    sentbytes = sendfile(dfd, sfd, offset, leftbytes);
+    if (sentbytes == -1) {
+      if (errno == EINTR) {
+        continue;
+      }
+      else {
+        return -1;
       }
+    } else if (sentbytes == 0) {
+      return (ssize_t)(size - leftbytes);
     }
-    mnodeDecDnodeRef(pDnode);
+
+    leftbytes -= sentbytes;
   }
 
-  sdbFreeIter(pIter);
+  return size;
+}
 
-  if (pSelDnode == NULL) {
-    mError("failed to alloc vnode to vgroup");
-    return TSDB_CODE_MND_NO_ENOUGH_DNODES;
-  }
+ssize_t taosTWriteImp(int fd, void *buf, size_t n) {
+  size_t nleft = n; 
+  ssize_t nwritten = 0;
+  char *tbuf = (char *)buf;
 
-  pVgroup->vnodeGid[0].dnodeId = pSelDnode->dnodeId;
-  pVgroup->vnodeGid[0].pDnode = pSelDnode;
+  while (nleft > 0) {
+    nwritten = write(fd, (void *)tbuf, nleft);
+    if (nwritten < 0) {
+      if (errno == EINTR) {
+        continue;
+      }
+      return -1;
+    }
+    nleft -= nwritten;
+    tbuf += nwritten;
+  }
 
-  mDebug("dnode:%d, alloc one vnode to vgroup, openVnodes:%d", pSelDnode->dnodeId, pSelDnode->openVnodes);
-  return TSDB_CODE_SUCCESS;
+  return n;
 }
-
-#endif 

src/os/src/detail/osPThread.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_PTHREAD
+
+bool    taosCheckPthreadValid(pthread_t thread) { return thread != 0; }
+int64_t taosGetPthreadId() { return (int64_t)pthread_self(); }
+
+#endif
\ No newline at end of file

src/os/src/detail/osSocket.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tulog.h"
+
+#ifndef TAOS_OS_FUNC_SOCKET
+
+int taosSetNonblocking(int sock, int on) {
+  int flags = 0;
+  if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
+    uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
+    return 1;
+  }
+
+  if (on)
+    flags |= O_NONBLOCK;
+  else
+    flags &= ~O_NONBLOCK;
+
+  if ((flags = fcntl(sock, F_SETFL, flags)) < 0) {
+    uError("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno));
+    return 1;
+  }
+
+  return 0;
+}
+
+int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) {
+  return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
+}
+
+void taosBlockSIGPIPE() {
+  sigset_t signal_mask;
+  sigemptyset(&signal_mask);
+  sigaddset(&signal_mask, SIGPIPE);
+  int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
+  if (rc != 0) {
+    uError("failed to block SIGPIPE");
+  }
+}
+
+#endif
\ No newline at end of file

src/os/linux/src/linuxSysPara.c → src/os/src/detail/osSysinfo.c

@@ -15,13 +15,14 @@
 
 #define _DEFAULT_SOURCE
 #include "os.h"
-#include "tsysctl.h"
 #include "tconfig.h"
 #include "tglobal.h"
 #include "tulog.h"
 #include "tutil.h"
 #include "tsystem.h"
 
+#ifndef TAOS_OS_FUNC_SYSINFO
+
 #define PROCESS_ITEM 12
 
 typedef struct {
@@ -200,7 +201,7 @@ static void taosGetSystemTimezone() {
    * Asia/Shanghai   (CST, +0800)
    * Europe/London   (BST, +0100)
    */
-  sprintf(tsTimezone, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
+  snprintf(tsTimezone, TSDB_TIMEZONE_LEN, "%s (%s, %s%02d00)", buf, tzname[daylight], tz >= 0 ? "+" : "-", abs(tz));
 
   // cfg_timezone->cfgStatus = TAOS_CFG_CSTATUS_DEFAULT;
   uInfo("timezone not configured, set to system default:%s", tsTimezone);
@@ -234,7 +235,7 @@ static void taosGetSystemLocale() {  // get and set default locale
       uError("can't get locale from system, set it to en_US.UTF-8");
       strcpy(tsLocale, "en_US.UTF-8");
     } else {
-      tstrncpy(tsLocale, locale, sizeof(tsLocale));
+      tstrncpy(tsLocale, locale, TSDB_LOCALE_LEN);
       uError("locale not configured, set to system default:%s", tsLocale);
     }
   }
@@ -247,7 +248,7 @@ static void taosGetSystemLocale() {  // get and set default locale
       str++;
 
       char *revisedCharset = taosCharsetReplace(str);
-      tstrncpy(tsCharset, revisedCharset, sizeof(tsCharset));
+      tstrncpy(tsCharset, revisedCharset, TSDB_LOCALE_LEN);
 
       free(revisedCharset);
       uWarn("charset not configured, set to system default:%s", tsCharset);
@@ -542,126 +543,31 @@ void taosKillSystem() {
   kill(tsProcId, 2);
 }
 
-int _sysctl(struct __sysctl_args *args );
-void taosSetCoreDump() {
-  if (0 == tsEnableCoreFile) {
-    return;
-  }
-  
-  // 1. set ulimit -c unlimited
-  struct rlimit rlim;
-  struct rlimit rlim_new;
-  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
-    uInfo("the old unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
-    rlim_new.rlim_cur = RLIM_INFINITY;
-    rlim_new.rlim_max = RLIM_INFINITY;
-    if (setrlimit(RLIMIT_CORE, &rlim_new) != 0) {
-      uInfo("set unlimited fail, error: %s", strerror(errno));
-      rlim_new.rlim_cur = rlim.rlim_max;
-      rlim_new.rlim_max = rlim.rlim_max;
-      (void)setrlimit(RLIMIT_CORE, &rlim_new);
-    }
-  }
-
-  if (getrlimit(RLIMIT_CORE, &rlim) == 0) {
-    uInfo("the new unlimited para: rlim_cur=%" PRIu64 ", rlim_max=%" PRIu64, rlim.rlim_cur, rlim.rlim_max);
+int tSystem(const char *cmd) {
+  FILE *fp;
+  int   res;
+  char  buf[1024];
+  if (cmd == NULL) {
+    uError("tSystem cmd is NULL!\n");
+    return -1;
   }
 
-#ifndef _TD_ARM_
-  // 2. set the path for saving core file
-  struct __sysctl_args args;
-  int     old_usespid = 0;
-  size_t  old_len     = 0;
-  int     new_usespid = 1;
-  size_t  new_len     = sizeof(new_usespid);
-  
-  int name[] = {CTL_KERN, KERN_CORE_USES_PID};
-  
-  memset(&args, 0, sizeof(struct __sysctl_args));
-  args.name    = name;
-  args.nlen    = sizeof(name)/sizeof(name[0]);
-  args.oldval  = &old_usespid;
-  args.oldlenp = &old_len;
-  args.newval  = &new_usespid;
-  args.newlen  = new_len;
-  
-  old_len = sizeof(old_usespid);
-  
-  if (syscall(SYS__sysctl, &args) == -1) {
-      uInfo("_sysctl(kern_core_uses_pid) set fail: %s", strerror(errno));
-  }
-  
-  uInfo("The old core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
-
+  if ((fp = popen(cmd, "r")) == NULL) {
+    uError("popen cmd:%s error: %s/n", cmd, strerror(errno));
+    return -1;
+  } else {
+    while (fgets(buf, sizeof(buf), fp)) {
+      uDebug("popen result:%s", buf);
+    }
 
-  old_usespid = 0;
-  old_len     = 0;
-  memset(&args, 0, sizeof(struct __sysctl_args));
-  args.name    = name;
-  args.nlen    = sizeof(name)/sizeof(name[0]);
-  args.oldval  = &old_usespid;
-  args.oldlenp = &old_len;
-  
-  old_len = sizeof(old_usespid);
-  
-  if (syscall(SYS__sysctl, &args) == -1) {
-      uInfo("_sysctl(kern_core_uses_pid) get fail: %s", strerror(errno));
-  }
-  
-  uInfo("The new core_uses_pid[%" PRIu64 "]: %d", old_len, old_usespid);
-#endif
-  
-#if 0
-  // 3. create the path for saving core file
-  int status; 
-  char coredump_dir[32] = "/var/log/taosdump";
-  if (opendir(coredump_dir) == NULL) {
-    status = mkdir(coredump_dir, S_IRWXU | S_IRWXG | S_IRWXO); 
-    if (status) {
-      uInfo("mkdir fail, error: %s\n", strerror(errno));
+    if ((res = pclose(fp)) == -1) {
+      uError("close popen file pointer fp error!\n");
+    } else {
+      uDebug("popen res is :%d\n", res);
     }
-  }
 
-  // 4. set kernel.core_pattern
-   struct __sysctl_args args;
-   char    old_corefile[128];
-   size_t  old_len;
-   char    new_corefile[128] = "/var/log/taosdump/core-%e-%p";
-   size_t  new_len = sizeof(new_corefile);
-   
-   int name[] = {CTL_KERN, KERN_CORE_PATTERN};
-
-   memset(&args, 0, sizeof(struct __sysctl_args));
-   args.name    = name;
-   args.nlen    = sizeof(name)/sizeof(name[0]);
-   args.oldval  = old_corefile;
-   args.oldlenp = &old_len;
-   args.newval  = new_corefile;
-   args.newlen  = new_len;
-
-   old_len = sizeof(old_corefile);
-
-   if (syscall(SYS__sysctl, &args) == -1) {
-       uInfo("_sysctl(kern_core_pattern) set fail: %s", strerror(errno));
-   }
-   
-   uInfo("The old kern_core_pattern: %*s\n", old_len, old_corefile);
-
-
-   memset(&args, 0, sizeof(struct __sysctl_args));
-   args.name    = name;
-   args.nlen    = sizeof(name)/sizeof(name[0]);
-   args.oldval  = old_corefile;
-   args.oldlenp = &old_len;
-   
-   old_len = sizeof(old_corefile);
-
-   if (syscall(SYS__sysctl, &args) == -1) {
-       uInfo("_sysctl(kern_core_pattern) get fail: %s", strerror(errno));
-   }
-   
-   uInfo("The new kern_core_pattern: %*s\n", old_len, old_corefile);
-#endif
-  
+    return res;
+  }
 }
 
+#endif
\ No newline at end of file

src/os/linux/src/linuxPlatform.c → src/os/src/detail/osTimer.c

@@ -13,18 +13,15 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#define _DEFAULT_SOURCE
 #include "os.h"
 #include "taosdef.h"
 #include "tglobal.h"
 #include "ttimer.h"
 #include "tulog.h"
 #include "tutil.h"
+#include <signal.h>
 
-int64_t str2int64(char *str) {
-  char *endptr = NULL;
-  return strtoll(str, &endptr, 10);
-}
+#ifndef TAOS_OS_FUNC_TIMER
 
 /*
   to make taosMsleep work,
@@ -54,33 +51,7 @@ void taosMsleep(int mseconds) {
   /* pthread_sigmask(SIG_UNBLOCK, &set, NULL); */
 }
 
-bool taosCheckPthreadValid(pthread_t thread) { return thread != 0; }
-
-int64_t taosGetPthreadId() { return (int64_t)pthread_self(); }
-
-int taosSetNonblocking(int sock, int on) {
-  int flags = 0;
-  if ((flags = fcntl(sock, F_GETFL, 0)) < 0) {
-    uError("fcntl(F_GETFL) error: %d (%s)\n", errno, strerror(errno));
-    return 1;
-  }
-
-  if (on)
-    flags |= O_NONBLOCK;
-  else
-    flags &= ~O_NONBLOCK;
 
-  if ((flags = fcntl(sock, F_SETFL, flags)) < 0) {
-    uError("fcntl(F_SETFL) error: %d (%s)\n", errno, strerror(errno));
-    return 1;
-  }
-
-  return 0;
-}
-
-int taosSetSockOpt(int socketfd, int level, int optname, void *optval, int optlen) {
-  return setsockopt(socketfd, level, optname, optval, (socklen_t)optlen);
-}
 static void taosDeleteTimer(void *tharg) {
   timer_t *pTimer = tharg;
   timer_delete(*pTimer);
@@ -89,8 +60,7 @@ static void taosDeleteTimer(void *tharg) {
 static pthread_t timerThread;
 static timer_t         timerId;
 static volatile bool stopTimer = false;
-
-void *taosProcessAlarmSignal(void *tharg) {
+static void *taosProcessAlarmSignal(void *tharg) {
   // Block the signal
   sigset_t sigset;
   sigemptyset(&sigset);
@@ -143,7 +113,6 @@ void *taosProcessAlarmSignal(void *tharg) {
   return NULL;
 }
 
-
 int taosInitTimer(void (*callback)(int), int ms) {
   pthread_attr_t tattr;
   pthread_attr_init(&tattr);
@@ -161,158 +130,4 @@ void taosUninitTimer() {
   pthread_join(timerThread, NULL);
 }
 
-ssize_t tread(int fd, void *buf, size_t count) {
-  size_t  leftbytes = count;
-  ssize_t readbytes;
-  char *  tbuf = (char *)buf;
-
-  while (leftbytes > 0) {
-    readbytes = read(fd, (void *)tbuf, leftbytes);
-    if (readbytes < 0) {
-      if (errno == EINTR) {
-        continue;
-      } else {
-        return -1;
-      }
-    } else if (readbytes == 0) {
-      return (ssize_t)(count - leftbytes);
-    }
-
-    leftbytes -= readbytes;
-    tbuf += readbytes;
-  }
-
-  return (ssize_t)count;
-}
-
-ssize_t tsendfile(int dfd, int sfd, off_t *offset, size_t size) {
-  size_t  leftbytes = size;
-  ssize_t sentbytes;
-
-  while (leftbytes > 0) {
-    /*
-     * TODO : Think to check if file is larger than 1GB
-     */
-    //if (leftbytes > 1000000000) leftbytes = 1000000000;
-    sentbytes = sendfile(dfd, sfd, offset, leftbytes);
-    if (sentbytes == -1) {
-      if (errno == EINTR) {
-        continue;
-      }
-      else {
-        return -1;
-      }
-    } else if (sentbytes == 0) {
-      return (ssize_t)(size - leftbytes);
-    }
-
-    leftbytes -= sentbytes;
-  }
-
-  return size;
-}
-
-ssize_t twrite(int fd, void *buf, size_t n) {
-  size_t nleft = n; 
-  ssize_t nwritten = 0;
-  char *tbuf = (char *)buf;
-
-  while (nleft > 0) {
-    nwritten = write(fd, (void *)tbuf, nleft);
-    if (nwritten < 0) {
-      if (errno == EINTR) {
-        continue;
-      }
-      return -1;
-    }
-    nleft -= nwritten;
-    tbuf += nwritten;
-  }
-
-  return n;
-}
-
-void taosBlockSIGPIPE() {
-  sigset_t signal_mask;
-  sigemptyset(&signal_mask);
-  sigaddset(&signal_mask, SIGPIPE);
-  int rc = pthread_sigmask(SIG_BLOCK, &signal_mask, NULL);
-  if (rc != 0) {
-    uError("failed to block SIGPIPE");
-  }
-}
-
-int tSystem(const char * cmd) 
-{ 
-  FILE * fp; 
-  int res; 
-  char buf[1024]; 
-  if (cmd == NULL) { 
-    uError("tSystem cmd is NULL!\n");
-    return -1;
-  } 
-  
-  if ((fp = popen(cmd, "r") ) == NULL) { 
-    uError("popen cmd:%s error: %s/n", cmd, strerror(errno)); 
-    return -1; 
-  } else {
-    while(fgets(buf, sizeof(buf), fp))  { 
-      uDebug("popen result:%s", buf); 
-    } 
-
-    if ((res = pclose(fp)) == -1) { 
-      uError("close popen file pointer fp error!\n");
-    } else { 
-      uDebug("popen res is :%d\n", res);
-    } 
-
-    return res;
-  }
-}
-
-#ifdef TAOS_RANDOM_NETWORK_FAIL
-
-#define RANDOM_NETWORK_FAIL_FACTOR  20
-
-ssize_t taos_send_random_fail(int sockfd, const void *buf, size_t len, int flags)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = ECONNRESET;
-    return -1;
-  }
-
-  return send(sockfd, buf, len, flags);
-}
-
-ssize_t taos_sendto_random_fail(int sockfd, const void *buf, size_t len, int flags,
-                      const struct sockaddr *dest_addr, socklen_t addrlen)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = ECONNRESET;
-    return -1;
-  }
-
-  return sendto(sockfd, buf, len, flags, dest_addr, addrlen);
-}
-
-ssize_t taos_read_random_fail(int fd, void *buf, size_t count)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = ECONNRESET;
-    return -1;
-  }
-
-  return read(fd, buf, count);
-}
-
-ssize_t taos_write_random_fail(int fd, const void *buf, size_t count)
-{
-  if (rand() % RANDOM_NETWORK_FAIL_FACTOR == 0) {
-    errno = EINTR;
-    return -1;
-  }
-
-  return write(fd, buf, count);
-}
-
-#endif /* TAOS_RANDOM_NETWORK_FAIL */
+#endif
\ No newline at end of file

src/os/src/detail/osUtil.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+
+#ifndef TAOS_OS_FUNC_UTIL
+
+int64_t tsosStr2int64(char *str) {
+  char *endptr = NULL;
+  return strtoll(str, &endptr, 10);
+}
+
+#endif
\ No newline at end of file

src/os/src/linux64/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+AUX_SOURCE_DIRECTORY(. SRC)
+
+ADD_LIBRARY(os ${SRC})
+TARGET_LINK_LIBRARIES(os m rt)
+

src/os/src/linux64/linuxEnv.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#define _DEFAULT_SOURCE
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+
+void osInit() {
+  strcpy(configDir, "/etc/taos");
+  strcpy(tsVnodeDir, "");
+  strcpy(tsDnodeDir, "");
+  strcpy(tsMnodeDir, "");
+  strcpy(tsDataDir, "/var/lib/taos");
+  strcpy(tsLogDir, "/var/log/taos");
+  strcpy(tsScriptDir, "/etc/taos");
+  strcpy(tsOsName, "Linux");
+}
\ No newline at end of file

src/os/windows/src/twindows.c → src/os/src/windows64/twindows.c

@@ -398,7 +398,7 @@ char *strndup(const char *s, size_t n) {
 void taosSetCoreDump() {}
 
 #ifdef _TD_GO_DLL_
-int64_t str2int64(char *str) {
+int64_t tsosStr2int64(char *str) {
   char *endptr = NULL;
   return strtoll(str, &endptr, 10);
 }

src/plugins/http/CMakeLists.txt

@@ -3,6 +3,7 @@ PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
@@ -10,7 +11,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/zlib-1.2.11/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(http ${SRC})

src/plugins/http/src/httpJson.c

@@ -45,7 +45,7 @@ int httpWriteBufByFd(struct HttpContext* pContext, const char* buf, int sz) {
 
   do {
     if (pContext->fd > 2){
-      len = (int)send(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL);
+      len = (int)taosSend(pContext->fd, buf + writeLen, (size_t)(sz - writeLen), MSG_NOSIGNAL);
     }
     else {
       return sz;

src/plugins/monitor/CMakeLists.txt

@@ -3,11 +3,11 @@ PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(./src SRC)
   ADD_LIBRARY(monitor ${SRC})

src/plugins/mqtt/CMakeLists.txt

@@ -3,6 +3,7 @@ PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
@@ -10,7 +11,6 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/include)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/MQTT-C/examples/templates)
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(mqtt ${SRC})

src/query/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)

src/query/src/qTsbuf.c

@@ -740,7 +740,7 @@ int32_t tsBufMerge(STSBuf* pDestBuf, const STSBuf* pSrcBuf, int32_t vnodeId) {
   int32_t size = pSrcBuf->fileSize - offset;
 
 #ifdef LINUX
-  ssize_t rc = tsendfile(fileno(pDestBuf->f), fileno(pSrcBuf->f), &offset, size);
+  ssize_t rc = taosTSendFile(fileno(pDestBuf->f), fileno(pSrcBuf->f), &offset, size);
 #else
   ssize_t rc = fsendfile(pDestBuf->f, pSrcBuf->f, &offset, size);
 #endif

src/rpc/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
@@ -12,10 +12,11 @@ INCLUDE_DIRECTORIES(inc)
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   AUX_SOURCE_DIRECTORY(./src SRC)
 ELSEIF (TD_DARWIN_64)
-  LIST(APPEND SRC ./src/thaship.c)
-  LIST(APPEND SRC ./src/trpc.c)
-  LIST(APPEND SRC ./src/tstring.c)
-  LIST(APPEND SRC ./src/tudp.c)
+  #LIST(APPEND SRC ./src/thaship.c)
+  #LIST(APPEND SRC ./src/trpc.c)
+  #LIST(APPEND SRC ./src/tstring.c)
+  #LIST(APPEND SRC ./src/tudp.c)
+  AUX_SOURCE_DIRECTORY(src SRC)
 ELSEIF (TD_WINDOWS_64)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/pthread)
   LIST(APPEND SRC ./src/thaship.c)

src/rpc/src/rpcMain.c

@@ -626,6 +626,7 @@ static void rpcReleaseConn(SRpcConn *pConn) {
   pConn->pReqMsg = NULL;
   pConn->reqMsgLen = 0;
   pConn->pContext = NULL;
+  pConn->chandle = NULL;
 
   taosFreeId(pRpc->idPool, pConn->sid);
   tDebug("%s, rpc connection is released", pConn->info);
@@ -656,7 +657,7 @@ static SRpcConn *rpcAllocateClientConn(SRpcInfo *pRpc) {
     pConn->sid = sid;
     pConn->tranId = (uint16_t)(random() & 0xFFFF);
     pConn->ownId = htonl(pConn->sid);
-    pConn->linkUid = (uint32_t)((int64_t)pConn + (int64_t)getpid());
+    pConn->linkUid = (uint32_t)((int64_t)pConn + (int64_t)getpid() + (int64_t)pConn->tranId);
     pConn->spi = pRpc->spi;
     pConn->encrypt = pRpc->encrypt;
     if (pConn->spi) memcpy(pConn->secret, pRpc->secret, TSDB_KEY_LEN);
@@ -913,7 +914,7 @@ static SRpcConn *rpcProcessMsgHead(SRpcInfo *pRpc, SRecvInfo *pRecv) {
   }
 
   sid = pConn->sid;
-  pConn->chandle = pRecv->chandle;
+  if (pConn->chandle == NULL) pConn->chandle = pRecv->chandle;
   pConn->peerIp = pRecv->ip; 
   pConn->peerPort = pRecv->port;
   if (pHead->port) pConn->peerPort = htons(pHead->port); 
@@ -1015,7 +1016,7 @@ static void *rpcProcessMsgFromPeer(SRecvInfo *pRecv) {
     if (code != 0) { // parsing error
       if (rpcIsReq(pHead->msgType)) {
         rpcSendErrorMsgToPeer(pRecv, code);
-        tDebug("%s %p %p, %s is sent with error code:%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
+        tDebug("%s %p %p, %s is sent with error code:0x%x", pRpc->label, pConn, (void *)pHead->ahandle, taosMsg[pHead->msgType+1], code);
       } 
     } else { // msg is passed to app only parsing is ok 
       rpcProcessIncomingMsg(pConn, pHead);

src/rpc/src/rpcUdp.c

@@ -242,7 +242,7 @@ int taosSendUdpData(uint32_t ip, uint16_t port, void *data, int dataLen, void *c
   destAdd.sin_addr.s_addr = ip;
   destAdd.sin_port = htons(port);
 
-  int ret = (int)sendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
+  int ret = (int)taosSendto(pConn->fd, data, (size_t)dataLen, 0, (struct sockaddr *)&destAdd, sizeof(destAdd));
 
   return ret;
 }

src/rpc/test/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
-  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-  INCLUDE_DIRECTORIES(../inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/rpc/inc)
 
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   LIST(APPEND CLIENT_SRC ./rclient.c)
   ADD_EXECUTABLE(rclient ${CLIENT_SRC})
   TARGET_LINK_LIBRARIES(rclient trpc)

src/sync/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(${TD_ENTERPRISE_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(inc)
+  AUX_SOURCE_DIRECTORY(src SRC)
+  LIST(REMOVE_ITEM SRC ./src/tarbitrator.c)
+
+  ADD_LIBRARY(sync ${SRC})
+  TARGET_LINK_LIBRARIES(sync tutil pthread common)
+
+  LIST(APPEND BIN_SRC ./src/tarbitrator.c)
+  LIST(APPEND BIN_SRC ./src/taosTcpPool.c)
+  ADD_EXECUTABLE(tarbitrator ${BIN_SRC})
+  TARGET_LINK_LIBRARIES(tarbitrator tutil sync common)
+
+  ADD_SUBDIRECTORY(test)
+ENDIF ()

src/sync/inc/syncInt.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_SYNCINT_H
+#define TDENGINE_SYNCINT_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define sFatal(...) { if (sDebugFlag & DEBUG_FATAL) { taosPrintLog("SYN FATAL ", sDebugFlag, __VA_ARGS__); }}
+#define sError(...) { if (sDebugFlag & DEBUG_ERROR) { taosPrintLog("SYN ERROR ", sDebugFlag, __VA_ARGS__); }}
+#define sWarn(...)  { if (sDebugFlag & DEBUG_WARN)  { taosPrintLog("SYN WARN ", sDebugFlag, __VA_ARGS__); }}
+#define sInfo(...)  { if (sDebugFlag & DEBUG_INFO)  { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
+#define sDebug(...) { if (sDebugFlag & DEBUG_DEBUG) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
+#define sTrace(...) { if (sDebugFlag & DEBUG_TRACE) { taosPrintLog("SYN ", sDebugFlag, __VA_ARGS__); }}
+
+#define TAOS_SMSG_SYNC_DATA    1
+#define TAOS_SMSG_FORWARD      2
+#define TAOS_SMSG_FORWARD_RSP  3
+#define TAOS_SMSG_SYNC_REQ     4 
+#define TAOS_SMSG_SYNC_RSP     5
+#define TAOS_SMSG_SYNC_MUST    6
+#define TAOS_SMSG_STATUS       7
+
+#define nodeRole    pNode->peerInfo[pNode->selfIndex]->role
+#define nodeVersion pNode->peerInfo[pNode->selfIndex]->version
+#define nodeSStatus pNode->peerInfo[pNode->selfIndex]->sstatus
+
+#pragma pack(push, 1)
+
+typedef struct {
+  char     type;        // msg type
+  char     pversion;    // protocol version
+  char     reserved[6]; // not used
+  int32_t  vgId;        // vg ID
+  int32_t  len;         // content length, does not include head
+  // char     cont[];      // message content starts from here
+} SSyncHead;
+
+typedef struct {
+  SSyncHead syncHead;
+  uint16_t  port;
+  char      fqdn[TSDB_FQDN_LEN];
+  int32_t   sourceId;  // only for arbitrator
+} SFirstPkt;
+
+typedef struct {
+  int8_t    role;
+  uint64_t  version;
+} SPeerStatus;
+
+typedef struct {
+  int8_t      role;
+  int8_t      ack;
+  uint64_t    version;
+  SPeerStatus peersStatus[];
+} SPeersStatus;
+
+typedef struct {
+  char      name[TSDB_FILENAME_LEN];
+  uint32_t  magic;
+  uint32_t  index;
+  uint64_t  fversion;
+  int32_t   size;
+} SFileInfo;
+
+typedef struct {
+  int8_t    sync;
+} SFileAck;
+
+typedef struct {
+  uint64_t  version;
+  int32_t   code;
+} SFwdRsp;
+  
+#pragma pack(pop)
+
+typedef struct {
+  char  *buffer;
+  int    bufferSize;
+  char  *offset;
+  int    forwards;
+  int    code;
+} SRecvBuffer;
+
+typedef struct {
+  uint64_t  version;
+  void     *mhandle;
+  int8_t    acks;
+  int8_t    nacks;
+  int8_t    confirmed;
+  int32_t   code;
+  uint64_t  time;
+} SFwdInfo;
+
+typedef struct {
+  int       first;
+  int       last;
+  int       fwds;  // number of forwards
+  SFwdInfo  fwdInfo[];
+} SSyncFwds;
+
+typedef struct SsyncPeer {
+  int32_t     nodeId;
+  uint32_t    ip;
+  uint16_t    port;
+  char        fqdn[TSDB_FQDN_LEN];  // peer ip string
+  char        id[TSDB_EP_LEN+16];   // peer vgId + end point
+  int8_t      role;
+  int8_t      sstatus;    // sync status
+  uint64_t    version;
+  uint64_t    sversion;   // track the peer version in retrieve process 
+  int         syncFd;
+  int         peerFd;     // forward FD
+  void       *timer;
+  void       *pConn;
+  int         notifyFd;
+  int         watchNum;
+  int        *watchFd;
+  int8_t      refCount;   // reference count
+  struct SSyncNode *pSyncNode;
+} SSyncPeer;
+
+typedef struct SSyncNode {
+  char         path[TSDB_FILENAME_LEN];
+  int8_t       replica;
+  int8_t       quorum;
+  uint32_t     vgId;
+  void        *ahandle;
+  int8_t       selfIndex;
+  SSyncPeer   *peerInfo[TAOS_SYNC_MAX_REPLICA+1];  // extra one for arbitrator
+  SSyncPeer   *pMaster;
+  int8_t       refCount;
+  SRecvBuffer *pRecv;
+  SSyncFwds   *pSyncFwds;  // saved forward info if quorum >1
+  void        *pFwdTimer;
+  FGetFileInfo    getFileInfo;
+  FGetWalInfo     getWalInfo;
+  FWriteToCache   writeToCache;
+  FConfirmForward confirmForward;
+  FNotifyRole     notifyRole;
+  FNotifyFileSynced notifyFileSynced;
+  pthread_mutex_t mutex;
+} SSyncNode;
+
+// sync module global
+extern int  tsSyncNum;
+extern char tsNodeFqdn[TSDB_FQDN_LEN];
+
+void *syncRetrieveData(void *param);
+void *syncRestoreData(void *param);
+int   syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
+void  syncRestartConnection(SSyncPeer *pPeer);
+void  syncBroadcastStatus(SSyncNode *pNode);
+void  syncAddPeerRef(SSyncPeer *pPeer);
+int   syncDecPeerRef(SSyncPeer *pPeer);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_VNODEPEER_H

src/os/linux/inc/tsysctl.h → src/sync/inc/syncMain.h

@@ -13,18 +13,21 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#ifndef TDENGINE_TSYSCTL_H
-#define TDENGINE_TSYSCTL_H
+#ifndef TDENGINE_PLUGINS_SYNC_H
+#define TDENGINE_PLUGINS_SYNC_H
 
 #ifdef __cplusplus
 extern "C" {
 #endif
 
-#ifndef _ALPINE
-  #include <error.h>
-  #include <sys/sysctl.h>
-#else
-  #include <linux/sysctl.h>  
+#include <stdbool.h>
+#include <stdint.h>
+
+int32_t syncTest1();
+int32_t syncTest2();
+
+#ifdef __cplusplus
+}
 #endif
 
 #endif

src/sync/inc/taosTcpPool.h

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef TDENGINE_TCP_POOL_H
+#define TDENGINE_TCP_POOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef void* ttpool_h;
+typedef void* tthread_h;
+
+typedef struct {
+  int       numOfThreads;
+  uint32_t  serverIp;
+  short     port;
+  int       bufferSize;
+  void     (*processBrokenLink)(void *ahandle);
+  int      (*processIncomingMsg)(void *ahandle, void *buffer);
+  void     (*processIncomingConn)(int fd, uint32_t ip);
+} SPoolInfo;
+
+ttpool_h   taosOpenTcpThreadPool(SPoolInfo *pInfo);
+void       taosCloseTcpThreadPool(ttpool_h);
+void      *taosAllocateTcpConn(void *, void *ahandle, int connFd);
+void       taosFreeTcpConn(void *);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TDENGINE_TCP_POOL_H
+                                   

src/sync/src/syncMain.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//#include <stdint.h>
+//#include <stdbool.h>
+#include "os.h"
+#include "hash.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "ttimer.h"
+#include "ttime.h"
+#include "tsocket.h"
+#include "tglobal.h"
+#include "taoserror.h"
+#include "taosTcpPool.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+// global configurable
+int       tsMaxSyncNum = 4;
+int       tsSyncTcpThreads = 2;
+int       tsMaxWatchFiles = 100;
+int       tsMaxFwdInfo = 200;
+int       tsSyncTimer = 1;
+//int       sDebugFlag = 135;
+//char      tsArbitrator[TSDB_FQDN_LEN] = {0};
+
+// module global, not configurable
+int       tsSyncNum;    // number of sync in process in whole system
+char      tsNodeFqdn[TSDB_FQDN_LEN];
+
+static int            tsNodeNum;    // number of nodes in system
+static ttpool_h       tsTcpPool;
+static void          *syncTmrCtrl = NULL;
+static void          *vgIdHash;
+static pthread_once_t syncModuleInit = PTHREAD_ONCE_INIT;
+
+// local functions
+static void  syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
+static void  syncRecoverFromMaster(SSyncPeer *pPeer);
+static void  syncCheckPeerConnection(void *param, void *tmrId);
+static void  syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack);
+static void  syncProcessBrokenLink(void *param);
+static int   syncProcessPeerMsg(void *param, void *buffer);
+static void  syncProcessIncommingConnection(int connFd, uint32_t sourceIp); 
+static void  syncRemovePeer(SSyncPeer *pPeer);
+static void  syncAddArbitrator(SSyncNode *pNode);
+static void  syncAddNodeRef(SSyncNode *pNode);
+static void  syncDecNodeRef(SSyncNode *pNode);
+static void  syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
+static void  syncMonitorFwdInfos(void *param, void *tmrId);
+static void  syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code);
+static void  syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle); 
+static void  syncRestartPeer(SSyncPeer *pPeer);
+static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo);
+
+char* syncRole[] = {
+  "offline",
+  "unsynced",
+  "slave",
+  "master"
+};
+
+static void syncModuleInitFunc() {
+  SPoolInfo info;
+
+  info.numOfThreads = tsSyncTcpThreads;
+  info.serverIp = 0;
+  info.port = tsSyncPort;
+  info.bufferSize = 640000;
+  info.processBrokenLink = syncProcessBrokenLink;
+  info.processIncomingMsg = syncProcessPeerMsg;
+  info.processIncomingConn = syncProcessIncommingConnection;
+
+  tsTcpPool = taosOpenTcpThreadPool(&info);
+  if (tsTcpPool == NULL) return;
+
+  syncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
+  if (syncTmrCtrl == NULL) {
+    taosCloseTcpThreadPool(tsTcpPool);
+    tsTcpPool = NULL;
+    return;
+  }
+    
+  vgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true); 
+  if (vgIdHash == NULL) {
+    taosTmrCleanUp(syncTmrCtrl);
+    taosCloseTcpThreadPool(tsTcpPool);
+    tsTcpPool = NULL;
+    syncTmrCtrl = NULL;
+    return;
+  } 
+
+  tstrncpy(tsNodeFqdn, tsLocalFqdn, sizeof(tsNodeFqdn));
+}
+
+void *syncStart(const SSyncInfo *pInfo) 
+{
+  const SSyncCfg *pCfg = &pInfo->syncCfg;
+
+  SSyncNode *pNode = (SSyncNode *) calloc(sizeof(SSyncNode), 1);
+  if (pNode == NULL) {
+    sError("no memory to allocate syncNode");
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    return NULL;
+  }
+
+  pthread_once(&syncModuleInit, syncModuleInitFunc); 
+  if (tsTcpPool == NULL) {
+    free(pNode);
+    syncModuleInit = PTHREAD_ONCE_INIT;
+    sError("failed to init sync module(%s)", tstrerror(errno));
+    return NULL;
+  }
+
+  atomic_add_fetch_32(&tsNodeNum, 1);
+  tstrncpy(pNode->path, pInfo->path, sizeof(pNode->path));
+  pthread_mutex_init(&pNode->mutex, NULL);
+
+  pNode->ahandle = pInfo->ahandle;
+  pNode->getFileInfo = pInfo->getFileInfo;
+  pNode->getWalInfo = pInfo->getWalInfo;
+  pNode->writeToCache = pInfo->writeToCache;
+  pNode->notifyRole = pInfo->notifyRole;
+  pNode->confirmForward = pInfo->confirmForward;
+  pNode->notifyFileSynced = pInfo->notifyFileSynced;
+ 
+  pNode->selfIndex = -1;
+  pNode->vgId = pInfo->vgId;
+  pNode->replica = pCfg->replica;
+  pNode->quorum = pCfg->quorum;
+  for (int i = 0; i < pCfg->replica; ++i) {
+    const SNodeInfo *pNodeInfo = pCfg->nodeInfo + i;
+    pNode->peerInfo[i] = syncAddPeer(pNode, pNodeInfo);
+    if ((strcmp(pNodeInfo->nodeFqdn, tsNodeFqdn) == 0) && (pNodeInfo->nodePort == tsSyncPort)) 
+      pNode->selfIndex = i;
+  }
+
+  if (pNode->selfIndex < 0) {
+    sInfo("vgId:%d, this node is not configured", pNode->vgId);
+    terrno = TSDB_CODE_SYN_INVALID_CONFIG;
+    syncStop(pNode);
+    return NULL;
+  }
+
+  nodeVersion = pInfo->version;    // set the initial version
+  nodeRole = (pNode->replica > 1) ? TAOS_SYNC_ROLE_UNSYNCED : TAOS_SYNC_ROLE_MASTER;
+  sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]);
+
+  pNode->pSyncFwds = calloc(sizeof(SSyncFwds) + tsMaxFwdInfo*sizeof(SFwdInfo), 1);
+  if (pNode->pSyncFwds == NULL) {
+    sError("vgId:%d, no memory to allocate syncFwds", pNode->vgId);
+    terrno = TAOS_SYSTEM_ERROR(errno);
+    syncStop(pNode);
+    return NULL;
+  }
+
+  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+  if (pNode->pFwdTimer == NULL) {
+    sError("vgId:%d, failed to allocate timer", pNode->vgId);
+    syncStop(pNode);
+    return NULL;
+  }
+
+  syncAddArbitrator(pNode);
+  syncAddNodeRef(pNode);
+  taosHashPut(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t), (char *)(&pNode), sizeof(SSyncNode *));
+ 
+  if (pNode->notifyRole) 
+   (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+
+  return pNode;
+}
+
+void syncStop(void *param) 
+{
+  SSyncNode  *pNode = param;
+  SSyncPeer  *pPeer;
+
+  if (pNode == NULL) return;
+  sInfo("vgId:%d, cleanup sync", pNode->vgId);
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeer = pNode->peerInfo[i];
+    if (pPeer) syncRemovePeer(pPeer); 
+  }
+
+  pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+  if (pPeer) syncRemovePeer(pPeer);
+
+  if (vgIdHash) taosHashRemove(vgIdHash, (const char *)&pNode->vgId, sizeof(int32_t));
+  if (pNode->pFwdTimer) taosTmrStop(pNode->pFwdTimer);
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  syncDecNodeRef(pNode);
+}
+
+int32_t syncReconfig(void *param, const SSyncCfg *pNewCfg) 
+{
+  SSyncNode  *pNode = param;
+  int         i, j;
+
+  if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
+  sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], 
+         pNewCfg->replica, pNode->replica);
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (i = 0; i < pNode->replica; ++i) {
+    for (j = 0; j < pNewCfg->replica; ++j) {
+      if ((strcmp(pNode->peerInfo[i]->fqdn, pNewCfg->nodeInfo[j].nodeFqdn) == 0) && 
+          (pNode->peerInfo[i]->port == pNewCfg->nodeInfo[j].nodePort)) 
+        break;
+    }
+
+    if (j >= pNewCfg->replica) {
+      syncRemovePeer(pNode->peerInfo[i]);
+      pNode->peerInfo[i] = NULL;
+    }
+  }
+
+  SSyncPeer *newPeers[TAOS_SYNC_MAX_REPLICA];
+  for (i = 0; i < pNewCfg->replica; ++i) {
+    const SNodeInfo *pNewNode = &pNewCfg->nodeInfo[i];
+
+    for (j = 0; j < pNode->replica; ++j) {
+      if (pNode->peerInfo[j] && (strcmp(pNode->peerInfo[j]->fqdn, pNewNode->nodeFqdn) == 0) && 
+         (pNode->peerInfo[j]->port == pNewNode->nodePort))
+        break;
+    }
+
+    if (j >= pNode->replica) {
+      newPeers[i] = syncAddPeer(pNode, pNewNode);
+    } else {
+      newPeers[i] = pNode->peerInfo[j];
+    }
+
+    if ((strcmp(pNewNode->nodeFqdn, tsNodeFqdn) == 0) && (pNewNode->nodePort == tsSyncPort)) 
+      pNode->selfIndex = i;
+  }
+
+  pNode->replica = pNewCfg->replica;
+  pNode->quorum = pNewCfg->quorum;
+  memcpy(pNode->peerInfo, newPeers, sizeof(SSyncPeer *) * pNewCfg->replica);
+
+  for (i = pNewCfg->replica; i < TAOS_SYNC_MAX_REPLICA; ++i)
+    pNode->peerInfo[i] = NULL;
+
+  syncAddArbitrator(pNode);
+
+  if (pNewCfg->replica <= 1) {
+    sInfo("vgId:%d, no peers are configured, work as master!", pNode->vgId);
+    nodeRole = TAOS_SYNC_ROLE_MASTER;
+    (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  sInfo("vgId:%d, %d replicas are configured, quorum:%d role:%s", pNode->vgId, pNode->replica, pNode->quorum, syncRole[nodeRole]);
+  syncBroadcastStatus(pNode);
+
+  return 0;
+}
+
+int32_t syncForwardToPeer(void *param, void *data, void *mhandle, int qtype)
+{
+  SSyncNode  *pNode = param;
+  SSyncPeer  *pPeer;
+  SSyncHead  *pSyncHead;
+  SWalHead   *pWalHead = data;
+  int         fwdLen;
+  int         code = 0;
+
+  if (pNode == NULL) return 0;
+
+  // always update version
+  nodeVersion = pWalHead->version;
+  if (pNode->replica == 1 || nodeRole != TAOS_SYNC_ROLE_MASTER ) return 0;
+
+  // only pkt from RPC or CQ can be forwarded
+  if (qtype != TAOS_QTYPE_RPC && qtype != TAOS_QTYPE_CQ) return 0;
+
+  // a hacker way to improve the performance
+  pSyncHead = (SSyncHead *) ( ((char *)pWalHead) - sizeof(SSyncHead));
+  pSyncHead->type = TAOS_SMSG_FORWARD;
+  pSyncHead->pversion = 0;
+  pSyncHead->len = sizeof(SWalHead) + pWalHead->len; 
+  fwdLen = pSyncHead->len + sizeof(SSyncHead);  //include the WAL and SYNC head 
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  if (pNode->quorum > 1) {
+    syncSaveFwdInfo(pNode, pWalHead->version, mhandle);
+    code = 1;
+  }
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeer = pNode->peerInfo[i];
+    if (pPeer == NULL || pPeer->peerFd <0) continue; 
+    if (pPeer->role != TAOS_SYNC_ROLE_SLAVE && pPeer->sstatus != TAOS_SYNC_STATUS_CACHE) continue; 
+  
+    int retLen = write(pPeer->peerFd, pSyncHead, fwdLen);
+    if (retLen == fwdLen) {
+      sDebug("%s, forward is sent, ver:%" PRIu64 " contLen:%d", pPeer->id, pWalHead->version, pWalHead->len);
+    } else {
+      sError("%s, failed to forward, ver:%" PRIu64 " retLen:%d", pPeer->id, pWalHead->version, retLen);
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  return code;
+}
+
+void syncConfirmForward(void *param, uint64_t version, int32_t code)
+{
+  SSyncNode  *pNode = param;
+  if (pNode == NULL) return;
+  if (pNode->quorum <= 1) return;
+
+  SSyncPeer  *pPeer = pNode->pMaster;
+  if (pPeer == NULL) return;
+
+  char msg[sizeof(SSyncHead) + sizeof(SFwdRsp)] = {0};
+
+  SSyncHead   *pHead = (SSyncHead *) msg;
+  pHead->type = TAOS_SMSG_FORWARD_RSP;
+  pHead->len = sizeof(SFwdRsp);
+
+  SFwdRsp *pFwdRsp = (SFwdRsp *)(msg + sizeof(SSyncHead));
+  pFwdRsp->version = version;
+  pFwdRsp->code = code;
+
+  int msgLen = sizeof(SSyncHead) + sizeof(SFwdRsp);
+  int retLen = write(pPeer->peerFd, msg, msgLen);
+
+  if (retLen == msgLen) {
+    sDebug("%s, forward-rsp is sent, ver:%" PRIu64, pPeer->id, version);
+  } else {
+    sDebug("%s, failed to send forward ack, restart", pPeer->id);
+    syncRestartConnection(pPeer);
+  }
+}
+
+void syncRecover(void *param) {
+  SSyncNode *pNode = param;
+  SSyncPeer *pPeer;
+
+  // to do: add a few lines to check if recover is OK 
+  // if take this node to unsync state, the whole system may not work
+
+  nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+  (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+  nodeVersion = 0;
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeer = (SSyncPeer *) pNode->peerInfo[i];
+    if (pPeer->peerFd >= 0) {
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+int syncGetNodesRole(void *param, SNodesRole *pNodesRole)
+{
+  SSyncNode *pNode = param;
+  
+  pNodesRole->selfIndex = pNode->selfIndex;
+  for (int i=0; i<pNode->replica; ++i) {
+    pNodesRole->nodeId[i] = pNode->peerInfo[i]->nodeId;
+    pNodesRole->role[i] = pNode->peerInfo[i]->role;
+  }
+
+  return 0;
+}
+
+static void syncAddArbitrator(SSyncNode *pNode)
+{
+  SSyncPeer *pPeer = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+
+  // if not configured, return right away
+  if (tsArbitrator[0] == 0) {
+    if (pPeer) syncRemovePeer(pPeer);
+    pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = NULL;
+    return;
+  }
+
+  SNodeInfo nodeInfo;
+  nodeInfo.nodeId = 0;
+  taosGetFqdnPortFromEp(tsArbitrator, nodeInfo.nodeFqdn, &nodeInfo.nodePort);
+  nodeInfo.nodePort += TSDB_PORT_SYNC;
+  
+  if (pPeer) {
+    if ((strcmp(nodeInfo.nodeFqdn, pPeer->fqdn) == 0) && (nodeInfo.nodePort == pPeer->port)) {
+      return;
+    } else {
+      syncRemovePeer(pPeer);
+      pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = NULL;
+    }
+  }   
+
+  pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo);
+}
+
+static void syncAddNodeRef(SSyncNode *pNode)
+{
+   atomic_add_fetch_8(&pNode->refCount, 1);
+}
+
+static void syncDecNodeRef(SSyncNode *pNode)
+{
+  if (atomic_sub_fetch_8(&pNode->refCount, 1) == 0) {
+    pthread_mutex_destroy(&pNode->mutex);
+    tfree(pNode->pRecv);
+    tfree(pNode->pSyncFwds);
+    tfree(pNode);
+
+    if (atomic_sub_fetch_32(&tsNodeNum, 1) == 0) { 
+      if (tsTcpPool) taosCloseTcpThreadPool(tsTcpPool);
+      if (syncTmrCtrl) taosTmrCleanUp(syncTmrCtrl);
+      if (vgIdHash) taosHashCleanup(vgIdHash);
+      syncTmrCtrl = NULL;
+      tsTcpPool = NULL;
+      vgIdHash = NULL;
+      syncModuleInit = PTHREAD_ONCE_INIT;
+      sDebug("sync module is cleaned up");
+    }
+  }
+}
+
+void syncAddPeerRef(SSyncPeer *pPeer)
+{
+   atomic_add_fetch_8(&pPeer->refCount, 1);
+}
+
+int syncDecPeerRef(SSyncPeer *pPeer)
+{
+  if (atomic_sub_fetch_8(&pPeer->refCount, 1) == 0) {
+    syncDecNodeRef(pPeer->pSyncNode);
+
+    sDebug("%s, resource is freed", pPeer->id);
+    tfree(pPeer->watchFd);
+    tfree(pPeer);
+    return 0;
+  }
+
+  return 1;
+}
+
+static void syncClosePeerConn(SSyncPeer *pPeer) 
+{
+  taosTmrStopA(&pPeer->timer);
+  tclose(pPeer->syncFd);
+  if (pPeer->peerFd >=0) {
+    pPeer->peerFd = -1;
+    taosFreeTcpConn(pPeer->pConn);
+  }
+}
+
+static void syncRemovePeer(SSyncPeer *pPeer)
+{
+  sInfo("%s, it is removed", pPeer->id);
+
+  pPeer->ip = 0;
+  syncClosePeerConn(pPeer);
+  syncDecPeerRef(pPeer);
+}
+
+static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) 
+{
+  uint32_t ip = taosGetIpFromFqdn(pInfo->nodeFqdn);
+  if (ip == -1) return NULL;
+ 
+  SSyncPeer *pPeer = (SSyncPeer *) calloc(1, sizeof(SSyncPeer));
+  if (pPeer == NULL) return NULL;
+
+  pPeer->nodeId = pInfo->nodeId;
+  tstrncpy(pPeer->fqdn, pInfo->nodeFqdn, sizeof(pPeer->fqdn));
+  pPeer->ip = ip;
+  pPeer->port = pInfo->nodePort;
+  snprintf(pPeer->id, sizeof(pPeer->id), "vgId:%d peer:%s:%d", pNode->vgId, pPeer->fqdn, pPeer->port);
+
+  pPeer->peerFd = -1;
+  pPeer->syncFd = -1;
+  pPeer->role = TAOS_SYNC_ROLE_OFFLINE;
+  pPeer->pSyncNode = pNode;
+  pPeer->refCount = 1;
+
+  sInfo("%s, it is configured", pPeer->id);
+  int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+  if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
+    sDebug("%s, start to check peer connection", pPeer->id);
+    taosTmrReset(syncCheckPeerConnection, 100, pPeer, syncTmrCtrl, &pPeer->timer);
+  }
+  
+  syncAddNodeRef(pNode);
+  return pPeer;
+}
+
+void syncBroadcastStatus(SSyncNode *pNode)
+{
+  SSyncPeer *pPeer;
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    if ( i == pNode->selfIndex ) continue;
+    pPeer = pNode->peerInfo[i];
+    syncSendPeersStatusMsgToPeer(pPeer, 1);
+  }
+} 
+
+static void syncChooseMaster(SSyncNode *pNode) {
+  SSyncPeer *pPeer;
+  int        onlineNum = 0;
+  int        index = -1;
+  int        replica = pNode->replica;
+
+  sDebug("vgId:%d, choose master", pNode->vgId);
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE)
+      onlineNum++;
+  }
+
+  if (onlineNum == pNode->replica) {
+    // if all peers are online, peer with highest version shall be master
+    index = 0;
+    for (int i = 1; i < pNode->replica; ++i) {
+      if (pNode->peerInfo[i]->version > pNode->peerInfo[index]->version)
+        index = i;
+    }
+  }
+
+  // add arbitrator connection
+  SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+  if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
+    onlineNum++;
+    replica = pNode->replica + 1;
+  }
+
+  if (index < 0 && onlineNum > replica/2.0) {
+    // over half of nodes are online
+    for (int i = 0; i < pNode->replica; ++i) {
+      //slave with highest version shall be master
+      pPeer = pNode->peerInfo[i];
+      if (pPeer->role == TAOS_SYNC_ROLE_SLAVE || pPeer->role == TAOS_SYNC_ROLE_MASTER) {
+        if (index < 0 || pPeer->version > pNode->peerInfo[index]->version)
+          index = i;
+      }
+    }
+  }
+
+  if (index >= 0) {
+    if (index == pNode->selfIndex) {
+      sInfo("vgId:%d, start to work as master", pNode->vgId);
+      nodeRole = TAOS_SYNC_ROLE_MASTER;
+      (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+    } else {
+      pPeer = pNode->peerInfo[index];
+      sInfo("%s, it shall work as master", pPeer->id);
+    }
+  } else {
+    sDebug("vgId:%d, failed to choose master", pNode->vgId);
+  }
+} 
+ 
+static SSyncPeer *syncCheckMaster(SSyncNode *pNode ) {
+  int onlineNum = 0;
+  int index = -1;
+  int replica = pNode->replica;
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    if (pNode->peerInfo[i]->role != TAOS_SYNC_ROLE_OFFLINE) 
+      onlineNum++;
+  }
+
+  // add arbitrator connection
+  SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
+  if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
+    onlineNum++;
+    replica = pNode->replica + 1;
+  }
+
+  if (onlineNum <= replica*0.5) {
+    if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
+      nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+      pNode->peerInfo[pNode->selfIndex]->role = nodeRole;
+      (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+      sInfo("vgId:%d, change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
+    }
+  } else {
+    for (int i=0; i<pNode->replica; ++i) {
+      SSyncPeer *pTemp = pNode->peerInfo[i];
+      if ( pTemp->role != TAOS_SYNC_ROLE_MASTER ) continue;
+      if ( index < 0 ) {
+        index = i;
+      } else { // multiple masters, it shall not happen 
+        if ( i == pNode->selfIndex ) {
+          sError("%s, peer is master, work as slave instead", pTemp->id);
+          nodeRole = TAOS_SYNC_ROLE_SLAVE;
+          (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+        }
+      }
+    }
+  }
+
+  SSyncPeer *pMaster = (index>=0) ? pNode->peerInfo[index]:NULL;
+  return pMaster;
+}
+
+static int syncValidateMaster(SSyncPeer *pPeer) {
+  SSyncNode *pNode = pPeer->pSyncNode;
+  int code = 0;
+
+  if (nodeRole == TAOS_SYNC_ROLE_MASTER && nodeVersion < pPeer->version) {
+    sDebug("%s, slave has higher version, restart all connections!!!",  pPeer->id);
+    nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+    (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+    code = -1;
+
+    for (int i = 0; i < pNode->replica; ++i) {
+      if ( i == pNode->selfIndex ) continue;
+      syncRestartPeer(pNode->peerInfo[i]);
+    }
+  } 
+
+  return code;
+}
+
+static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus peersStatus[], int8_t newRole)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  int8_t     peerOldRole = pPeer->role;
+  int8_t     selfOldRole = nodeRole;
+  int8_t     i, syncRequired = 0;
+
+  pNode->peerInfo[pNode->selfIndex]->version = nodeVersion;
+  pPeer->role = newRole;
+
+  sDebug("%s, own role:%s, new peer role:%s", pPeer->id, 
+          syncRole[nodeRole], syncRole[pPeer->role]);  
+
+  SSyncPeer *pMaster = syncCheckMaster(pNode);
+
+  if ( pMaster ) {
+    // master is there
+    pNode->pMaster = pMaster;
+    sDebug("%s, it is the master, ver:%" PRIu64, pMaster->id, pMaster->version);
+
+    if (syncValidateMaster(pPeer) < 0) return;
+
+    if (nodeRole == TAOS_SYNC_ROLE_UNSYNCED) {
+      if ( nodeVersion < pMaster->version) {
+        syncRequired = 1;
+      } else {
+        sInfo("%s is master, work as slave, ver:%" PRIu64, pMaster->id, pMaster->version);
+        nodeRole = TAOS_SYNC_ROLE_SLAVE;
+        (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+      }
+    } else if ( nodeRole == TAOS_SYNC_ROLE_SLAVE && pMaster == pPeer) {
+      // nodeVersion = pMaster->version;
+    }
+  } else {
+    // master not there, if all peer's state and version are consistent, choose the master
+    int consistent = 0;
+    if (peersStatus) {
+      for (i = 0; i < pNode->replica; ++i) {
+        SSyncPeer *pTemp = pNode->peerInfo[i];
+        if (pTemp->role != peersStatus[i].role) break;
+        if ((pTemp->role != TAOS_SYNC_ROLE_OFFLINE) && (pTemp->version != peersStatus[i].version)) break; 
+      }
+ 
+      if (i >= pNode->replica) consistent = 1;
+    } else {
+      if (pNode->replica == 2) consistent = 1;
+    }
+
+    if (consistent)
+      syncChooseMaster(pNode);
+  }
+
+  if (syncRequired) {
+    syncRecoverFromMaster(pMaster);
+  }
+
+  if (peerOldRole != newRole || nodeRole != selfOldRole)
+    syncBroadcastStatus(pNode);
+}
+
+static void syncRestartPeer(SSyncPeer *pPeer) {
+  sDebug("%s, restart connection", pPeer->id);
+
+  syncClosePeerConn(pPeer);
+
+  pPeer->sstatus = TAOS_SYNC_STATUS_INIT;
+
+  int ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+  if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort) )
+    taosTmrReset(syncCheckPeerConnection, tsSyncTimer*1000, pPeer, syncTmrCtrl, &pPeer->timer);
+}
+
+void syncRestartConnection(SSyncPeer *pPeer)
+{
+  if (pPeer->ip == 0) return;
+
+  syncRestartPeer(pPeer);
+  syncCheckRole(pPeer, NULL, TAOS_SYNC_ROLE_OFFLINE);
+}
+
+static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  sDebug("%s, sync-req is received", pPeer->id);
+
+  if (pPeer->ip == 0) return;
+
+  if (nodeRole != TAOS_SYNC_ROLE_MASTER) {
+    sError("%s, I am not master anymore", pPeer->id);
+    tclose(pPeer->syncFd);
+    return;
+  }
+
+  if (pPeer->sstatus != TAOS_SYNC_STATUS_INIT) {
+    sDebug("%s, sync is already started", pPeer->id);
+    return; // already started
+  }
+
+  // start a new thread to retrieve the data
+  syncAddPeerRef(pPeer);
+  pthread_attr_t  thattr;
+  pthread_t       thread;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
+  int ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
+  pthread_attr_destroy(&thattr);
+
+  if (ret != 0) {
+    sError("%s, failed to create sync thread(%s)", pPeer->id, strerror(errno));
+    syncDecPeerRef(pPeer);
+  } else {
+    pPeer->sstatus = TAOS_SYNC_STATUS_START;
+    sDebug("%s, thread is created to retrieve data", pPeer->id);
+  }
+}
+
+static void syncNotStarted(void *param, void *tmrId)
+{
+  SSyncPeer *pPeer = param;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  pthread_mutex_lock(&(pNode->mutex));
+  pPeer->timer = NULL;
+  sInfo("%s, sync connection is still not up, restart", pPeer->id);
+  syncRestartConnection(pPeer);
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+static void syncTryRecoverFromMaster(void *param, void *tmrId) {
+  SSyncPeer   *pPeer = param;
+  SSyncNode   *pNode = pPeer->pSyncNode;
+
+  pthread_mutex_lock(&(pNode->mutex));
+  syncRecoverFromMaster(pPeer);
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+static void syncRecoverFromMaster(SSyncPeer *pPeer)
+{
+  SSyncNode   *pNode = pPeer->pSyncNode;
+
+  if ( nodeSStatus != TAOS_SYNC_STATUS_INIT) {
+    sDebug("%s, sync is already started, status:%d", pPeer->id, nodeSStatus);
+    return;
+  } 
+
+  taosTmrStopA(&pPeer->timer);
+  if (tsSyncNum >= tsMaxSyncNum) {
+    sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum);
+    taosTmrReset(syncTryRecoverFromMaster, 500, pPeer, syncTmrCtrl, &pPeer->timer);
+    return;
+  }
+
+  sDebug("%s, try to sync", pPeer->id)
+
+  SFirstPkt firstPkt;
+  memset(&firstPkt, 0, sizeof(firstPkt));
+  firstPkt.syncHead.type = TAOS_SMSG_SYNC_REQ;
+  firstPkt.syncHead.vgId = pNode->vgId;
+  firstPkt.syncHead.len = sizeof(firstPkt) - sizeof(SSyncHead);
+  tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
+  firstPkt.port = tsSyncPort;
+  taosTmrReset(syncNotStarted, tsSyncTimer*1000, pPeer, syncTmrCtrl, &pPeer->timer);
+
+  if (write(pPeer->peerFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt) ) {
+    sError("%s, failed to send sync-req to peer", pPeer->id);
+  } else {
+    nodeSStatus = TAOS_SYNC_STATUS_START;
+    sInfo("%s, sync-req is sent", pPeer->id);
+  }
+
+  return;
+}
+
+static void syncProcessFwdResponse(char *cont, SSyncPeer *pPeer) 
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  SFwdRsp    *pFwdRsp = (SFwdRsp *) cont;
+  SSyncFwds  *pSyncFwds = pNode->pSyncFwds;
+  SFwdInfo   *pFwdInfo;
+
+  sDebug("%s, forward-rsp is received, ver:%" PRIu64, pPeer->id, pFwdRsp->version);
+  SFwdInfo *pFirst = pSyncFwds->fwdInfo + pSyncFwds->first;
+
+  if (pFirst->version <= pFwdRsp->version && pSyncFwds->fwds > 0) {
+    // find the forwardInfo from first
+    for (int i=0; i<pSyncFwds->fwds; ++i) {
+      pFwdInfo = pSyncFwds->fwdInfo + (i+pSyncFwds->first)%tsMaxFwdInfo;
+      if (pFwdRsp->version == pFwdInfo->version) break;
+    }
+ 
+    syncProcessFwdAck(pNode, pFwdInfo, pFwdRsp->code);
+    syncRemoveConfirmedFwdInfo(pNode);
+  }
+}
+
+
+static void syncProcessForwardFromPeer(char *cont, SSyncPeer *pPeer)
+{
+  SSyncNode   *pNode = pPeer->pSyncNode;
+  SWalHead    *pHead = (SWalHead *)cont;
+
+  sDebug("%s, forward is received, ver:%" PRIu64, pPeer->id, pHead->version);
+
+  if (nodeRole == TAOS_SYNC_ROLE_SLAVE) {
+    //nodeVersion = pHead->version;
+    (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
+  } else { 
+    if (nodeSStatus != TAOS_SYNC_STATUS_INIT) {
+      syncSaveIntoBuffer(pPeer, pHead);
+    } else {
+      sError("%s, forward discarded, ver:%" PRIu64, pPeer->id, pHead->version);
+    }
+  }
+
+  return;
+}
+
+static void syncProcessPeersStatusMsg(char *cont, SSyncPeer *pPeer)
+{
+  SSyncNode    *pNode = pPeer->pSyncNode;
+  SPeersStatus *pPeersStatus = (SPeersStatus *)cont;
+
+  sDebug("%s, status msg received, self:%s ver:%" PRIu64 " peer:%s ver:%" PRIu64 ", ack:%d", pPeer->id,
+         syncRole[nodeRole], nodeVersion, syncRole[pPeersStatus->role], pPeersStatus->version, pPeersStatus->ack);
+
+  pPeer->version = pPeersStatus->version;
+  syncCheckRole(pPeer, pPeersStatus->peersStatus, pPeersStatus->role);
+
+  if (pPeersStatus->ack)
+    syncSendPeersStatusMsgToPeer(pPeer, 0);
+}
+
+static int syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead, char *cont) {
+  if (pPeer->peerFd <0) return -1;
+
+  int hlen = taosReadMsg(pPeer->peerFd, pHead, sizeof(SSyncHead));
+  if (hlen != sizeof(SSyncHead)) {
+    sDebug("%s, failed to read msg, hlen:%d", pPeer->id, hlen);
+    return -1;
+  }
+
+  // head.len = htonl(head.len);
+  if (pHead->len <0) {
+    sError("%s, invalid pkt length, len:%d", pPeer->id, pHead->len);
+    return -1;
+  } 
+
+  int bytes = taosReadMsg(pPeer->peerFd, cont, pHead->len);
+  if (bytes != pHead->len) {
+    sError("%s, failed to read, bytes:%d len:%d", pPeer->id, bytes, pHead->len);
+    return -1;
+  }
+
+  return 0;
+}
+
+static int syncProcessPeerMsg(void *param, void *buffer)
+{
+  SSyncPeer  *pPeer = param;
+  SSyncHead   head;
+  char       *cont = (char *)buffer;
+
+  SSyncNode *pNode = pPeer->pSyncNode;
+  pthread_mutex_lock(&(pNode->mutex));
+
+  int code = syncReadPeerMsg(pPeer, &head, cont);
+
+  if (code == 0) {
+    if (head.type == TAOS_SMSG_FORWARD) {
+      syncProcessForwardFromPeer(cont, pPeer);
+    } else if (head.type == TAOS_SMSG_FORWARD_RSP) {
+      syncProcessFwdResponse(cont, pPeer);
+    } else if (head.type == TAOS_SMSG_SYNC_REQ) {
+      syncProcessSyncRequest(cont, pPeer);
+    } else if (head.type == TAOS_SMSG_STATUS) {
+      syncProcessPeersStatusMsg(cont, pPeer);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  return code;
+}
+
+#define statusMsgLen sizeof(SSyncHead)+sizeof(SPeersStatus)+sizeof(SPeerStatus)*TAOS_SYNC_MAX_REPLICA
+
+static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  char       msg[statusMsgLen] = {0};
+
+  if (pPeer->peerFd <0 || pPeer->ip ==0) return;
+
+  SSyncHead    *pHead = (SSyncHead *) msg;
+  SPeersStatus *pPeersStatus = (SPeersStatus *) (msg + sizeof(SSyncHead));
+
+  pHead->type = TAOS_SMSG_STATUS;
+  pHead->len = statusMsgLen - sizeof(SSyncHead);
+
+  pPeersStatus->version = nodeVersion;
+  pPeersStatus->role = nodeRole;
+  pPeersStatus->ack = ack;
+
+  for (int i = 0; i < pNode->replica; ++i) {
+    pPeersStatus->peersStatus[i].role = pNode->peerInfo[i]->role;
+    pPeersStatus->peersStatus[i].version = pNode->peerInfo[i]->version;
+  }
+
+  int retLen = write(pPeer->peerFd, msg, statusMsgLen);
+  if (retLen == statusMsgLen) {
+    sDebug("%s, status msg is sent", pPeer->id);
+  } else {
+    sDebug("%s, failed to send status msg, restart", pPeer->id);
+    syncRestartConnection(pPeer);
+  }
+
+  return;
+}
+
+static void syncSetupPeerConnection(SSyncPeer *pPeer) {
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  taosTmrStopA(&pPeer->timer);
+  if (pPeer->peerFd >= 0) {
+    sDebug("%s, send role version to peer", pPeer->id);
+    syncSendPeersStatusMsgToPeer(pPeer, 1);
+    return;
+  }
+
+  int connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
+  if (connFd < 0) {
+    sDebug("%s, failed to open tcp socket(%s)", pPeer->id, strerror(errno));
+    taosTmrReset(syncCheckPeerConnection, tsSyncTimer *1000, pPeer, syncTmrCtrl, &pPeer->timer);
+    return;
+  }
+
+  SFirstPkt firstPkt;
+  memset(&firstPkt, 0, sizeof(firstPkt));
+  firstPkt.syncHead.vgId = pPeer->nodeId ? pNode->vgId:0;
+  firstPkt.syncHead.type = TAOS_SMSG_STATUS;
+  tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn)); 
+  firstPkt.port = tsSyncPort;
+  firstPkt.sourceId = pNode->vgId;  // tell arbitrator its vgId
+
+  if ( write(connFd, &firstPkt, sizeof(firstPkt)) == sizeof(firstPkt)) {
+    sDebug("%s, connection to peer server is setup", pPeer->id);
+    pPeer->peerFd = connFd; 
+    pPeer->role = TAOS_SYNC_ROLE_UNSYNCED;
+    pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);
+    syncAddPeerRef(pPeer);
+  } else {
+    sDebug("try later");
+    close(connFd);
+    taosTmrReset(syncCheckPeerConnection, tsSyncTimer *1000, pPeer, syncTmrCtrl, &pPeer->timer);
+  }
+}
+
+static void syncCheckPeerConnection(void *param, void *tmrId) 
+{
+  SSyncPeer *pPeer = param;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  pthread_mutex_lock(&(pNode->mutex));
+
+  sDebug("%s, check peer connection", pPeer->id);
+  syncSetupPeerConnection(pPeer); 
+
+  pthread_mutex_unlock(&(pNode->mutex));
+}
+
+static void syncCreateRestoreDataThread(SSyncPeer *pPeer) 
+{
+  taosTmrStopA(&pPeer->timer);
+
+  pthread_attr_t thattr;
+  pthread_t thread;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
+
+  syncAddPeerRef(pPeer);
+  int ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
+  pthread_attr_destroy(&thattr);
+
+  if (ret < 0) {
+    sError("%s, failed to create sync thread", pPeer->id);
+    tclose(pPeer->syncFd);
+    syncDecPeerRef(pPeer);
+  } else { 
+    sInfo("%s, sync connection is up", pPeer->id);
+  }
+}
+
+static void syncProcessIncommingConnection(int connFd, uint32_t sourceIp) 
+{
+  char  ipstr[24];
+  int   i;
+   
+  tinet_ntoa(ipstr, sourceIp);
+  sDebug("peer TCP connection from ip:%s", ipstr);
+
+  SFirstPkt firstPkt;
+  if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
+    sError("failed to read peer first pkt from ip:%s(%s)", ipstr, strerror(errno));
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  int32_t vgId = firstPkt.syncHead.vgId;
+  SSyncNode **ppNode = (SSyncNode **)taosHashGet(vgIdHash, (const char *)&vgId, sizeof(int32_t));
+  if (ppNode == NULL || *ppNode == NULL) {
+    sError("vgId:%d, vgId could not be found", vgId);
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  SSyncNode *pNode = *ppNode;
+  pthread_mutex_lock(&(pNode->mutex));
+
+  SSyncPeer *pPeer;
+  for (i = 0; i < pNode->replica; ++i) {
+    pPeer = pNode->peerInfo[i];
+    if (pPeer && (strcmp(pPeer->fqdn, firstPkt.fqdn) == 0) && (pPeer->port == firstPkt.port))
+      break;
+  }
+
+  pPeer = (i < pNode->replica) ? pNode->peerInfo[i] : NULL;
+  if (pPeer == NULL) {
+    sError("vgId:%d, peer:%s not configured", pNode->vgId, firstPkt.fqdn);
+    taosCloseSocket(connFd);
+    // syncSendVpeerCfgMsg(sync);
+  } else {
+    // first packet tells what kind of link
+    if (firstPkt.syncHead.type == TAOS_SMSG_SYNC_DATA) {
+      pPeer->syncFd = connFd;
+      syncCreateRestoreDataThread(pPeer);
+    } else {
+      sDebug("%s, TCP connection is already up, close one", pPeer->id);
+      syncClosePeerConn(pPeer);
+      pPeer->peerFd = connFd;
+      pPeer->pConn = taosAllocateTcpConn(tsTcpPool, pPeer, connFd);
+      syncAddPeerRef(pPeer);
+      sDebug("%s, ready to exchange data", pPeer->id);
+      syncSendPeersStatusMsgToPeer(pPeer, 1);
+    }
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+
+  return;
+}
+
+static void syncProcessBrokenLink(void *param) {
+  if (param == NULL) return;  // the connection for arbitrator
+  SSyncPeer *pPeer = param;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  syncAddNodeRef(pNode);
+  pthread_mutex_lock(&(pNode->mutex));
+
+  sDebug("%s, TCP link is broken(%s)", pPeer->id, strerror(errno));
+  pPeer->peerFd = -1;
+
+  if (syncDecPeerRef(pPeer) != 0) {
+    syncRestartConnection(pPeer);
+  }
+
+  pthread_mutex_unlock(&(pNode->mutex));
+  syncDecNodeRef(pNode);
+}
+
+static void syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) 
+{
+  SSyncFwds *pSyncFwds = pNode->pSyncFwds;
+  uint64_t   time = taosGetTimestampMs();
+
+  if (pSyncFwds->fwds >= tsMaxFwdInfo) {
+    pSyncFwds->first = (pSyncFwds->first + 1) % tsMaxFwdInfo;
+    pSyncFwds->fwds--;
+  } 
+
+  if (pSyncFwds->fwds > 0) 
+    pSyncFwds->last = (pSyncFwds->last+1) % tsMaxFwdInfo;
+  SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->last;
+  pFwdInfo->version = version;
+  pFwdInfo->mhandle = mhandle;
+  pFwdInfo->acks = 0;
+  pFwdInfo->confirmed = 0;
+  pFwdInfo->time = time;
+
+  pSyncFwds->fwds++;
+  sDebug("vgId:%d, fwd info is saved, ver:%" PRIu64 " fwds:%d ", pNode->vgId, version, pSyncFwds->fwds);
+}
+
+static void syncRemoveConfirmedFwdInfo(SSyncNode *pNode)
+{
+  SSyncFwds *pSyncFwds = pNode->pSyncFwds;
+
+  int fwds = pSyncFwds->fwds;
+  for (int i=0; i<fwds; ++i) {
+    SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + pSyncFwds->first; 
+    if (pFwdInfo->confirmed == 0) break;
+
+    pSyncFwds->first = (pSyncFwds->first+1) % tsMaxFwdInfo;
+    pSyncFwds->fwds--;
+    if (pSyncFwds->fwds == 0) pSyncFwds->first = pSyncFwds->last;
+    //sDebug("vgId:%d, fwd info is removed, ver:%d, fwds:%d", 
+    //        pNode->vgId, pFwdInfo->version, pSyncFwds->fwds);
+    memset(pFwdInfo, 0, sizeof(SFwdInfo));
+  }
+}
+
+static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code) 
+{
+  int confirm = 0;
+  if (pFwdInfo->code == 0) pFwdInfo->code = code;
+
+  if (code == 0) {
+    pFwdInfo->acks++;
+    if (pFwdInfo->acks >= pNode->quorum-1) 
+      confirm = 1;
+  } else {
+    pFwdInfo->nacks++;
+    if (pFwdInfo->nacks > pNode->replica-pNode->quorum) 
+      confirm = 1;
+  }
+
+  if (confirm && pFwdInfo->confirmed == 0) {
+    sDebug("vgId:%d, forward is confirmed, ver:%" PRIu64 " code:%x", pNode->vgId, pFwdInfo->version, pFwdInfo->code);
+    (*pNode->confirmForward)(pNode->ahandle, pFwdInfo->mhandle, pFwdInfo->code);
+    pFwdInfo->confirmed = 1;
+  }
+}
+
+static void syncMonitorFwdInfos(void *param, void *tmrId)
+{
+  SSyncNode *pNode = param;
+  SSyncFwds *pSyncFwds = pNode->pSyncFwds;
+  uint64_t   time = taosGetTimestampMs();
+
+  if (pSyncFwds->fwds > 0) {
+    pthread_mutex_lock(&(pNode->mutex));
+    for (int i=0; i<pSyncFwds->fwds; ++i) {
+      SFwdInfo *pFwdInfo = pSyncFwds->fwdInfo + (pSyncFwds->first+i) % tsMaxFwdInfo; 
+      if (time - pFwdInfo->time < 2000) break;
+      syncProcessFwdAck(pNode, pFwdInfo, TSDB_CODE_RPC_NETWORK_UNAVAIL);
+    }
+
+    syncRemoveConfirmedFwdInfo(pNode);
+    pthread_mutex_unlock(&(pNode->mutex));
+  } 
+  
+  pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, 300, pNode, syncTmrCtrl);
+}
+ 
+
+

src/sync/src/syncRestore.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "ttimer.h"
+#include "tsocket.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+static void syncRemoveExtraFile(SSyncPeer *pPeer, uint32_t sindex, uint32_t eindex) {
+  char       name[TSDB_FILENAME_LEN*2] = {0};
+  char       fname[TSDB_FILENAME_LEN*3] = {0};
+  uint32_t   magic; 
+  uint64_t   fversion;
+  int32_t    size;
+  uint32_t   index = sindex;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  if (sindex < 0 || eindex < sindex) return;
+
+  while (1) {
+    name[0] = 0;
+    magic = (*pNode->getFileInfo)(pNode->ahandle, name, &index, eindex, &size, &fversion);
+    if (magic == 0) break;
+
+    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, name);
+    remove(fname);
+    sDebug("%s, %s is removed", pPeer->id, fname);
+
+    index++;
+    if (index > eindex) break;
+  }        
+}
+
+static int syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) 
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  SFileInfo  minfo; memset(&minfo, 0, sizeof(minfo)); /* = {0}; */  // master file info
+  SFileInfo  sinfo; memset(&sinfo, 0, sizeof(sinfo)); /* = {0}; */  // slave file info
+  SFileAck   fileAck; 
+  int        code = -1;
+  char       name[TSDB_FILENAME_LEN * 2] = {0};
+  uint32_t   pindex = 0;    // index in last restore
+
+  *fversion = 0;
+  sinfo.index = 0;
+  while (1) {
+    // read file info
+    int ret = taosReadMsg(pPeer->syncFd, &(minfo), sizeof(minfo));
+    if (ret < 0 ) break;
+
+    // if no more file from master, break;
+    if (minfo.name[0] == 0 || minfo.magic == 0) {
+      sDebug("%s, no more files to restore", pPeer->id);
+
+      // remove extra files after the current index
+      syncRemoveExtraFile(pPeer, sinfo.index+1, TAOS_SYNC_MAX_INDEX);
+      code = 0; 
+      break;
+    }
+   
+    // remove extra files on slave between the current and last index
+    syncRemoveExtraFile(pPeer, pindex+1, minfo.index-1);
+    pindex = minfo.index;
+
+    // check the file info
+    sinfo = minfo;
+    sDebug("%s, get file info:%s", pPeer->id, minfo.name);
+    sinfo.magic = (*pNode->getFileInfo)(pNode->ahandle, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion);
+
+    // if file not there or magic is not the same, file shall be synced
+    memset(&fileAck, 0, sizeof(fileAck));
+    fileAck.sync = (sinfo.magic != minfo.magic || sinfo.name[0] == 0) ? 1:0;
+
+    // send file ack
+    ret = taosWriteMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck));
+    if (ret <0)  break;
+ 
+    // if sync is not required, continue
+    if (fileAck.sync == 0) {
+      sDebug("%s, %s is the same", pPeer->id, minfo.name);
+      continue;
+    }
+
+    // if sync is required, open file, receive from master, and write to file
+    // get the full path to file
+    snprintf(name, sizeof(name), "%s/%s", pNode->path, minfo.name);
+
+    int dfd = open(name, O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU | S_IRWXG | S_IRWXO);
+    if ( dfd < 0 ) {
+      sError("%s, failed to open file:%s", pPeer->id, name);
+      break;
+    }
+
+    ret = taosCopyFds(pPeer->syncFd, dfd, minfo.size);
+    fsync(dfd);
+    close(dfd);
+    if (ret<0) break;
+
+    sDebug("%s, %s is received, size:%d", pPeer->id, minfo.name, minfo.size);
+
+  }
+
+  if (code == 0 && (minfo.fversion != sinfo.fversion)) {
+   // data file is changed, code shall be set to 1 
+    *fversion = minfo.fversion;
+    code = 1;  
+  }
+
+  if (code < 0) {
+    sError("%s, failed to restore %s(%s)", pPeer->id, name, strerror(errno));
+  }
+
+  return code;
+}
+
+static int syncRestoreWal(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  int         ret, code = -1;
+
+  void *buffer = calloc(1024000, 1);  // size for one record
+  if (buffer == NULL) return -1;
+
+  SWalHead *pHead = (SWalHead *)buffer;
+
+  while (1) {
+    ret = taosReadMsg(pPeer->syncFd, pHead, sizeof(SWalHead));
+    if (ret <0)  break;
+
+    if (pHead->len == 0) {code = 0; break;}  // wal sync over
+    
+    ret = taosReadMsg(pPeer->syncFd, pHead->cont, pHead->len);
+    if (ret <0)  break;
+
+    sDebug("%s, restore a record, ver:%" PRIu64, pPeer->id, pHead->version);
+    (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_WAL);
+  }
+
+  if (code<0) {
+    sError("%s, failed to restore wal(%s)", pPeer->id, strerror(errno));
+  }
+
+  free(buffer);
+  return code;
+}
+
+static char *syncProcessOneBufferedFwd(SSyncPeer *pPeer, char *offset)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  SWalHead  *pHead = (SWalHead *) offset;
+
+  (*pNode->writeToCache)(pNode->ahandle, pHead, TAOS_QTYPE_FWD);
+  offset += pHead->len + sizeof(SWalHead);
+
+  return offset;
+}
+
+static int syncProcessBufferedFwd(SSyncPeer *pPeer)
+{
+  SSyncNode   *pNode = pPeer->pSyncNode;
+  SRecvBuffer *pRecv = pNode->pRecv;
+  int          forwards = 0;
+
+  sDebug("%s, number of buffered forwards:%d", pPeer->id, pRecv->forwards);
+
+  char *offset = pRecv->buffer;
+  while (forwards < pRecv->forwards) {
+    offset = syncProcessOneBufferedFwd(pPeer, offset);
+    forwards++;
+  }
+  
+  pthread_mutex_lock(&pNode->mutex);
+
+  while (forwards < pRecv->forwards && pRecv->code == 0) {
+    offset = syncProcessOneBufferedFwd(pPeer, offset);
+    forwards++;
+  }
+
+  nodeRole = TAOS_SYNC_ROLE_SLAVE;
+  sDebug("%s, finish processing buffered fwds:%d", pPeer->id, forwards);
+
+  pthread_mutex_unlock(&pNode->mutex);
+
+  return pRecv->code;
+}
+
+int syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead)
+{ 
+  SSyncNode   *pNode = pPeer->pSyncNode;
+  SRecvBuffer *pRecv = pNode->pRecv;
+
+  if (pRecv == NULL) return -1;
+  int len = pHead->len + sizeof(SWalHead);
+
+  if (pRecv->bufferSize - (pRecv->offset - pRecv->buffer) >= len) {
+    memcpy(pRecv->offset, pHead, len);
+    pRecv->offset += len;
+    pRecv->forwards++;
+    sDebug("%s, fwd is saved into queue, ver:%" PRIu64 " fwds:%d", pPeer->id, pHead->version, pRecv->forwards);
+  } else {
+    sError("%s, buffer size:%d is too small", pPeer->id, pRecv->bufferSize);
+    pRecv->code = -1;  // set error code
+  }
+
+  return pRecv->code;
+}
+
+static void syncCloseRecvBuffer(SSyncNode *pNode)
+{
+  if (pNode->pRecv) {
+    tfree(pNode->pRecv->buffer);
+  }
+
+  tfree(pNode->pRecv);
+}
+
+static int syncOpenRecvBuffer(SSyncNode *pNode) 
+{
+  syncCloseRecvBuffer(pNode);
+
+  SRecvBuffer *pRecv = calloc(sizeof(SRecvBuffer), 1);
+  if (pRecv == NULL) return -1;
+
+  pRecv->bufferSize = 5000000;
+  pRecv->buffer = malloc(pRecv->bufferSize);
+  if (pRecv->buffer == NULL) {
+    free(pRecv);
+    return -1;
+  }
+
+  pRecv->offset = pRecv->buffer;
+  pRecv->forwards = 0;
+
+  pNode->pRecv = pRecv;
+
+  return 0;
+}
+
+static int syncRestoreDataStepByStep(SSyncPeer *pPeer)
+{
+  SSyncNode *pNode = pPeer->pSyncNode;
+  nodeSStatus = TAOS_SYNC_STATUS_FILE;
+  uint64_t fversion = 0;
+
+  sDebug("%s, start to restore file", pPeer->id);
+  int code = syncRestoreFile(pPeer, &fversion);
+  if (code < 0) {
+    sError("%s, failed to restore file", pPeer->id);
+    return -1;
+  }
+
+  // if code > 0, data file is changed, notify app, and pass the version 
+  if (code > 0 && pNode->notifyFileSynced) {
+    if ( (*pNode->notifyFileSynced)(pNode->ahandle, fversion) < 0 ) {
+      sError("%s, app not in ready state", pPeer->id);
+      return -1;
+    }
+  }
+
+  nodeVersion = fversion;
+
+  sDebug("%s, start to restore wal", pPeer->id);
+  if (syncRestoreWal(pPeer) < 0) {
+    sError("%s, failed to restore wal", pPeer->id);
+    return -1;
+  }
+
+  nodeSStatus = TAOS_SYNC_STATUS_CACHE;
+  sDebug("%s, start to insert buffered points", pPeer->id);
+  if (syncProcessBufferedFwd(pPeer) < 0) {
+    sError("%s, failed to insert buffered points", pPeer->id);
+    return -1;
+  }
+
+  return 0;
+}
+
+void *syncRestoreData(void *param)
+{
+  SSyncPeer  *pPeer = (SSyncPeer *)param;
+  SSyncNode  *pNode = pPeer->pSyncNode;
+
+  taosBlockSIGPIPE();
+  __sync_fetch_and_add(&tsSyncNum, 1);
+
+  if (syncOpenRecvBuffer(pNode) < 0) {
+    sError("%s, failed to allocate recv buffer", pPeer->id);
+  } else { 
+    if ( syncRestoreDataStepByStep(pPeer) == 0) {
+      sInfo("%s, it is synced successfully", pPeer->id);
+      nodeRole = TAOS_SYNC_ROLE_SLAVE;
+      syncBroadcastStatus(pNode);
+      (*pNode->notifyRole)(pNode->ahandle, nodeRole);
+    } else {
+      sError("%s, failed to restore data, restart connection", pPeer->id);
+      nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  nodeSStatus = TAOS_SYNC_STATUS_INIT;
+  tclose(pPeer->syncFd)
+  syncCloseRecvBuffer(pNode);
+  __sync_fetch_and_sub(&tsSyncNum, 1);
+  syncDecPeerRef(pPeer);
+
+  return NULL;
+}
+

src/sync/src/syncRetrieve.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <stdint.h>
+#include <stdbool.h>
+#include <sys/inotify.h>
+#include <unistd.h>
+#include "os.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "tglobal.h"
+#include "ttimer.h"
+#include "tsocket.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+static int syncAddIntoWatchList(SSyncPeer *pPeer, char *name) 
+{
+  sDebug("%s, start to monitor:%s", pPeer->id, name);
+
+  if (pPeer->notifyFd <=0) {
+    pPeer->watchNum = 0;
+    pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
+    if (pPeer->notifyFd < 0) {
+      sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno));
+      return -1;
+    }
+
+    if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int)*tsMaxWatchFiles);
+    if (pPeer->watchFd == NULL) {
+      sError("%s, failed to allocate watchFd", pPeer->id);
+      return -1;
+    }
+
+    memset(pPeer->watchFd, -1, sizeof(int)*tsMaxWatchFiles);
+  }
+
+  int *wd = pPeer->watchFd + pPeer->watchNum;
+
+  if (*wd >= 0) {
+    if (inotify_rm_watch(pPeer->notifyFd, *wd) < 0) {
+      sError("%s, failed to remove wd:%d(%s)", pPeer->id, *wd, strerror(errno));
+      return -1;
+    }
+  }
+
+  *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY);
+  if (*wd == -1) {
+    sError("%s, failed to add %s(%s)", pPeer->id, name, strerror(errno));
+    return -1;
+  }
+
+  pPeer->watchNum++;
+  pPeer->watchNum = (pPeer->watchNum +1) % tsMaxWatchFiles;
+
+  return 0;
+}
+
+static int syncAreFilesModified(SSyncPeer *pPeer) 
+{
+  if (pPeer->notifyFd <=0) return 0;
+
+  char buf[2048]; 
+  int len = read(pPeer->notifyFd, buf, sizeof(buf));
+  if (len <0 && errno != EAGAIN) {
+    sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));    
+    return -1;
+  }
+    
+  int code = 0; 
+  if (len >0) { 
+    sDebug("%s, processed file is changed", pPeer->id);    
+    code = 1;
+  }
+
+  return code;  
+}
+
+static int syncRetrieveFile(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  SFileInfo   fileInfo;
+  SFileAck    fileAck;
+  int         code = -1;
+  char        name[TSDB_FILENAME_LEN * 2] = {0};
+
+  memset(&fileInfo, 0, sizeof(fileInfo));
+  memset(&fileAck, 0, sizeof(fileAck));
+
+  while (1) {
+    // retrieve file info
+    fileInfo.name[0] = 0;
+    fileInfo.magic = (*pNode->getFileInfo)(pNode->ahandle, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX, &fileInfo.size, &fileInfo.fversion);   
+    //fileInfo.size = htonl(size);
+
+    // send the file info
+    int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(fileInfo));
+    if (ret < 0 ) break;
+
+    // if no file anymore, break
+    if (fileInfo.magic == 0 || fileInfo.name[0] == 0) { 
+      sDebug("%s, no more files to sync", pPeer->id);    
+      code = 0; break; 
+    }
+
+    // wait for the ack from peer
+    ret = taosReadMsg(pPeer->syncFd, &(fileAck), sizeof(fileAck));
+    if (ret <0)  break;
+
+    // set the peer sync version
+    pPeer->sversion = fileInfo.fversion;
+
+    // get the full path to file
+    snprintf(name, sizeof(name), "%s/%s", pNode->path, fileInfo.name);
+    
+    // add the file into watch list
+    if ( syncAddIntoWatchList(pPeer, name) <0) break;
+
+    // if sync is not required, continue
+    if (fileAck.sync == 0) {
+      fileInfo.index++; 
+      sDebug("%s, %s is the same", pPeer->id, fileInfo.name);    
+      continue; 
+    }
+
+    // send the file to peer
+    int sfd = open(name, O_RDONLY);
+    if ( sfd < 0 ) break;
+
+    ret = taosTSendFile(pPeer->syncFd, sfd, NULL, fileInfo.size); 
+    close(sfd); 
+    if (ret <0) break;
+
+    sDebug("%s, %s is sent, size:%d", pPeer->id, name, fileInfo.size);    
+    fileInfo.index++; 
+
+    // check if processed files are modified 
+    if (syncAreFilesModified(pPeer) != 0) break;
+  }
+
+  if (code < 0) {
+    sError("%s, failed to retrieve file(%s)", pPeer->id, strerror(errno));
+  }
+
+  return code;
+}
+
+/* if only a partial record is read out, set the IN_MODIFY flag in event,
+   so upper layer will reload the file to get a complete record */
+static int syncReadOneWalRecord(int sfd, SWalHead *pHead, uint32_t *pEvent) 
+{ 
+  int ret;
+
+  ret = read(sfd, pHead, sizeof(SWalHead));
+  if (ret < 0) return -1;
+  if (ret == 0) return 0;
+
+  if (ret != sizeof(SWalHead)) {
+    // file is not at end yet, it shall be reloaded
+    *pEvent = *pEvent | IN_MODIFY;
+    return 0;
+  }
+
+  ret = read(sfd, pHead->cont, pHead->len);
+  if (ret <0) return -1;
+
+  if (ret != pHead->len) {
+    // file is not at end yet, it shall be reloaded
+    *pEvent = *pEvent | IN_MODIFY;
+    return 0;
+  }
+
+  return sizeof(SWalHead) + pHead->len;
+}    
+
+static int syncMonitorLastWal(SSyncPeer *pPeer, char *name) 
+{ 
+  pPeer->watchNum = 0;
+  tclose(pPeer->notifyFd);
+  pPeer->notifyFd = inotify_init1(IN_NONBLOCK);
+  if (pPeer->notifyFd < 0) {
+    sError("%s, failed to init inotify(%s)", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  if (pPeer->watchFd == NULL) pPeer->watchFd = malloc(sizeof(int)*tsMaxWatchFiles);
+  if (pPeer->watchFd == NULL) {
+    sError("%s, failed to allocate watchFd", pPeer->id);
+    return -1;
+  }
+
+  memset(pPeer->watchFd, -1, sizeof(int)*tsMaxWatchFiles);
+  int *wd = pPeer->watchFd;
+ 
+  *wd = inotify_add_watch(pPeer->notifyFd, name, IN_MODIFY | IN_CLOSE_WRITE);
+  if (*wd == -1) {
+    sError("%s, failed to watch last wal(%s)", pPeer->id, strerror(errno));
+    return -1;
+  }
+
+  return 0; 
+}
+
+static uint32_t syncCheckLastWalChanges(SSyncPeer *pPeer, uint32_t *pEvent) 
+{
+  char       buf[2048]; 
+  int  len = read(pPeer->notifyFd, buf, sizeof(buf));
+  if (len <0 && errno != EAGAIN) {
+    sError("%s, failed to read notify FD(%s)", pPeer->id, strerror(errno));    
+    return -1;
+  }
+    
+  if (len == 0) return 0;
+
+  struct inotify_event *event;
+  for (char *ptr = buf; ptr < buf + len; ptr += sizeof(struct inotify_event) + event->len) {
+    event = (struct inotify_event *) ptr;
+    if (event->mask & IN_MODIFY) *pEvent = *pEvent | IN_MODIFY;
+    if (event->mask & IN_CLOSE_WRITE) *pEvent = *pEvent | IN_CLOSE_WRITE;
+  }
+
+  if (pEvent != 0)
+    sDebug("%s, last wal event:0x%x", pPeer->id, *pEvent);
+
+  return 0;
+}
+
+static int syncRetrieveLastWal(SSyncPeer *pPeer, char *name, uint64_t fversion, int64_t offset, uint32_t *pEvent) 
+{
+  SWalHead  *pHead = (SWalHead *) malloc(640000);
+  int        code = -1;
+  int32_t    bytes = 0;
+  int        sfd;
+
+  sfd = open(name, O_RDONLY);
+  if (sfd < 0) return -1;
+  lseek(sfd, offset, SEEK_SET);
+  sDebug("%s, retrieve last wal, offset:%" PRId64 " fversion:%" PRIu64, pPeer->id, offset, fversion);
+
+  while (1) {
+    int wsize = syncReadOneWalRecord(sfd, pHead, pEvent); 
+    if (wsize <0) break;
+    if (wsize == 0) { code = 0; break; }
+
+    sDebug("%s, last wal is forwarded, ver:%" PRIu64, pPeer->id, pHead->version);
+    int ret = taosWriteMsg(pPeer->syncFd, pHead, wsize);
+    if ( ret != wsize ) break;
+    pPeer->sversion = pHead->version;
+
+    bytes += wsize;
+ 
+    if (pHead->version >= fversion && fversion > 0) {
+      code = 0; 
+      bytes = 0; 
+      break;
+    }
+  }
+
+  free(pHead);
+  tclose(sfd); 
+
+  if (code == 0) return bytes;
+  return -1;
+}
+
+static int syncProcessLastWal(SSyncPeer *pPeer, char *wname, uint32_t index) 
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  int         code = -1;
+  char        fname[TSDB_FILENAME_LEN * 2];  // full path to wal file
+
+  if (syncAreFilesModified(pPeer) != 0) return -1;
+
+  while (1) {
+    int32_t  once = 0; // last WAL has once ever been processed 
+    int64_t  offset = 0;
+    uint64_t fversion = 0;
+    uint32_t event = 0;
+
+    // get full path to wal file
+    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
+    sDebug("%s, start to retrieve last wal:%s", pPeer->id, fname);
+
+    // monitor last wal
+    if (syncMonitorLastWal(pPeer, fname) <0) break;
+
+    while (1) {
+      int32_t bytes = syncRetrieveLastWal(pPeer, fname, fversion, offset, &event);
+      if (bytes < 0) break;
+
+      // check file changes
+      if (syncCheckLastWalChanges(pPeer, &event) <0) break;
+
+      // if file is not updated or updated once, set the fversion and sstatus
+      if (((event & IN_MODIFY) == 0) || once) {
+        if (fversion == 0) {
+          pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;  // start to forward pkt
+          fversion = nodeVersion;    // must read data to fversion
+        }
+      }
+
+      // if all data up to fversion is read out, it is over
+      if (pPeer->sversion >= fversion && fversion > 0) {
+        code = 0; 
+        sDebug("%s, data up to fversion:%ld has been read out, bytes:%d", pPeer->id, fversion, bytes);
+        break;
+      }  
+
+      // if all data are read out, and no update
+      if ((bytes == 0) && ((event & IN_MODIFY) == 0)) {
+        // wal file is closed, break
+        if (event & IN_CLOSE_WRITE) { 
+          code = 0; 
+          sDebug("%s, current wal is closed", pPeer->id);
+          break;
+        }
+ 
+        // wal not closed, it means some data not flushed to disk, wait for a while
+        usleep(10000);
+      }
+
+      // if bytes>0, file is updated, or fversion is not reached but file still open, read again 
+      once = 1;
+      offset += bytes; 
+      sDebug("%s, retrieve last wal, bytes:%d", pPeer->id, bytes);
+      event = event & (~IN_MODIFY); // clear IN_MODIFY flag 
+    }
+
+    if (code < 0) break;
+    if (pPeer->sversion >= fversion && fversion > 0) break;  
+
+    index++;  wname[0] = 0;
+    code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index);
+    if ( code < 0) break;  
+    if ( wname[0] == 0 ) {code = 0; break;}
+
+    // current last wal is closed, there is a new one 
+    sDebug("%s, last wal is closed, try new one", pPeer->id);
+  }
+
+  tclose(pPeer->notifyFd);
+
+  return code;
+}
+
+static int syncRetrieveWal(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+  char        fname[TSDB_FILENAME_LEN * 3];
+  char        wname[TSDB_FILENAME_LEN * 2];
+  int32_t     size;
+  struct stat fstat;
+  int         code = -1;
+  uint32_t    index = 0;
+
+  while (1) {
+    // retrieve wal info
+    wname[0] = 0;
+    code = (*pNode->getWalInfo)(pNode->ahandle, wname, &index);   
+    if (code < 0) break;  // error
+    if (wname[0] == 0) {  // no wal file
+      sDebug("%s, no wal file", pPeer->id);
+      break;
+    }    
+      
+    if (code == 0) {   // last wal 
+      code = syncProcessLastWal(pPeer, wname, index);
+      break;
+    }
+
+    // get the full path to wal file
+    snprintf(fname, sizeof(fname), "%s/%s", pNode->path, wname);
+
+    // send wal file, 
+    // inotify is not required, old wal file won't be modified, even remove is ok
+    if ( stat(fname, &fstat) < 0 ) break;
+    size = fstat.st_size;
+
+    sDebug("%s, retrieve wal:%s size:%d", pPeer->id, fname, size);    
+    int sfd = open(fname, O_RDONLY);
+    if (sfd < 0) break;
+
+    code = taosTSendFile(pPeer->syncFd, sfd, NULL, size); 
+    close(sfd); 
+    if (code <0) break;
+
+    index++; 
+
+    if (syncAreFilesModified(pPeer) != 0) break; 
+  }
+
+  if (code == 0) {
+    sDebug("%s, wal retrieve is finished", pPeer->id);    
+    pPeer->sstatus = TAOS_SYNC_STATUS_CACHE;
+    SWalHead walHead;
+    memset(&walHead, 0, sizeof(walHead));
+    code = taosWriteMsg(pPeer->syncFd, &walHead, sizeof(walHead));
+  } else {
+    sError("%s, failed to send wal(%s)", pPeer->id, strerror(errno));
+  }
+
+  return code;
+}
+
+static int syncRetrieveDataStepByStep(SSyncPeer *pPeer)
+{
+  SSyncNode  *pNode = pPeer->pSyncNode;
+
+  SFirstPkt firstPkt;
+  memset(&firstPkt, 0, sizeof(firstPkt));
+  firstPkt.syncHead.type = TAOS_SMSG_SYNC_DATA;
+  firstPkt.syncHead.vgId = pNode->vgId;
+  tstrncpy(firstPkt.fqdn, tsNodeFqdn, sizeof(firstPkt.fqdn));
+  firstPkt.port = tsSyncPort;
+
+  if (write(pPeer->syncFd, (char *) &firstPkt, sizeof(firstPkt)) < 0) {
+    sError("%s, failed to send syncCmd", pPeer->id);
+    return -1;
+  }
+
+  pPeer->sversion = 0;  
+  pPeer->sstatus = TAOS_SYNC_STATUS_FILE;
+  sDebug("%s, start to retrieve file", pPeer->id);
+  if (syncRetrieveFile(pPeer) < 0) {
+    sError("%s, failed to retrieve file", pPeer->id);
+    return -1;
+  }
+
+  // if no files are synced, there must be wal to sync, sversion must be larger than one
+  if (pPeer->sversion == 0) 
+    pPeer->sversion = 1;
+
+  sDebug("%s, start to retrieve wal", pPeer->id);
+  if (syncRetrieveWal(pPeer) < 0) {
+    sError("%s, failed to retrieve wal", pPeer->id);
+    return -1;
+  }
+
+  return 0;
+}
+
+void *syncRetrieveData(void *param)
+{
+  SSyncPeer   *pPeer = (SSyncPeer *)param;
+  taosBlockSIGPIPE();
+
+  pPeer->syncFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
+  if (pPeer->syncFd < 0) {
+    sError("%s, failed to open socket to sync", pPeer->id);
+  } else {
+    sInfo("%s, sync tcp is setup", pPeer->id);
+  
+    if (syncRetrieveDataStepByStep(pPeer) == 0) {
+      sDebug("%s, sync retrieve process is successful", pPeer->id);
+    } else {
+      sError("%s, failed to retrieve data, restart connection", pPeer->id);
+      syncRestartConnection(pPeer);
+    }
+  }
+
+  tclose(pPeer->notifyFd);
+  tclose(pPeer->syncFd);
+  syncDecPeerRef(pPeer);
+
+  return NULL;
+}

src/sync/src/taosTcpPool.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tulog.h"
+#include "tutil.h"
+#include "tsocket.h"
+#include "taoserror.h"
+#include "taosTcpPool.h"
+
+typedef struct SThreadObj {
+  pthread_t thread;
+  bool      stop;
+  int       pollFd;
+  int       numOfFds;
+  struct SPoolObj *pPool;
+} SThreadObj;
+
+typedef struct SPoolObj {
+  SPoolInfo    info;
+  SThreadObj **pThread;
+  pthread_t    thread;
+  int          nextId;
+  int          acceptFd;  // FD for accept new connection
+} SPoolObj;
+
+typedef struct {
+  SThreadObj *pThread;
+  void       *ahandle;
+  int         fd;
+  int         closedByApp; 
+} SConnObj;
+
+static void *taosAcceptPeerTcpConnection(void *argv);
+static void *taosProcessTcpData(void *param);
+static SThreadObj *taosGetTcpThread(SPoolObj *pPool);
+static void taosStopPoolThread(SThreadObj* pThread);
+
+void *taosOpenTcpThreadPool(SPoolInfo *pInfo)
+{
+  pthread_attr_t thattr;
+
+  SPoolObj *pPool = calloc(sizeof(SPoolObj), 1);
+  if (pPool == NULL) {
+    uError("TCP server, no enough memory");
+    return NULL;
+  }
+
+  pPool->info = *pInfo;
+  
+  pPool->pThread = (SThreadObj **) calloc(sizeof(SThreadObj *), pInfo->numOfThreads);
+  if (pPool->pThread == NULL) {
+    uError("TCP server, no enough memory");
+    free(pPool);
+    return NULL;
+  }
+
+  pPool->acceptFd = taosOpenTcpServerSocket(pInfo->serverIp, pInfo->port);
+  if (pPool->acceptFd < 0) {
+    free(pPool->pThread); free(pPool);
+    uError("failed to create TCP server socket, port:%d (%s)", pInfo->port, strerror(errno));
+    return NULL;
+  }
+
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  if (pthread_create(&(pPool->thread), &thattr, (void *) taosAcceptPeerTcpConnection, pPool) != 0) {
+    uError("TCP server, failed to create accept thread, reason:%s", strerror(errno));
+    close(pPool->acceptFd);
+    free(pPool->pThread); free(pPool);
+    return NULL;
+  }
+
+  pthread_attr_destroy(&thattr);
+
+  uDebug("%p TCP pool is created", pPool);
+  return pPool;
+}
+
+void taosCloseTcpThreadPool(void *param)
+{
+  SPoolObj    *pPool = (SPoolObj *)param;
+  SThreadObj  *pThread;
+
+  shutdown(pPool->acceptFd, SHUT_RD);  
+  pthread_join(pPool->thread, NULL);
+
+  for (int i = 0; i < pPool->info.numOfThreads; ++i) {
+    pThread = pPool->pThread[i];
+    if (pThread) taosStopPoolThread(pThread); 
+  }
+
+  tfree(pPool->pThread);
+  free(pPool);
+  uDebug("%p TCP pool is closed", pPool);
+}
+
+void *taosAllocateTcpConn(void *param, void *pPeer, int connFd)
+{
+  struct epoll_event event;
+  SPoolObj *pPool = (SPoolObj *)param;
+
+  SConnObj *pConn = (SConnObj *) calloc(sizeof(SConnObj), 1);
+  if (pConn == NULL) {
+    terrno = TAOS_SYSTEM_ERROR(errno); 
+    return NULL;
+  }
+
+  SThreadObj *pThread = taosGetTcpThread(pPool);
+  if (pThread == NULL) {
+    free(pConn);
+    return NULL;
+  }
+
+  pConn->fd = connFd;
+  pConn->pThread = pThread;
+  pConn->ahandle = pPeer;
+  pConn->closedByApp = 0;
+
+  event.events = EPOLLIN | EPOLLRDHUP;
+  event.data.ptr = pConn;
+
+  if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, connFd, &event) < 0) {
+    uError("failed to add fd:%d(%s)", connFd, strerror(errno));
+    terrno = TAOS_SYSTEM_ERROR(errno); 
+    free(pConn);
+    pConn = NULL;
+  } else {
+    pThread->numOfFds++;
+    uDebug("%p fd:%d is added to epoll thread, num:%d", pThread, connFd, pThread->numOfFds);
+  }
+
+  return pConn;
+}
+
+void taosFreeTcpConn(void *param)
+{
+  SConnObj *pConn = (SConnObj *)param;
+  SThreadObj   *pThread = pConn->pThread;
+
+  uDebug("%p TCP connection will be closed, fd:%d", pThread, pConn->fd);
+  pConn->closedByApp = 1;
+  shutdown(pConn->fd, SHUT_WR);
+}
+
+static void taosProcessBrokenLink(SConnObj *pConn) {
+  SThreadObj *pThread = pConn->pThread;
+  SPoolObj   *pPool = pThread->pPool;
+  SPoolInfo  *pInfo = &pPool->info;
+  
+  if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR);
+  (*pInfo->processBrokenLink)(pConn->ahandle);
+
+  pThread->numOfFds--;
+  epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL);
+  uDebug("%p fd:%d is removed from epoll thread, num:%d", pThread, pConn->fd, pThread->numOfFds);
+  tclose(pConn->fd);
+  free(pConn);
+}
+
+#define maxEvents 10
+
+static void *taosProcessTcpData(void *param) {
+  SThreadObj     *pThread = (SThreadObj *) param;
+  SPoolObj       *pPool = pThread->pPool;
+  SPoolInfo      *pInfo = &pPool->info;
+  SConnObj       *pConn = NULL;
+  struct epoll_event events[maxEvents];
+
+  void *buffer = malloc(pInfo->bufferSize);
+  taosBlockSIGPIPE();
+
+  while (1) {
+    if (pThread->stop) break; 
+    int fdNum = epoll_wait(pThread->pollFd, events, maxEvents, -1);
+    if (pThread->stop) {
+      uDebug("%p TCP epoll thread is exiting...", pThread);
+      break;
+    }
+
+    if (fdNum < 0) { 
+      uError("epoll_wait failed (%s)", strerror(errno));
+      continue;
+    }
+
+    for (int i = 0; i < fdNum; ++i) {
+      pConn = events[i].data.ptr;
+      assert(pConn);
+
+      if (events[i].events & EPOLLERR) {
+        taosProcessBrokenLink(pConn);
+        continue;
+      }
+
+      if (events[i].events & EPOLLHUP) {
+        taosProcessBrokenLink(pConn);
+        continue;
+      }
+
+      if (events[i].events & EPOLLRDHUP) {
+        taosProcessBrokenLink(pConn);
+        continue;
+      }
+
+      if (pConn->closedByApp == 0) {
+        if ((*pInfo->processIncomingMsg)(pConn->ahandle, buffer) < 0) {
+          taosFreeTcpConn(pConn);
+          continue;
+        }
+      } 
+    }
+  }
+
+  close(pThread->pollFd);
+  free(pThread);
+  free(buffer);
+  uDebug("%p TCP epoll thread exits", pThread);
+  return NULL;  
+}
+
+static void *taosAcceptPeerTcpConnection(void *argv) {
+  SPoolObj   *pPool = (SPoolObj *)argv;
+  SPoolInfo  *pInfo = &pPool->info;
+
+  taosBlockSIGPIPE();
+
+  while (1) {
+    struct sockaddr_in clientAddr;
+    socklen_t addrlen = sizeof(clientAddr);
+    int connFd = accept(pPool->acceptFd, (struct sockaddr *) &clientAddr, &addrlen);
+    if (connFd < 0) {
+      if (errno == EINVAL) {
+        uDebug("%p TCP server accept is exiting...", pPool);
+        break;
+      } else {
+        uError("TCP accept failure, reason:%s", strerror(errno));
+        continue;
+      }
+    }
+
+    //uDebug("TCP connection from: 0x%x:%d", clientAddr.sin_addr.s_addr, clientAddr.sin_port); 
+    taosKeepTcpAlive(connFd);
+    (*pInfo->processIncomingConn)(connFd, clientAddr.sin_addr.s_addr);
+  }
+
+  tclose(pPool->acceptFd);
+  return NULL;
+}
+
+static SThreadObj *taosGetTcpThread(SPoolObj *pPool) {
+  SThreadObj *pThread = pPool->pThread[pPool->nextId];
+
+  if (pThread) return pThread;
+
+  pThread = (SThreadObj *) calloc(1, sizeof(SThreadObj));
+  if (pThread == NULL) return NULL;
+
+  pThread->pPool = pPool;
+  pThread->pollFd = epoll_create(10);  // size does not matter
+  if (pThread->pollFd < 0) {
+    free(pThread);
+    return NULL;
+  }
+
+  pthread_attr_t thattr;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  int ret = pthread_create(&(pThread->thread), &thattr, (void *) taosProcessTcpData, pThread);
+  pthread_attr_destroy(&thattr);
+
+  if (ret != 0) {
+    close(pThread->pollFd);
+    free(pThread);
+    return NULL;
+  }
+
+  uDebug("%p TCP epoll thread is created", pThread);
+  pPool->pThread[pPool->nextId] = pThread;
+  pPool->nextId++;
+  pPool->nextId = pPool->nextId % pPool->info.numOfThreads;
+
+  return pThread;
+}
+
+static void taosStopPoolThread(SThreadObj* pThread) {
+  pThread->stop = true;
+  
+  if (pThread->thread == pthread_self()) {
+    pthread_detach(pthread_self());
+    return;
+  }
+
+  // save thread ID into a local variable, since pThread is freed when the thread exits 
+  pthread_t thread = pThread->thread;
+
+  // signal the thread to stop, try graceful method first,
+  // and use pthread_cancel when failed
+  struct epoll_event event = { .events = EPOLLIN };
+  eventfd_t fd = eventfd(1, 0);
+  if (fd == -1) {
+    // failed to create eventfd, call pthread_cancel instead, which may result in data corruption
+    uError("failed to create eventfd(%s)", strerror(errno));
+    pthread_cancel(pThread->thread);
+  } else if (epoll_ctl(pThread->pollFd, EPOLL_CTL_ADD, fd, &event) < 0) {
+    // failed to call epoll_ctl, call pthread_cancel instead, which may result in data corruption
+    uError("failed to call epoll_ctl(%s)", strerror(errno));
+    pthread_cancel(pThread->thread);
+  }
+
+  pthread_join(thread, NULL);
+  tclose(fd);
+}
+

src/sync/src/tarbitrator.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//#define _DEFAULT_SOURCE
+#include "os.h"
+#include "hash.h"
+#include "tlog.h"
+#include "tutil.h"
+#include "ttimer.h"
+#include "ttime.h"
+#include "tsocket.h"
+#include "tglobal.h"
+#include "taoserror.h"
+#include "taosTcpPool.h"
+#include "twal.h"
+#include "tsync.h"
+#include "syncInt.h"
+
+static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context);
+static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp);
+static void arbProcessBrokenLink(void *param);
+static int  arbProcessPeerMsg(void *param, void *buffer);
+static sem_t    tsArbSem;
+static ttpool_h tsArbTcpPool;
+
+typedef struct {
+  char  id[TSDB_EP_LEN+24];
+  int   nodeFd;
+  void *pConn;
+} SNodeConn;
+
+int main(int argc, char *argv[]) {
+  char     arbLogPath[TSDB_FILENAME_LEN + 16] = {0};
+
+  for (int i=1; i<argc; ++i) {
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+      tsServerPort = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
+      debugFlag = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-g")==0 && i < argc-1) {
+      if (strlen(argv[++i]) > TSDB_FILENAME_LEN) continue; 
+      tstrncpy(arbLogPath, argv[i], sizeof(arbLogPath));
+    } else {
+      printf("\nusage: %s [options] \n", argv[0]);
+      printf("  [-p port]: server port number, default is:%d\n", tsServerPort);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", debugFlag);
+      printf("  [-g logFilePath]: log file pathe, default:%s\n", arbLogPath);
+      printf("  [-h help]: print out this help\n\n");
+      exit(0);
+    }
+  }
+  
+ if (sem_init(&tsArbSem, 0, 0) != 0) {
+    printf("failed to create exit semphore\n");
+    exit(EXIT_FAILURE);
+  }
+
+  /* Set termination handler. */
+  struct sigaction act = {{0}};
+  act.sa_flags = SA_SIGINFO;
+  act.sa_sigaction = arbSignalHandler;
+  sigaction(SIGTERM, &act, NULL);
+  sigaction(SIGHUP, &act, NULL);
+  sigaction(SIGINT, &act, NULL);
+
+  tsAsyncLog = 0;
+  strcat(arbLogPath, "/arbitrator.log");
+  taosInitLog(arbLogPath, 1000000, 10);
+
+  taosGetFqdn(tsNodeFqdn);
+  tsSyncPort = tsServerPort + TSDB_PORT_SYNC;
+
+  SPoolInfo info;
+  info.numOfThreads = 1;
+  info.serverIp = 0;
+  info.port = tsSyncPort;
+  info.bufferSize = 640000;
+  info.processBrokenLink = arbProcessBrokenLink;
+  info.processIncomingMsg = arbProcessPeerMsg;
+  info.processIncomingConn = arbProcessIncommingConnection;
+  tsArbTcpPool = taosOpenTcpThreadPool(&info);
+  
+  if (tsArbTcpPool == NULL) {
+   sDebug("failed to open TCP thread pool, exit...");
+   return -1;
+  }
+
+  sInfo("TAOS arbitrator: %s:%d is running", tsNodeFqdn, tsServerPort);
+
+  for (int res = sem_wait(&tsArbSem); res != 0; res = sem_wait(&tsArbSem)) {
+    if (res != EINTR) break;
+  }
+
+  taosCloseTcpThreadPool(tsArbTcpPool);
+  sInfo("TAOS arbitrator is shut down\n");
+  closelog();
+
+  return 0;
+}
+
+static void arbProcessIncommingConnection(int connFd, uint32_t sourceIp)
+{
+  char  ipstr[24];
+  tinet_ntoa(ipstr, sourceIp);
+  sDebug("peer TCP connection from ip:%s", ipstr);
+
+  SFirstPkt firstPkt;
+  if (taosReadMsg(connFd, &firstPkt, sizeof(firstPkt)) != sizeof(firstPkt)) {
+    sError("failed to read peer first pkt from ip:%s(%s)", ipstr, strerror(errno));
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  SNodeConn *pNode = (SNodeConn *) calloc(sizeof(SNodeConn), 1);
+  if (pNode == NULL) {
+    sError("failed to allocate memory(%s)", strerror(errno));
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  snprintf(pNode->id, sizeof(pNode->id), "vgId:%d peer:%s:%d", firstPkt.sourceId, firstPkt.fqdn, firstPkt.port); 
+  if (firstPkt.syncHead.vgId) {  
+    sDebug("%s, vgId in head is not zero, close the connection", pNode->id);
+    tfree(pNode);
+    taosCloseSocket(connFd);
+    return;
+  }
+
+  sDebug("%s, arbitrator request is accepted", pNode->id);
+  pNode->nodeFd = connFd;
+  pNode->pConn = taosAllocateTcpConn(tsArbTcpPool, pNode, connFd);
+
+  return;
+}
+
+static void arbProcessBrokenLink(void *param) {
+  SNodeConn *pNode = param;
+
+  sDebug("%s, TCP link is broken(%s), close connection", pNode->id, strerror(errno));
+  tfree(pNode);
+}
+
+static int arbProcessPeerMsg(void *param, void *buffer)
+{
+  SNodeConn  *pNode = param;
+  SSyncHead   head;
+  int         bytes = 0;
+  char       *cont = (char *)buffer;
+
+  int hlen = taosReadMsg(pNode->nodeFd, &head, sizeof(head));
+  if (hlen != sizeof(head)) {
+    sDebug("%s, failed to read msg, hlen:%d", pNode->id, hlen);
+    return -1;
+  }
+
+  bytes = taosReadMsg(pNode->nodeFd, cont, head.len);
+  if (bytes != head.len) {
+    sDebug("%s, failed to read, bytes:%d len:%d", pNode->id, bytes, head.len);
+    return -1;
+  }
+
+  sDebug("%s, msg is received, len:%d", pNode->id, head.len);
+  return 0;
+}
+
+static void arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context) {
+
+  struct sigaction act = {{0}};
+  act.sa_handler = SIG_IGN;
+  sigaction(SIGTERM, &act, NULL);
+  sigaction(SIGHUP, &act, NULL);
+  sigaction(SIGINT, &act, NULL);
+
+  sInfo("shut down signal is %d, sender PID:%d", signum, sigInfo->si_pid);
+
+  // inform main thread to exit
+  sem_post(&tsArbSem);
+}
+

src/sync/test/CMakeLists.txt

+CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
+PROJECT(TDengine)
+
+IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
+  INCLUDE_DIRECTORIES(../inc)
+
+  LIST(APPEND CLIENT_SRC ./syncClient.c)
+  ADD_EXECUTABLE(syncClient ${CLIENT_SRC})
+  TARGET_LINK_LIBRARIES(syncClient sync trpc common)
+
+  LIST(APPEND SERVER_SRC ./syncServer.c)
+  ADD_EXECUTABLE(syncServer ${SERVER_SRC})
+  TARGET_LINK_LIBRARIES(syncServer sync trpc common)
+ENDIF ()
+
+

src/sync/test/syncClient.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "os.h"
+#include "tglobal.h"
+#include "tulog.h"
+#include "trpc.h"
+#include "taoserror.h"
+
+typedef struct {
+  int       index;
+  SRpcEpSet epSet;
+  int       num;
+  int       numOfReqs;
+  int       msgSize;
+  sem_t     rspSem; 
+  sem_t    *pOverSem; 
+  pthread_t thread;
+  void     *pRpc;
+} SInfo;
+
+void processResponse(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+  SInfo *pInfo = (SInfo *)pMsg->ahandle;
+  uDebug("thread:%d, response is received, type:%d contLen:%d code:0x%x", pInfo->index, pMsg->msgType, pMsg->contLen, pMsg->code);
+
+  if (pEpSet) pInfo->epSet = *pEpSet;
+  rpcFreeCont(pMsg->pCont);
+
+  sem_post(&pInfo->rspSem); 
+}
+
+int tcount = 0;
+
+void *sendRequest(void *param) {
+  SInfo  *pInfo = (SInfo *)param;
+  SRpcMsg rpcMsg = {0}; 
+  
+  uDebug("thread:%d, start to send request", pInfo->index);
+
+  while ( pInfo->numOfReqs == 0 || pInfo->num < pInfo->numOfReqs) {
+    pInfo->num++;
+    rpcMsg.pCont = rpcMallocCont(pInfo->msgSize);
+    rpcMsg.contLen = pInfo->msgSize;
+    rpcMsg.ahandle = pInfo;
+    rpcMsg.msgType = 1;
+    uDebug("thread:%d, send request, contLen:%d num:%d", pInfo->index, pInfo->msgSize, pInfo->num);
+    rpcSendRequest(pInfo->pRpc, &pInfo->epSet, &rpcMsg);
+    if ( pInfo->num % 20000 == 0 ) 
+      uInfo("thread:%d, %d requests have been sent", pInfo->index, pInfo->num);
+    sem_wait(&pInfo->rspSem);
+  }
+
+  uDebug("thread:%d, it is over", pInfo->index);
+  tcount++;
+
+  return NULL;
+}
+
+int main(int argc, char *argv[]) {
+  SRpcInit  rpcInit;
+  SRpcEpSet epSet;
+  char      secret[TSDB_KEY_LEN] = "mypassword";
+  int      msgSize = 128;
+  int      numOfReqs = 0;
+  int      appThreads = 1;
+  char     serverIp[40] = "127.0.0.1";
+  struct   timeval systemTime;
+  int64_t  startTime, endTime;
+  pthread_attr_t thattr;
+
+  // server info
+  epSet.numOfEps = 1;
+  epSet.inUse = 0;
+  epSet.port[0] = 7000;
+  epSet.port[1] = 7000;
+  strcpy(epSet.fqdn[0], serverIp);
+  strcpy(epSet.fqdn[1], "192.168.0.1");
+
+  // client info
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort    = 0;
+  rpcInit.label        = "APP";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp          = processResponse;
+  rpcInit.sessions     = 100;
+  rpcInit.idleTime     = tsShellActivityTimer*1000;
+  rpcInit.user         = "michael";
+  rpcInit.secret       = secret;
+  rpcInit.ckey         = "key";
+  rpcInit.spi          = 1;
+  rpcInit.connType     = TAOS_CONN_CLIENT;
+
+  for (int i=1; i<argc; ++i) { 
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+      epSet.port[0] = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-i") ==0 && i < argc-1) {
+      strcpy(epSet.fqdn[0], argv[++i]); 
+    } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
+      rpcInit.numOfThreads = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
+      msgSize = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-s")==0 && i < argc-1) {
+      rpcInit.sessions = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-n")==0 && i < argc-1) {
+      numOfReqs = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-a")==0 && i < argc-1) {
+      appThreads = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-o")==0 && i < argc-1) {
+      tsCompressMsgSize = atoi(argv[++i]); 
+    } else if (strcmp(argv[i], "-u")==0 && i < argc-1) {
+      rpcInit.user = argv[++i];
+    } else if (strcmp(argv[i], "-k")==0 && i < argc-1) {
+      rpcInit.secret = argv[++i];
+    } else if (strcmp(argv[i], "-spi")==0 && i < argc-1) {
+      rpcInit.spi = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
+      rpcDebugFlag = atoi(argv[++i]);
+    } else {
+      printf("\nusage: %s [options] \n", argv[0]);
+      printf("  [-i ip]: first server IP address, default is:%s\n", serverIp);
+      printf("  [-p port]: server port number, default is:%d\n", epSet.port[0]);
+      printf("  [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
+      printf("  [-s sessions]: number of rpc sessions, default is:%d\n", rpcInit.sessions);
+      printf("  [-m msgSize]: message body size, default is:%d\n", msgSize);
+      printf("  [-a threads]: number of app threads, default is:%d\n", appThreads);
+      printf("  [-n requests]: number of requests per thread, default is:%d\n", numOfReqs);
+      printf("  [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize);
+      printf("  [-u user]: user name for the connection, default is:%s\n", rpcInit.user);
+      printf("  [-k secret]: password for the connection, default is:%s\n", rpcInit.secret);
+      printf("  [-spi SPI]: security parameter index, default is:%d\n", rpcInit.spi);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag);
+      printf("  [-h help]: print out this help\n\n");
+      exit(0);
+    }
+  }
+
+  taosInitLog("client.log", 100000, 10);
+
+  void *pRpc = rpcOpen(&rpcInit);
+  if (pRpc == NULL) {
+    uError("failed to initialize RPC");
+    return -1;
+  }
+
+  uInfo("client is initialized");
+
+  gettimeofday(&systemTime, NULL);
+  startTime = systemTime.tv_sec*1000000 + systemTime.tv_usec;
+
+  SInfo *pInfo = (SInfo *)calloc(1, sizeof(SInfo)*appThreads);
+ 
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+
+  for (int i=0; i<appThreads; ++i) {
+    pInfo->index = i;
+    pInfo->epSet = epSet;
+    pInfo->numOfReqs = numOfReqs;
+    pInfo->msgSize = msgSize;
+    sem_init(&pInfo->rspSem, 0, 0);
+    pInfo->pRpc = pRpc;
+    pthread_create(&pInfo->thread, &thattr, sendRequest, pInfo);
+    pInfo++;
+  }
+
+  do {
+    usleep(1);
+  } while ( tcount < appThreads);
+
+  gettimeofday(&systemTime, NULL);
+  endTime = systemTime.tv_sec*1000000 + systemTime.tv_usec;  
+  float usedTime = (endTime - startTime)/1000.0;  // mseconds
+
+  uInfo("it takes %.3f mseconds to send %d requests to server", usedTime, numOfReqs*appThreads);
+  uInfo("Performance: %.3f requests per second, msgSize:%d bytes", 1000.0*numOfReqs*appThreads/usedTime, msgSize);
+
+  taosCloseLog();
+
+  return 0;
+}
+
+

src/sync/test/syncServer.c

+/*
+ * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ */
+
+//#define _DEFAULT_SOURCE
+#include <stdint.h>
+#include "os.h"
+#include "tulog.h"
+#include "tglobal.h"
+#include "tsocket.h"
+#include "trpc.h"
+#include "tqueue.h"
+#include "twal.h"
+#include "tsync.h"
+
+int msgSize = 128;
+int commit = 0;
+int dataFd = -1;
+void *qhandle = NULL;
+int walNum = 0;
+uint64_t  tversion = 0;
+void *syncHandle;
+int   role;
+int  nodeId;
+char path[256];
+int  numOfWrites ;
+SSyncInfo syncInfo;
+SSyncCfg *pCfg;
+
+int writeIntoWal(SWalHead *pHead)
+{ 
+  if (dataFd < 0) {
+    char  walName[280];
+    snprintf(walName, sizeof(walName), "%s/wal/wal.%d", path, walNum);
+    remove(walName);
+    dataFd = open(walName, O_CREAT | O_WRONLY, S_IRWXU | S_IRWXG | S_IRWXO);  
+    if (dataFd < 0) { 
+      uInfo("failed to open wal file:%s(%s)", walName, strerror(errno));
+      return -1;
+    } else {
+      walNum++;
+      uInfo("file:%s is opened to write, walNum:%d", walName, walNum);
+    }
+  }
+
+  if (write(dataFd, pHead, sizeof(SWalHead) + pHead->len) < 0) {
+    uError("ver:%" PRIu64 ", failed to write wal file(%s)", pHead->version, strerror(errno));
+  } else {
+    uDebug("ver:%" PRIu64 ", written to wal", pHead->version);
+  }
+
+  numOfWrites++;
+  if (numOfWrites >= 10000) {
+    uInfo("%d request have been written into disk", numOfWrites);
+    close(dataFd);
+    dataFd = -1;
+    numOfWrites = 0;
+  }
+ 
+  return 0;
+}
+
+void confirmForward(void *ahandle, void *mhandle, int32_t code)
+{
+  SRpcMsg  *pMsg = (SRpcMsg *)mhandle;
+  SWalHead *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
+
+  uDebug("ver:%" PRIu64 ", confirm is received", pHead->version);
+
+  rpcFreeCont(pMsg->pCont);
+
+  SRpcMsg    rpcMsg;
+  rpcMsg.pCont = rpcMallocCont(msgSize);
+  rpcMsg.contLen = msgSize;
+  rpcMsg.handle = pMsg->handle;
+  rpcMsg.code = code;
+  rpcSendResponse(&rpcMsg);
+
+  taosFreeQitem(mhandle); 
+}
+
+int processRpcMsg(void *item) {
+  SRpcMsg   *pMsg = (SRpcMsg *)item;
+  SWalHead  *pHead = (SWalHead *)(((char *)pMsg->pCont) - sizeof(SWalHead));
+  int        code = -1;
+
+  if (role != TAOS_SYNC_ROLE_MASTER) {
+    uError("not master, write failed, role:%s", syncRole[role]);
+  } else {
+
+    pHead->version = ++tversion;
+    pHead->msgType = pMsg->msgType;
+    pHead->len = pMsg->contLen;
+
+    uDebug("ver:%" PRIu64 ", pkt from client processed", pHead->version);
+    writeIntoWal(pHead); 
+    syncForwardToPeer(syncHandle, pHead, item, TAOS_QTYPE_RPC);
+
+    code = 0;
+  }
+
+  if (pCfg->quorum <= 1) { 
+    taosFreeQitem(item); 
+    rpcFreeCont(pMsg->pCont);
+
+    SRpcMsg    rpcMsg;
+    rpcMsg.pCont = rpcMallocCont(msgSize);
+    rpcMsg.contLen = msgSize;
+    rpcMsg.handle = pMsg->handle;
+    rpcMsg.code = code;
+    rpcSendResponse(&rpcMsg);
+  }
+
+  return code;
+}
+
+int processFwdMsg(void *item) {
+
+  SWalHead *pHead = (SWalHead *)item;
+
+  if (pHead->version <= tversion) {
+    uError("ver:%" PRIu64 ", forward is even lower than local:%" PRIu64, pHead->version, tversion);
+    return -1;
+  }
+
+  uDebug("ver:%" PRIu64 ", forward from peer is received", pHead->version);
+  writeIntoWal(pHead);
+  tversion = pHead->version;
+
+  if (pCfg->quorum > 1) syncConfirmForward(syncHandle, pHead->version, 0);
+
+  // write into cache
+
+/*
+  if (pHead->handle) {
+    syncSendFwdAck(syncHandle, pHead->handle, 0);  
+  }
+*/
+
+  taosFreeQitem(item);
+
+  return 0;
+}
+
+int processWalMsg(void *item) {
+
+  SWalHead *pHead = (SWalHead *)item;
+
+  if (pHead->version <= tversion) {
+    uError("ver:%" PRIu64 ", wal is even lower than local:%" PRIu64, pHead->version, tversion);
+    return -1;
+  };
+
+  uDebug("ver:%" PRIu64 ", wal from peer is received", pHead->version);
+  writeIntoWal(pHead);
+  tversion = pHead->version;
+
+  // write into cache
+
+/*
+  if (pHead->handle) {
+    syncSendFwdAck(syncHandle, pHead->handle, 0);  
+  }
+*/
+
+  taosFreeQitem(item);
+
+  return 0;
+}
+
+void *processWriteQueue(void *param) {
+  int        type;
+  void      *item;
+
+  while (1) {
+    int ret = taosReadQitem(qhandle, &type, &item);
+    if (ret <= 0) {
+      usleep(1000);
+      continue;
+    }     
+
+    if (type == TAOS_QTYPE_RPC) {
+      processRpcMsg(item);
+    } else if (type == TAOS_QTYPE_WAL) {
+      processWalMsg(item);
+    } else if (type == TAOS_QTYPE_FWD) {
+      processFwdMsg(item);
+    } 
+
+  }
+
+  return NULL;
+}
+
+int retrieveAuthInfo(char *meterId, char *spi, char *encrypt, char *secret, char *ckey) {
+  // app shall retrieve the auth info based on meterID from DB or a data file
+  // demo code here only for simple demo
+  int ret = 0;
+
+  if (strcmp(meterId, "michael") == 0) {
+    *spi = 1;
+    *encrypt = 0;
+    strcpy(secret, "mypassword");
+    strcpy(ckey, "key");
+  } else if (strcmp(meterId, "jeff") == 0) {
+    *spi = 0;
+    *encrypt = 0;
+  } else {
+    ret = -1;  // user not there
+  }
+
+  return ret;
+}
+
+void processRequestMsg(SRpcMsg *pMsg, SRpcEpSet *pEpSet) {
+  
+  SRpcMsg *pTemp;
+
+  pTemp = taosAllocateQitem(sizeof(SRpcMsg));
+  memcpy(pTemp, pMsg, sizeof(SRpcMsg));
+  
+  uDebug("request is received, type:%d, len:%d", pMsg->msgType, pMsg->contLen);
+  taosWriteQitem(qhandle, TAOS_QTYPE_RPC, pTemp); 
+}
+
+uint32_t getFileInfo(void *ahandle, char *name, uint32_t *index, uint32_t eindex, int32_t *size, uint64_t *fversion) 
+{
+  uint32_t     magic;
+  struct stat  fstat;
+  char         aname[280];
+
+  if (*index == 2) {
+    uInfo("wait for a while .....");
+    sleep(3);
+  }
+
+  if (name[0] == 0) {
+    // find the file 
+    snprintf(aname, sizeof(aname), "%s/data/data.%d", path, *index);
+    sprintf(name, "data/data.%d", *index); 
+  } else {
+    snprintf(aname, sizeof(aname), "%s/%s", path, name);
+  }
+
+  uInfo("get file info:%s", aname);
+  if ( stat(aname, &fstat) < 0 ) return 0; 
+
+  *size = fstat.st_size;
+  magic = fstat.st_size;
+
+  return magic;
+}
+
+int  getWalInfo(void *ahandle, char *name, uint32_t *index) {
+
+  struct stat  fstat;
+  char         aname[280];
+
+  name[0] = 0;
+  if (*index + 1> walNum) return 0;
+
+  snprintf(aname, sizeof(aname), "%s/wal/wal.%d", path, *index);
+  sprintf(name, "wal/wal.%d", *index); 
+  uInfo("get wal info:%s", aname);
+
+  if ( stat(aname, &fstat) < 0 ) return -1;
+
+  if (*index >= walNum-1) return 0;  // no more
+
+  return 1;
+
+}
+
+int writeToCache(void *ahandle, void *data, int type) {
+  SWalHead *pHead = data;
+
+  uDebug("pkt from peer is received, ver:%" PRIu64 " len:%d type:%d", pHead->version, pHead->len, type);
+
+  int   msgSize = pHead->len + sizeof(SWalHead);
+  void *pMsg = taosAllocateQitem(msgSize);
+  memcpy(pMsg, pHead, msgSize);
+  taosWriteQitem(qhandle, type, pMsg); 
+
+  return 0;
+}
+
+void confirmFwd(void *ahandle, int64_t version) {
+
+  return;
+}
+
+void notifyRole(void *ahandle, int8_t r) {
+  role = r;
+  printf("current role:%s\n", syncRole[role]);
+}
+
+
+void initSync() {
+
+  pCfg->replica = 1;
+  pCfg->quorum = 1;
+  syncInfo.vgId = 1;
+  syncInfo.ahandle = &syncInfo;
+  syncInfo.getFileInfo = getFileInfo;
+  syncInfo.getWalInfo = getWalInfo;
+  syncInfo.writeToCache = writeToCache;
+  syncInfo.confirmForward = confirmForward;
+  syncInfo.notifyRole = notifyRole;
+
+  pCfg->nodeInfo[0].nodeId = 1;
+  pCfg->nodeInfo[0].nodePort = 7010;
+  taosGetFqdn(pCfg->nodeInfo[0].nodeFqdn);
+
+  pCfg->nodeInfo[1].nodeId = 2;
+  pCfg->nodeInfo[1].nodePort = 7110;
+  taosGetFqdn(pCfg->nodeInfo[1].nodeFqdn);
+
+  pCfg->nodeInfo[2].nodeId = 3;
+  pCfg->nodeInfo[2].nodePort = 7210;
+  taosGetFqdn(pCfg->nodeInfo[2].nodeFqdn);
+
+  pCfg->nodeInfo[3].nodeId = 4;
+  pCfg->nodeInfo[3].nodePort = 7310;
+  taosGetFqdn(pCfg->nodeInfo[3].nodeFqdn);
+
+  pCfg->nodeInfo[4].nodeId = 5;
+  pCfg->nodeInfo[4].nodePort = 7410;
+  taosGetFqdn(pCfg->nodeInfo[4].nodeFqdn);
+}
+
+void doSync()
+{
+  for (int i=0; i<5; ++i) {
+    if (tsSyncPort == pCfg->nodeInfo[i].nodePort) 
+      nodeId = pCfg->nodeInfo[i].nodeId;
+  }
+
+  snprintf(path, sizeof(path), "/root/test/d%d", nodeId);
+  strcpy(syncInfo.path, path);
+
+  if ( syncHandle == NULL) {
+      syncHandle = syncStart(&syncInfo);
+  } else {
+      if (syncReconfig(syncHandle, pCfg) < 0) syncHandle = NULL;
+  }
+
+  uInfo("nodeId:%d path:%s syncPort:%d", nodeId, path, tsSyncPort);
+}
+
+int main(int argc, char *argv[]) {
+  SRpcInit rpcInit;
+  char     dataName[20] = "server.data";
+  pCfg =  &syncInfo.syncCfg;
+
+  initSync();
+
+  memset(&rpcInit, 0, sizeof(rpcInit));
+  rpcInit.localPort    = 7000;
+  rpcInit.label        = "SER";
+  rpcInit.numOfThreads = 1;
+  rpcInit.cfp          = processRequestMsg;
+  rpcInit.sessions     = 1000;
+  rpcInit.idleTime     = tsShellActivityTimer*1500; 
+  rpcInit.afp          = retrieveAuthInfo;
+
+  for (int i=1; i<argc; ++i) {
+    if (strcmp(argv[i], "-p")==0 && i < argc-1) {
+      rpcInit.localPort = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-t")==0 && i < argc-1) {
+      rpcInit.numOfThreads = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-m")==0 && i < argc-1) {
+      msgSize = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-s")==0 && i < argc-1) {
+      rpcInit.sessions = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-o")==0 && i < argc-1) {
+      tsCompressMsgSize = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-w")==0 && i < argc-1) {
+      commit = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-v")==0 && i < argc-1) {
+      syncInfo.version = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-r")==0 && i < argc-1) {
+      pCfg->replica = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-q")==0 && i < argc-1) {
+      pCfg->quorum = atoi(argv[++i]);
+    } else if (strcmp(argv[i], "-d")==0 && i < argc-1) {
+      rpcDebugFlag = atoi(argv[++i]);
+    } else {
+      printf("\nusage: %s [options] \n", argv[0]);
+      printf("  [-p port]: server port number, default is:%d\n", rpcInit.localPort);
+      printf("  [-t threads]: number of rpc threads, default is:%d\n", rpcInit.numOfThreads);
+      printf("  [-s sessions]: number of sessions, default is:%d\n", rpcInit.sessions);
+      printf("  [-m msgSize]: message body size, default is:%d\n", msgSize);
+      printf("  [-o compSize]: compression message size, default is:%d\n", tsCompressMsgSize);
+      printf("  [-w write]: write received data to file(0, 1, 2), default is:%d\n", commit);
+      printf("  [-v version]: initial node version, default is:%ld\n", syncInfo.version);
+      printf("  [-r replica]: replicacation number, default is:%d\n", pCfg->replica);
+      printf("  [-q quorum]: quorum, default is:%d\n", pCfg->quorum);
+      printf("  [-d debugFlag]: debug flag, default:%d\n", rpcDebugFlag);
+      printf("  [-h help]: print out this help\n\n");
+      exit(0);
+    }
+  }
+ 
+  uDebugFlag = rpcDebugFlag;
+  dDebugFlag = rpcDebugFlag; 
+  //tmrDebugFlag = rpcDebugFlag; 
+  tsAsyncLog = 0;
+  taosInitLog("server.log", 1000000, 10);
+
+  rpcInit.connType = TAOS_CONN_SERVER;
+  void *pRpc = rpcOpen(&rpcInit);
+  if (pRpc == NULL) {
+    uError("failed to start RPC server");
+    return -1;
+  }
+
+  tsSyncPort = rpcInit.localPort + 10;
+  qhandle = taosOpenQueue();
+
+  doSync();
+
+  pthread_attr_t thattr;
+  pthread_t      thread;
+  pthread_attr_init(&thattr);
+  pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
+  if (pthread_create(&thread, &thattr, processWriteQueue, NULL) != 0) {
+    uError("failed to create thread, reason:%s", strerror(errno));
+    return -1;
+  }
+
+  printf("server is running, localPort:%d\n", rpcInit.localPort);
+  SNodesRole nroles;
+
+  while (1) {
+    char c = getchar();
+
+    switch(c) {
+      case '1':
+        pCfg->replica = 1; doSync();
+        break;        
+      case '2':
+        pCfg->replica = 2; doSync();
+        break;
+      case '3':
+        pCfg->replica = 3; doSync();
+        break;
+      case '4':
+        pCfg->replica = 4; doSync();
+        break;
+      case '5':
+        pCfg->replica = 5; doSync();
+        break;
+      case 's':
+        syncGetNodesRole(syncHandle, &nroles);
+        for (int i=0; i<pCfg->replica; ++i) 
+          printf("=== nodeId:%d role:%s\n", nroles.nodeId[i], syncRole[nroles.role[i]]);
+        break;
+      default:
+        break;
+    }
+
+    if (c=='q') break;
+
+  }
+
+  syncStop(syncHandle);
+
+  if (dataFd >= 0) {
+    close(dataFd);
+    remove(dataName);
+  }
+
+  return 0;
+}
+
+

src/tsdb/CMakeLists.txt

@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   # INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/vnode/common/inc)

src/tsdb/src/tsdbFile.c

@@ -12,23 +12,14 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include <dirent.h>
-#include <fcntl.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
-#include <libgen.h>
+#define _DEFAULT_SOURCE
+#include "os.h"
 
 #include "talgo.h"
 #include "tchecksum.h"
 #include "tsdbMain.h"
 #include "tutil.h"
 #include "ttime.h"
-#include "tfile.h"
 
 #ifdef TSDB_IDX
 const char *tsdbFileSuffix[] = {".idx", ".head", ".data", ".last", "", ".i", ".h", ".l"};
@@ -317,7 +308,7 @@ int tsdbUpdateFileHeader(SFile *pFile, uint32_t version) {
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
-  if (twrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTWrite(pFile->fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("failed to write %d bytes to file %s since %s", TSDB_FILE_HEAD_SIZE, pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -383,7 +374,7 @@ static int tsdbInitFile(SFile *pFile, STsdbRepo *pRepo, int fid, int type) {
   pFile->fd = -1;
   if (tsdbOpenFile(pFile, O_RDONLY) < 0) goto _err;
 
-  if (tread(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTRead(pFile->fd, buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pRepo), TSDB_FILE_HEAD_SIZE,
               pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);

src/tsdb/src/tsdbMain.c

@@ -545,7 +545,7 @@ static int32_t tsdbSaveConfig(char *rootDir, STsdbCfg *pCfg) {
 
   taosCalcChecksumAppend(0, (uint8_t *)buf, TSDB_FILE_HEAD_SIZE);
 
-  if (twrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTWrite(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("vgId:%d failed to write %d bytes to file %s since %s", pCfg->tsdbId, TSDB_FILE_HEAD_SIZE, fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -586,7 +586,7 @@ static int tsdbLoadConfig(char *rootDir, STsdbCfg *pCfg) {
     goto _err;
   }
 
-  if (tread(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
+  if (taosTRead(fd, (void *)buf, TSDB_FILE_HEAD_SIZE) < TSDB_FILE_HEAD_SIZE) {
     tsdbError("failed to read %d bytes from file %s since %s", TSDB_FILE_HEAD_SIZE, fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;

src/tsdb/src/tsdbRWHelper.c

@@ -13,13 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "talgo.h"
 #include "tchecksum.h"
 #include "tcoding.h"
 #include "tscompression.h"
 #include "tsdbMain.h"
-#include "tfile.h"
 
 #define TSDB_GET_COMPCOL_LEN(nCols) (sizeof(SCompData) + sizeof(SCompCol) * (nCols) + sizeof(TSCKSUM))
 #define TSDB_KEY_COL_OFFSET 0
@@ -379,7 +379,7 @@ int tsdbMoveLastBlockIfNeccessary(SRWHelper *pHelper) {
         return -1;
       }
 
-      if (tsendfile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
+      if (taosTSendFile(helperNewLastF(pHelper)->fd, helperLastF(pHelper)->fd, NULL, pCompBlock->len) < pCompBlock->len) {
         tsdbError("vgId:%d failed to sendfile from file %s to file %s since %s", REPO_ID(pHelper->pRepo),
                   helperLastF(pHelper)->fname, helperNewLastF(pHelper)->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
@@ -424,7 +424,7 @@ int tsdbWriteCompInfo(SRWHelper *pHelper) {
     pIdx->tid = pHelper->tableInfo.tid;
     ASSERT(pIdx->offset >= TSDB_FILE_HEAD_SIZE);
 
-    if (twrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
+    if (taosTWrite(pFile->fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
       tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
                 pFile->fname, strerror(errno));
       terrno = TAOS_SYSTEM_ERROR(errno);
@@ -481,7 +481,7 @@ int tsdbWriteCompIdx(SRWHelper *pHelper) {
 
   pFile->info.offset = offset;
 
-  if (twrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) {
+  if (taosTWrite(pFile->fd, (void *)pHelper->pWIdx, pFile->info.len) < pFile->info.len) {
     tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
               pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -514,7 +514,7 @@ int tsdbLoadCompIdx(SRWHelper *pHelper, void *target) {
         return -1;
       }
 
-      if (tread(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) {
+      if (taosTRead(fd, (void *)(pHelper->pBuffer), pFile->info.len) < pFile->info.len) {
         tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pFile->info.len,
                   pFile->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
@@ -585,7 +585,7 @@ int tsdbLoadCompInfo(SRWHelper *pHelper, void *target) {
       }
 
       pHelper->pCompInfo = trealloc((void *)pHelper->pCompInfo, pIdx->len);
-      if (tread(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
+      if (taosTRead(fd, (void *)(pHelper->pCompInfo), pIdx->len) < pIdx->len) {
         tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pIdx->len,
                   helperHeadF(pHelper)->fname, strerror(errno));
         terrno = TAOS_SYSTEM_ERROR(errno);
@@ -626,7 +626,7 @@ int tsdbLoadCompData(SRWHelper *pHelper, SCompBlock *pCompBlock, void *target) {
     return -1;
   }
 
-  if (tread(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
+  if (taosTRead(pFile->fd, (void *)pHelper->pCompData, tsize) < tsize) {
     tsdbError("vgId:%d failed to read %zu bytes from file %s since %s", REPO_ID(pHelper->pRepo), tsize, pFile->fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -841,7 +841,7 @@ static int tsdbWriteBlockToFile(SRWHelper *pHelper, SFile *pFile, SDataCols *pDa
                                        sizeof(TSCKSUM));
 
   // Write the whole block to file
-  if (twrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
+  if (taosTWrite(pFile->fd, (void *)pCompData, lsize) < lsize) {
     tsdbError("vgId:%d failed to write %d bytes to file %s since %s", REPO_ID(helperRepo(pHelper)), lsize, pFile->fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1222,7 +1222,7 @@ static int tsdbLoadColData(SRWHelper *pHelper, SFile *pFile, SCompBlock *pCompBl
     return -1;
   }
 
-  if (tread(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
+  if (taosTRead(pFile->fd, pHelper->pBuffer, pCompCol->len) < pCompCol->len) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompCol->len, pFile->fname,
               strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
@@ -1337,7 +1337,7 @@ static int tsdbLoadBlockDataImpl(SRWHelper *pHelper, SCompBlock *pCompBlock, SDa
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
   }
-  if (tread(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
+  if (taosTRead(fd, (void *)pCompData, pCompBlock->len) < pCompBlock->len) {
     tsdbError("vgId:%d failed to read %d bytes from file %s since %s", REPO_ID(pHelper->pRepo), pCompBlock->len,
               pFile->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);

src/util/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 
@@ -11,7 +11,7 @@ IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/lz4/inc)
   AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(tutil ${SRC})
-  TARGET_LINK_LIBRARIES(tutil pthread os m rt lz4)
+  TARGET_LINK_LIBRARIES(tutil pthread osdetail m rt lz4)
   FIND_PATH(ICONV_INCLUDE_EXIST iconv.h /usr/include/ /usr/local/include/)
   IF (ICONV_INCLUDE_EXIST)
     ADD_DEFINITIONS(-DUSE_LIBICONV)
@@ -68,44 +68,12 @@ ELSEIF (TD_WINDOWS_64)
   LIST(APPEND SRC ./src/tutil.c)
   LIST(APPEND SRC ./src/version.c)
   ADD_LIBRARY(tutil ${SRC}) 
-  TARGET_LINK_LIBRARIES(tutil iconv regex pthread os winmm IPHLPAPI ws2_32 lz4)
+  TARGET_LINK_LIBRARIES(tutil iconv regex pthread osdetail winmm IPHLPAPI ws2_32 lz4)
 ELSEIF(TD_DARWIN_64)
   ADD_DEFINITIONS(-DUSE_LIBICONV)
-  LIST(APPEND SRC ./src/hash.c)
-  LIST(APPEND SRC ./src/ihash.c)
-  LIST(APPEND SRC ./src/lz4.c)
-  LIST(APPEND SRC ./src/shash.c)
-  LIST(APPEND SRC ./src/tbase64.c)
-  LIST(APPEND SRC ./src/tcache.c)
-  LIST(APPEND SRC ./src/tcompression.c)
-  LIST(APPEND SRC ./src/textbuffer.c)
-  LIST(APPEND SRC ./src/tglobalcfg.c)
-  LIST(APPEND SRC ./src/thash.c)
-  LIST(APPEND SRC ./src/thashutil.c)
-  LIST(APPEND SRC ./src/thistogram.c)
-  LIST(APPEND SRC ./src/tidpool.c)
-  LIST(APPEND SRC ./src/tinterpolation.c)
-  LIST(APPEND SRC ./src/tlog.c)
-  LIST(APPEND SRC ./src/tlosertree.c)
-  LIST(APPEND SRC ./src/tmd5.c)
-  LIST(APPEND SRC ./src/tmem.c)
-  LIST(APPEND SRC ./src/tmempool.c)
-  LIST(APPEND SRC ./src/tmodule.c)
-  LIST(APPEND SRC ./src/tnote.c)
-  LIST(APPEND SRC ./src/tpercentile.c)
-  LIST(APPEND SRC ./src/tsched.c)
-  LIST(APPEND SRC ./src/tskiplist.c)
-  LIST(APPEND SRC ./src/tsocket.c)
-  LIST(APPEND SRC ./src/tstrbuild.c)
-  LIST(APPEND SRC ./src/ttime.c)
-  LIST(APPEND SRC ./src/ttimer.c)
-  LIST(APPEND SRC ./src/ttokenizer.c)
-  LIST(APPEND SRC ./src/ttypes.c)
-  LIST(APPEND SRC ./src/tutil.c)
-  LIST(APPEND SRC ./src/version.c)
-  LIST(APPEND SRC ./src/hash.c)
+  AUX_SOURCE_DIRECTORY(src SRC)
   ADD_LIBRARY(tutil ${SRC})
-  TARGET_LINK_LIBRARIES(tutil iconv pthread os lz4)
+  TARGET_LINK_LIBRARIES(tutil iconv pthread osdetail lz4)
 ENDIF()
 
 #  TARGET_LINK_LIBRARIES(tutil mstorage)

src/util/inc/talgo.h

@@ -20,6 +20,8 @@
 extern "C" {
 #endif
 
+#include "os.h"
+
 #define TD_EQ 0x1
 #define TD_GT 0x2
 #define TD_LT 0x4

src/util/inc/tfile.h

-/*
- * Copyright (c) 2019 TAOS Data, Inc. <jhtao@taosdata.com>
- *
- * This program is free software: you can use, redistribute, and/or modify
- * it under the terms of the GNU Affero General Public License, version 3
- * or later ("AGPL"), as published by the Free Software Foundation.
- *
- * This program is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- */
-
-#ifndef TDENGINE_TFILE_H
-#define TDENGINE_TFILE_H
-
-#ifdef TAOS_RANDOM_FILE_FAIL
-
-void taosSetRandomFileFailFactor(int factor);
-void taosSetRandomFileFailOutput(const char *path);
-ssize_t taos_tread(int fd, void *buf, size_t count, const char *file, uint32_t line);
-ssize_t taos_twrite(int fd, void *buf, size_t count, const char *file, uint32_t line);
-off_t taos_lseek(int fd, off_t offset, int whence, const char *file, uint32_t line);
-
-#define tread(fd, buf, count)  taos_tread(fd, buf, count, __FILE__, __LINE__)
-#define twrite(fd, buf, count)  taos_twrite(fd, buf, count, __FILE__, __LINE__)
-#define lseek(fd, offset, whence)  taos_lseek(fd, offset, whence, __FILE__, __LINE__)
-
-#endif  // TAOS_RANDOM_FILE_FAIL
-
-#endif  // TDENGINE_TFILE_H

src/util/inc/tutil.h

@@ -143,8 +143,6 @@ char *strbetween(char *string, char *begin, char *end);
 
 char *paGetToken(char *src, char **token, int32_t *tokenLen);
 
-void taosMsleep(int32_t mseconds);
-
 int32_t taosByteArrayToHexStr(char bytes[], int32_t len, char hexstr[]);
 
 int32_t taosHexStrToByteArray(char hexstr[], char bytes[]);
@@ -225,6 +223,9 @@ ssize_t taos_getline(char **lineptr, size_t *n, FILE *stream, const char *file,
 
 #endif // TAOS_MEM_CHECK
 
+
+char *taosCharsetReplace(char *charsetstr);
+
 #ifdef __cplusplus
 }
 #endif

src/util/src/tbuffer.c

@@ -13,6 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
+#include "os.h"
 #include <stdlib.h>
 #include <memory.h>
 #include <assert.h>

src/util/src/tcache.c

@@ -259,7 +259,7 @@ SCacheObj *taosCacheInit(int32_t keyType, int64_t refreshTimeInSeconds, bool ext
     return NULL;
   }
 
-  pthread_attr_t thattr = {{0}};
+  pthread_attr_t thattr;
   pthread_attr_init(&thattr);
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_JOINABLE);
 
@@ -288,7 +288,7 @@ void *taosCachePut(SCacheObj *pCacheObj, const void *key, size_t keyLen, const v
       uDebug("cache:%s, key:%p, %p added into cache, added:%" PRIu64 ", expire:%" PRIu64 ", totalNum:%d totalSize:%" PRId64
              "bytes size:%" PRId64 "bytes",
              pCacheObj->name, key, pNode->data, pNode->addedTime, pNode->expireTime,
-             (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, dataSize);
+             (int32_t)taosHashGetSize(pCacheObj->pHashTable), pCacheObj->totalSize, (int64_t)dataSize);
     } else {
       uError("cache:%s, key:%p, failed to added into cache, out of memory", pCacheObj->name, key);
     }

src/util/src/tkvstore.c

@@ -12,22 +12,15 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
-#include <errno.h>
-#include <fcntl.h>
-#include <libgen.h>
-#include <string.h>
-#include <sys/stat.h>
-#include <sys/types.h>
-#include <unistd.h>
 
-#include "hash.h"
+#define _DEFAULT_SOURCE
 #include "os.h"
+#include "hash.h"
 #include "taoserror.h"
 #include "tchecksum.h"
 #include "tcoding.h"
 #include "tkvstore.h"
 #include "tulog.h"
-#include "tfile.h"
 
 #define TD_KVSTORE_HEADER_SIZE 512
 #define TD_KVSTORE_MAJOR_VERSION 1
@@ -182,7 +175,7 @@ int tdKVStoreStartCommit(SKVStore *pStore) {
     goto _err;
   }
 
-  if (tsendfile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+  if (taosTSendFile(pStore->sfd, pStore->fd, NULL, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
     uError("failed to send file %d bytes since %s", TD_KVSTORE_HEADER_SIZE, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     goto _err;
@@ -242,13 +235,13 @@ int tdUpdateKVStoreRecord(SKVStore *pStore, uint64_t uid, void *cont, int contLe
   ASSERT(tlen == POINTER_DISTANCE(pBuf, buf));
   ASSERT(tlen == sizeof(SKVRecord));
 
-  if (twrite(pStore->fd, buf, tlen) < tlen) {
+  if (taosTWrite(pStore->fd, buf, tlen) < tlen) {
     uError("failed to write %d bytes to file %s since %s", tlen, pStore->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
 
-  if (twrite(pStore->fd, cont, contLen) < contLen) {
+  if (taosTWrite(pStore->fd, cont, contLen) < contLen) {
     uError("failed to write %d bytes to file %s since %s", contLen, pStore->fname, strerror(errno));
     return -1;
   }
@@ -286,8 +279,8 @@ int tdDropKVStoreRecord(SKVStore *pStore, uint64_t uid) {
   void *pBuf = buf;
   tdEncodeKVRecord(&pBuf, &rInfo);
 
-  if (twrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
-    uError("failed to write %" PRId64 " bytes to file %s since %s", POINTER_DISTANCE(pBuf, buf), pStore->fname, strerror(errno));
+  if (taosTWrite(pStore->fd, buf, POINTER_DISTANCE(pBuf, buf)) < POINTER_DISTANCE(pBuf, buf)) {
+    uError("failed to write %" PRId64 " bytes to file %s since %s", (int64_t)(POINTER_DISTANCE(pBuf, buf)), pStore->fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
   }
@@ -335,7 +328,7 @@ static int tdLoadKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
     return -1;
   }
 
-  if (tread(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+  if (taosTRead(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
     uError("failed to read %d bytes from file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -366,7 +359,7 @@ static int tdUpdateKVStoreHeader(int fd, char *fname, SStoreInfo *pInfo) {
   ASSERT(POINTER_DISTANCE(pBuf, buf) + sizeof(TSCKSUM) <= TD_KVSTORE_HEADER_SIZE);
 
   taosCalcChecksumAppend(0, (uint8_t *)buf, TD_KVSTORE_HEADER_SIZE);
-  if (twrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
+  if (taosTWrite(fd, buf, TD_KVSTORE_HEADER_SIZE) < TD_KVSTORE_HEADER_SIZE) {
     uError("failed to write %d bytes to file %s since %s", TD_KVSTORE_HEADER_SIZE, fname, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
     return -1;
@@ -499,7 +492,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
   ASSERT(pStore->info.size == TD_KVSTORE_HEADER_SIZE);
 
   while (true) {
-    ssize_t tsize = tread(pStore->fd, tbuf, sizeof(SKVRecord));
+    ssize_t tsize = taosTRead(pStore->fd, tbuf, sizeof(SKVRecord));
     if (tsize == 0) break;
     if (tsize < sizeof(SKVRecord)) {
       uError("failed to read %zu bytes from file %s at offset %" PRId64 "since %s", sizeof(SKVRecord), pStore->fname,
@@ -562,7 +555,7 @@ static int tdRestoreKVStore(SKVStore *pStore) {
       goto _err;
     }
 
-    if (tread(pStore->fd, buf, pRecord->size) < pRecord->size) {
+    if (taosTRead(pStore->fd, buf, pRecord->size) < pRecord->size) {
       uError("failed to read %" PRId64 " bytes from file %s since %s, offset %" PRId64, pRecord->size, pStore->fname,
              strerror(errno), pRecord->offset);
       terrno = TAOS_SYSTEM_ERROR(errno);

src/util/src/tlog.c

@@ -301,11 +301,11 @@ static int32_t taosOpenLogFile(char *fn, int32_t maxLines, int32_t maxFileNum) {
   lseek(tsLogObj.logHandle->fd, 0, SEEK_END);
 
   sprintf(name, "==================================================\n");
-  twrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
   sprintf(name, "                new log file                      \n");
-  twrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
   sprintf(name, "==================================================\n");
-  twrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
+  taosTWrite(tsLogObj.logHandle->fd, name, (uint32_t)strlen(name));
 
   return 0;
 }
@@ -355,7 +355,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
     if (tsAsyncLog) {
       taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
     } else {
-      twrite(tsLogObj.logHandle->fd, buffer, len);
+      taosTWrite(tsLogObj.logHandle->fd, buffer, len);
     }
 
     if (tsLogObj.maxLines > 0) {
@@ -365,7 +365,7 @@ void taosPrintLog(const char *flags, int32_t dflag, const char *format, ...) {
     }
   }
 
-  if (dflag & DEBUG_SCREEN) twrite(1, buffer, (uint32_t)len);
+  if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len);
 }
 
 void taosDumpData(unsigned char *msg, int32_t len) {
@@ -384,7 +384,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
     pos += 3;
     if (c >= 16) {
       temp[pos++] = '\n';
-      twrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
+      taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
       c = 0;
       pos = 0;
     }
@@ -392,7 +392,7 @@ void taosDumpData(unsigned char *msg, int32_t len) {
 
   temp[pos++] = '\n';
 
-  twrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
+  taosTWrite(tsLogObj.logHandle->fd, temp, (uint32_t)pos);
 
   return;
 }
@@ -432,7 +432,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
     if (tsAsyncLog) {
       taosPushLogBuffer(tsLogObj.logHandle, buffer, len);
     } else {
-      twrite(tsLogObj.logHandle->fd, buffer, len);
+      taosTWrite(tsLogObj.logHandle->fd, buffer, len);
     }
     
     if (tsLogObj.maxLines > 0) {
@@ -442,7 +442,7 @@ void taosPrintLongString(const char *flags, int32_t dflag, const char *format, .
     }
   }
 
-  if (dflag & DEBUG_SCREEN) twrite(1, buffer, (uint32_t)len);
+  if (dflag & DEBUG_SCREEN) taosTWrite(1, buffer, (uint32_t)len);
 }
 
 #if 0
@@ -571,7 +571,7 @@ static void *taosAsyncOutputLog(void *param) {
     while (1) {
       log_size = taosPollLogBuffer(tLogBuff, tempBuffer, TSDB_DEFAULT_LOG_BUF_UNIT);
       if (log_size) {
-        twrite(tLogBuff->fd, tempBuffer, log_size);
+        taosTWrite(tLogBuff->fd, tempBuffer, log_size);
         LOG_BUF_START(tLogBuff) = (LOG_BUF_START(tLogBuff) + log_size) % LOG_BUF_SIZE(tLogBuff);
       } else {
         break;

src/util/src/tnote.c

@@ -264,7 +264,7 @@ void taosNotePrint(taosNoteInfo * pNote, const char * const format, ...)
     buffer[len] = 0;
 
     if (pNote->taosNoteFd >= 0)  {
-        twrite(pNote->taosNoteFd, buffer, (unsigned int)len);
+        taosTWrite(pNote->taosNoteFd, buffer, (unsigned int)len);
 
         if (pNote->taosNoteMaxLines > 0) {
             pNote->taosNoteLines++;

src/util/src/tsocket.c

@@ -164,7 +164,7 @@ int taosNonblockwrite(int fd, char *ptr, int nbytes) {
       return -1;
     }
 
-    nwritten = (int)send(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
+    nwritten = (int)taosSend(fd, ptr, (size_t)nleft, MSG_NOSIGNAL);
     if (nwritten <= 0) {
       if (errno == EAGAIN || errno == EINTR) continue;
 

src/vnode/CMakeLists.txt

@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/query/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/tsdb/inc)

src/vnode/src/vnodeMain.c

@@ -46,15 +46,6 @@ static int      vnodeGetWalInfo(void *ahandle, char *name, uint32_t *index);
 static void     vnodeNotifyRole(void *ahandle, int8_t role);
 static int      vnodeNotifyFileSynced(void *ahandle, uint64_t fversion);
 
-#ifndef _SYNC
-tsync_h syncStart(const SSyncInfo *info) { return NULL; }
-int32_t syncForwardToPeer(tsync_h shandle, void *pHead, void *mhandle, int qtype) { return 0; }
-void    syncStop(tsync_h shandle) {}
-int32_t syncReconfig(tsync_h shandle, const SSyncCfg * cfg) { return 0; }
-int     syncGetNodesRole(tsync_h shandle, SNodesRole * cfg) { return 0; }
-void    syncConfirmForward(tsync_h shandle, uint64_t version, int32_t code) {}
-#endif
-
 int32_t vnodeInitResources() {
   vnodeInitWriteFp();
   vnodeInitReadFp();
@@ -289,14 +280,10 @@ int32_t vnodeOpen(int32_t vnode, char *rootDir) {
   syncInfo.notifyFileSynced = vnodeNotifyFileSynced;
   pVnode->sync = syncStart(&syncInfo);
 
-#ifndef _SYNC
-  pVnode->role = TAOS_SYNC_ROLE_MASTER;
-#else
   if (pVnode->sync == NULL) {
     vnodeCleanUp(pVnode);
     return terrno;
   }
-#endif
 
   pVnode->qMgmt = qOpenQueryMgmt(pVnode->vgId);
   pVnode->events = NULL;

src/wal/CMakeLists.txt

 CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
-INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(inc)
 
 AUX_SOURCE_DIRECTORY(${CMAKE_CURRENT_SOURCE_DIR}/src SRC)

src/wal/src/walMain.c

@@ -13,14 +13,7 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <sys/types.h>
-#include <dirent.h>
-#include <unistd.h>
-#include <fcntl.h> 
-
+#define _DEFAULT_SOURCE
 #include "os.h"
 #include "tlog.h"
 #include "tchecksum.h"
@@ -29,7 +22,6 @@
 #include "taoserror.h"
 #include "twal.h"
 #include "tqueue.h"
-#include "tfile.h"
 
 #define walPrefix "wal"
 
@@ -214,7 +206,7 @@ int walWrite(void *handle, SWalHead *pHead) {
   taosCalcChecksumAppend(0, (uint8_t *)pHead, sizeof(SWalHead));
   int contLen = pHead->len + sizeof(SWalHead);
 
-  if(twrite(pWal->fd, pHead, contLen) != contLen) {
+  if(taosTWrite(pWal->fd, pHead, contLen) != contLen) {
     wError("wal:%s, failed to write(%s)", pWal->name, strerror(errno));
     terrno = TAOS_SYSTEM_ERROR(errno);
   } else {
@@ -373,7 +365,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
   wDebug("wal:%s, start to restore", name);
 
   while (1) {
-    int ret = tread(fd, pHead, sizeof(SWalHead));
+    int ret = taosTRead(fd, pHead, sizeof(SWalHead));
     if ( ret == 0)  break;  
 
     if (ret != sizeof(SWalHead)) {
@@ -388,7 +380,7 @@ static int walRestoreWalFile(SWal *pWal, void *pVnode, FWalWrite writeFp) {
       break;
     } 
 
-    ret = tread(fd, pHead->cont, pHead->len);
+    ret = taosTRead(fd, pHead->cont, pHead->len);
     if ( ret != pHead->len) {
       wWarn("wal:%s, failed to read body, skip, len:%d ret:%d", name, pHead->len, ret);
       terrno = TAOS_SYSTEM_ERROR(errno);

src/wal/test/CMakeLists.txt

@@ -2,8 +2,8 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
-  INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+  INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
   INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
   INCLUDE_DIRECTORIES(../inc)

tests/examples/go/src/taosapp/taosapp.go

@@ -13,13 +13,13 @@
  * along with this program. If not, see <http://www.gnu.org/licenses/>.
  */
 package main
- 
+
 import (
 	"database/sql"
     "time"
     "log"
     "fmt"
-	_ "taosSql"
+	_ "github.com/taosdata/driver-go/taosSql"
 )
 
 func main() {
@@ -29,7 +29,7 @@ func main() {
 
     fmt.Printf("\n======== start demo test ========\n")
     // open connect to taos server
-    db, err := sql.Open(taosDriverName, "root:taosdata@/tcp(127.0.0.1:0)/demodb")
+    db, err := sql.Open(taosDriverName, "root:taosdata@/tcp(127.0.0.1:0)/")
     if err != nil {
         log.Fatalf("Open database error: %s\n", err)
     }
@@ -58,11 +58,11 @@ func main() {
 
 func drop_database(db *sql.DB, demodb string) {
     st := time.Now().Nanosecond()
-    res, err := db.Exec("drop database " + demodb)
-    checkErr(err)
+    res, err := db.Exec("drop database if exists " + demodb)
+    checkErr(err, "drop database if exists " + demodb)
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "drop db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
 
@@ -73,10 +73,10 @@ func create_database(db *sql.DB, demodb string) {
     st := time.Now().Nanosecond()
     // create database
     res, err := db.Exec("create database " + demodb)
-    checkErr(err)
+    checkErr(err, "create db, db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
 
@@ -89,10 +89,10 @@ func use_database(db *sql.DB, demodb string) {
     st := time.Now().Nanosecond()
     // use database
     res, err := db.Exec("use " +  demodb)  // notes: must no quote to db name
-    checkErr(err)
+    checkErr(err, "use db db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "use db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
 
@@ -103,10 +103,10 @@ func create_table(db *sql.DB, demot string) {
     st := time.Now().Nanosecond()
     // create table
     res, err := db.Exec("create table " + demot + " (ts timestamp, id int, name binary(8), len tinyint, flag bool, notes binary(8), fv float, dv double)")
-    checkErr(err)
+    checkErr(err, "create table db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create table res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("create table result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -120,10 +120,10 @@ func insert_data(db *sql.DB, demot string) {
                                       " (now+1s, 101, 'shanghai', 11, true, 'two', 789.123, 789.123)" +
                                       " (now+2s, 102, 'shenzhen', 12,  false, 'three', 456.789, 456.789)")
 
-    checkErr(err)
+    checkErr(err, "insert data, db.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "insert data res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -133,7 +133,7 @@ func select_data(db *sql.DB, demot string) {
     st := time.Now().Nanosecond()
 
     rows, err := db.Query("select * from ? " , demot)  // go text mode
-    checkErr(err)
+    checkErr(err, "select db.Query")
 
     fmt.Printf("%10s%s%8s %5s %9s%s %s %8s%s %7s%s %8s%s %4s%s %5s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv")
     var affectd int
@@ -148,7 +148,7 @@ func select_data(db *sql.DB, demot string) {
         var dv float64
 
         err = rows.Scan(&ts, &id, &name, &len, &flag, &notes, &fv, &dv)
-        checkErr(err)
+        checkErr(err, "select rows.Scan")
 
         fmt.Printf("%s\t", ts)
         fmt.Printf("%d\t",id)
@@ -170,15 +170,11 @@ func select_data(db *sql.DB, demot string) {
 func drop_database_stmt(db *sql.DB,demodb string) {
     st := time.Now().Nanosecond()
     // drop test db
-    stmt, err := db.Prepare("drop database ?")
-    checkErr(err)
-    defer stmt.Close()
-
-    res, err := stmt.Exec(demodb)
-    checkErr(err)
+    res, err := db.Exec("drop database if exists " + demodb)
+    checkErr(err, "drop database " + demodb)
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "drop db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("drop database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -189,15 +185,15 @@ func create_database_stmt(db *sql.DB,demodb string)  {
     // create database
     //var stmt interface{}
     stmt, err := db.Prepare("create database ?")
-    checkErr(err)
+    checkErr(err, "create db, db.Prepare")
 
     //var res driver.Result
     res, err := stmt.Exec(demodb)
-    checkErr(err)
+    checkErr(err, "create db, stmt.Exec")
 
     //fmt.Printf("Query OK, %d row(s) affected()", res.RowsAffected())
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("create database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -208,13 +204,13 @@ func use_database_stmt (db *sql.DB,demodb string) {
     // create database
     //var stmt interface{}
     stmt, err := db.Prepare("use " + demodb)
-    checkErr(err)
+    checkErr(err, "use db, db.Prepare")
 
     res, err := stmt.Exec()
-    checkErr(err)
+    checkErr(err, "use db, stmt.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "use db, res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("use database result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -224,14 +220,14 @@ func create_table_stmt (db *sql.DB,demot string)  {
     st := time.Now().Nanosecond()
     // create table
     // (ts timestamp, id int, name binary(8), len tinyint, flag bool, notes binary(8), fv float, dv double)
-    stmt, err := db.Prepare("create table ? (? timestamp, ? int, ? binary(8), ? tinyint, ? bool, ? binary(8), ? float, ? double)")
-    checkErr(err)
+    stmt, err := db.Prepare("create table ? (? timestamp, ? int, ? binary(10), ? tinyint, ? bool, ? binary(8), ? float, ? double)")
+    checkErr(err, "create table db.Prepare")
 
     res, err := stmt.Exec(demot, "ts", "id", "name", "len", "flag", "notes", "fv", "dv")
-    checkErr(err)
+    checkErr(err, "create table stmt.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "create table res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("create table result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -241,15 +237,15 @@ func insert_data_stmt(db *sql.DB,demot string)  {
     st := time.Now().Nanosecond()
     // insert data into table
     stmt, err := db.Prepare("insert into ? values(?, ?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?, ?) (?, ?, ?, ?, ?, ?, ?, ?)")
-    checkErr(err)
+    checkErr(err, "insert db.Prepare")
 
     res, err := stmt.Exec(demot, "now" ,    1000, "'haidian'" ,    6, true, "'AI world'",  6987.654, 321.987,
                                  "now+1s", 1001, "'changyang'" ,  7, false,  "'DeepMode'", 12356.456, 128634.456,
                                  "now+2s", 1002, "'chuangping'" , 8, true, "'database'",  3879.456, 65433478.456,)
-    checkErr(err)
+    checkErr(err, "insert data, stmt.Exec")
 
     affectd, err := res.RowsAffected()
-    checkErr(err)
+    checkErr(err, "res.RowsAffected")
 
     et := time.Now().Nanosecond()
     fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
@@ -259,12 +255,12 @@ func select_data_stmt(db *sql.DB, demot string) {
     st := time.Now().Nanosecond()
 
     stmt, err := db.Prepare("select ?, ?, ?, ?, ?, ?, ?, ? from ?" )  // go binary mode
-    checkErr(err)
+    checkErr(err, "db.Prepare")
 
     rows, err := stmt.Query("ts", "id","name","len", "flag","notes", "fv", "dv", demot)
-    checkErr(err)
+    checkErr(err, "stmt.Query")
 
-    fmt.Printf("%10s%s%8s %5s %9s%s %s %8s%s %7s%s %8s%s %11s%s %14s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv")
+    fmt.Printf("%10s%s%8s %5s %8s%s %s %10s%s %7s%s %8s%s %11s%s %14s%s\n", " ","ts", " ", "id"," ", "name"," ","len", " ","flag"," ", "notes", " ", "fv", " ", " ", "dv")
     var affectd int
     for rows.Next() {
         var ts string
@@ -279,7 +275,7 @@ func select_data_stmt(db *sql.DB, demot string) {
         err = rows.Scan(&ts, &id, &name, &len, &flag, &notes, &fv, &dv)
         //fmt.Println("start scan fields from row.rs, &fv:", &fv)
         //err = rows.Scan(&fv)
-        checkErr(err)
+        checkErr(err, "rows.Scan")
 
         fmt.Printf("%s\t", ts)
         fmt.Printf("%d\t",id)
@@ -298,8 +294,9 @@ func select_data_stmt(db *sql.DB, demot string) {
     fmt.Printf("insert data result:\n %d row(s) affectd (%6.6fs)\n\n", affectd, (float32(et-st))/1E9)
 }
 
-func checkErr(err error) {
+func checkErr(err error, prompt string) {
     if err != nil {
+	fmt.Printf("%s\n", prompt)
         panic(err)
     }
 }

tests/examples/python/read_example.py

@@ -26,7 +26,6 @@ if __name__ == '__main__':
 
     # Generate a cursor object to run SQL commands
     c1 = conn.cursor()
-
     # Create a database named db
     try:
         c1.execute('create database if not exists db ')
@@ -50,9 +49,11 @@ if __name__ == '__main__':
         raise(err)
 
     # insert data 
-    for i in range(10000):
+    for i in range(10):
         try:
-           c1.execute("insert into t values ('%s', %d, %f, '%s')" % (start_time, random.randint(1,10), random.randint(1,10)/10.0, 'hello'))
+           value = c1.execute("insert into t values ('%s', %d, %f, '%s')" % (start_time, random.randint(1,10), random.randint(1,10)/10.0, 'hello'))
+           #if insert, value is the affected rows
+           print(value)
         except Exception as err:
             conn.close()
             raise(err)
@@ -70,6 +71,11 @@ if __name__ == '__main__':
     # Use fetchall to fetch data in a list
     data = c1.fetchall()
 
+    for col in data:
+        print(col)
+
+    print('Another query method ')
+
     try:
         c1.execute('select * from db.t')
     except Exception as err:

tests/perftest-scripts/cassandraTestWriteLoop.sh

@@ -3,7 +3,6 @@
 DATA_DIR=/mnt/root/testdata
 NUM_LOOP=1
 NUM_OF_FILES=100
-OUT_FILE=cassandraWrite.out
 
 rowsPerRequest=(1 10 50 100 500 1000 2000)
 
@@ -14,46 +13,56 @@ function printTo {
 }
 
 function runTest {
-  for c in `seq 1 $clients`; do
-    avgRPR[$c]=0
-  done
+  declare -A avgRPR
 
-  printf "R/R, "
-  for c in `seq 1 $clients`; do
-    if [ "$c" == "1" ]; then
-      printf "$c client, "
-    else
-      printf "$c clients, "
-    fi
+  for r in ${!rowsPerRequest[@]}; do
+    for c in `seq 1 $clients`; do
+      avgRPR[$r, $c]=0
+    done
   done
-  printf "\n"
 
-  for r in ${rowsPerRequest[@]}; do
-    printf "$r, "
+  for r in ${!rowsPerRequest[@]}; do
     for c in `seq 1 $clients`; do
       totalRPR=0
+      OUT_FILE=cassandraWrite-rows${rowsPerRequest[$r]}-clients$c.out
       for i in `seq 1 $NUM_LOOP`; do
         printTo "loop i:$i, java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
             -datadir $DATA_DIR \
             -numofFiles $NUM_OF_FILES \
-            -rowsperrequest $r \
+            -rowsperrequest ${rowsPerRequest[$r]} \
             -writeclients $c \
             -conf $CAS_TEST_DIR/application.conf"
         java -jar $CAS_TEST_DIR/cassandratest/target/cassandratest-1.0-SNAPSHOT-jar-with-dependencies.jar \
             -datadir $DATA_DIR \
             -numofFiles $NUM_OF_FILES \
-            -rowsperrequest $r \
+            -rowsperrequest ${rowsPerRequest[$r]} \
             -writeclients $c \
             -conf $CAS_TEST_DIR/application.conf \
-            2>&1 > $OUT_FILE
+            2>&1 | tee $OUT_FILE
         RPR=`cat $OUT_FILE | grep "insertation speed:" | awk '{print $(NF-1)}'`
         totalRPR=`echo "scale=4; $totalRPR + $RPR" | bc`
-        printTo "rows:$r, clients:$c, i:$i RPR:$RPR"
+        printTo "r:$r rows:${rowsPerRequest[$r]}, clients:$c, i:$i RPR:$RPR"
       done
-      avgRPR[$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc`
+      avgRPR[$r,$c]=`echo "scale=4; $totalRPR / $NUM_LOOP" | bc`
+      printTo "r:$r c:$c avgRPR:${avgRPR[$r,$c]}"
     done
+
+  done
+
+  printf "R/R, "
+  for c in `seq 1 $clients`; do
+    if [ "$c" == "1" ]; then
+      printf "$c client, "
+    else
+      printf "$c clients, "
+    fi
+  done
+  printf "\n"
+
+  for r in ${!rowsPerRequest[@]}; do
+    printf "${rowsPerRequest[$r]}, "
     for c in `seq 1 $clients`; do
-      printf "${avgRPR[$c]}, "
+      printf "${avgRPR[$r,$c]}, "
     done
     printf "\n"
   done

tests/test/c/CMakeLists.txt

@@ -2,9 +2,9 @@ CMAKE_MINIMUM_REQUIRED(VERSION 2.8)
 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 
 IF ((TD_LINUX_64) OR (TD_LINUX_32 AND TD_ARM))
   #add_executable(insertPerTable insertPerTable.c)

tests/tsim/CMakeLists.txt

 PROJECT(TDengine)
 
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/inc)
+INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/os/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/util/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/common/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/src/client/inc)
 INCLUDE_DIRECTORIES(${TD_COMMUNITY_DIR}/deps/cJson/inc)
-INCLUDE_DIRECTORIES(${TD_OS_DIR}/inc)
 INCLUDE_DIRECTORIES(inc)
 
 IF (TD_WINDOWS_64)