Merge branch 'develop' into feature/TD-2502-v2

packaging/tools/post.sh

@@ -266,8 +266,14 @@ function install_config() {
         ${csudo} chmod 644 ${cfg_install_dir}/*
     fi
     
+    # Save standard input to 6 and open / dev / TTY on standard input
+    exec 6<&0 0</dev/tty 
+
     local_fqdn_check
     
+    # restore the backup standard input, and turn off 6
+    exec 0<&6 6<&-
+
     ${csudo} mv ${cfg_dir}/taos.cfg ${cfg_dir}/taos.cfg.org
     ${csudo} ln -s ${cfg_install_dir}/taos.cfg ${cfg_dir}
     #FQDN_FORMAT="(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)"
@@ -422,7 +428,7 @@ function install_service() {
 }
 
 function install_TDengine() {
-    echo -e "${GREEN}Start to install TDEngine...${NC}"
+    echo -e "${GREEN}Start to install TDengine...${NC}"
 
     #install log and data dir , then ln to /usr/local/taos
     ${csudo} mkdir -p ${log_dir} && ${csudo} chmod 777 ${log_dir}

packaging/tools/preun.sh

@@ -119,4 +119,4 @@ if ((${service_mod}==2)); then
     kill_taosd
 fi
 
-echo -e "${GREEN}TDEngine is removed successfully!${NC}"
+echo -e "${GREEN}TDengine is removed successfully!${NC}"

packaging/tools/set_core.sh

@@ -2,19 +2,39 @@
 #
 # This file is used to set config for core when taosd crash
 
+# Color setting
+RED='\033[0;31m'
+GREEN='\033[1;32m'
+GREEN_DARK='\033[0;32m'
+GREEN_UNDERLINE='\033[4;32m'
+NC='\033[0m'
+
 set -e
 # set -x
+corePath=$1
 
 csudo=""
 if command -v sudo > /dev/null; then
   csudo="sudo"
 fi
 
-#ulimit -c unlimited
+if [[ ! -n ${corePath} ]]; then
+  echo -e -n "${GREEN}Please enter a file directory to save the coredump file${NC}:"
+	read corePath
+  while true; do
+    if [[ ! -z "$corePath" ]]; then
+      break
+    else
+      read -p "Please enter a file directory to save the coredump file:" corePath
+    fi
+  done
+fi
+
+ulimit -c unlimited
 ${csudo} sed -i '/ulimit -c unlimited/d' /etc/profile ||:
 ${csudo} sed -i '$a\ulimit -c unlimited' /etc/profile ||:
 source /etc/profile
 
-${csudo} mkdir -p /coredump  ||:
-${csudo} sysctl -w kernel.core_pattern='/coredump/core-%e-%p'  ||:
-${csudo} echo '/coredump/core-%e-%p' | ${csudo} tee /proc/sys/kernel/core_pattern  ||:
+${csudo} mkdir -p ${corePath}  ||:
+${csudo} sysctl -w kernel.core_pattern=${corePath}/core-%e-%p  ||:
+${csudo} echo "${corePath}/core-%e-%p" | ${csudo} tee /proc/sys/kernel/core_pattern  ||:

src/balance/src/bnMain.c

@@ -330,7 +330,7 @@ void bnReset() {
   tsAccessSquence = 0;
 }
 
-static int32_t bnMonitorVgroups() {
+static bool bnMonitorVgroups() {
   void *  pIter = NULL;
   SVgObj *pVgroup = NULL;
   bool    hasUpdatingVgroup = false;
@@ -489,6 +489,7 @@ void bnCheckStatus() {
         mInfo("dnode:%d, set to offline state, access seq:%d last seq:%d laststat:%d", pDnode->dnodeId, tsAccessSquence,
               pDnode->lastAccess, pDnode->status);
         bnSetVgroupOffline(pDnode);
+        bnStartTimer(3000);
       }
     }
     mnodeDecDnodeRef(pDnode);

src/balance/src/bnThread.c

@@ -31,7 +31,10 @@ static void *bnThreadFunc(void *arg) {
     }
 
     pthread_cond_wait(&tsBnThread.cond, &tsBnThread.mutex);
+    mDebug("balance thread wakes up to work");
     bool updateSoon = bnStart();
+    mDebug("balance thread finished this poll, updateSoon:%d", updateSoon);
+    
     bnStartTimer(updateSoon ? 1000 : -1);
     pthread_mutex_unlock(&(tsBnThread.mutex));
   }
@@ -101,8 +104,8 @@ static void bnProcessTimer(void *handle, void *tmrId) {
   tsBnThread.timer = NULL;
   tsAccessSquence++;
 
-  bnCheckStatus();
   bnStartTimer(-1);
+  bnCheckStatus();
 
   if (handle == NULL) {
     if (tsAccessSquence % tsBalanceInterval == 0) {
@@ -121,6 +124,7 @@ void bnStartTimer(int64_t mseconds) {
 
   bool updateSoon = (mseconds != -1);
   if (updateSoon) {
+    mTrace("balance function will be called after %" PRId64 " ms", mseconds);
     taosTmrReset(bnProcessTimer, mseconds, (void *)mseconds, tsMnodeTmr, &tsBnThread.timer);
   } else {
     taosTmrReset(bnProcessTimer, tsStatusInterval * 1000, NULL, tsMnodeTmr, &tsBnThread.timer);

src/client/src/tscAsync.c

@@ -388,10 +388,10 @@ void tscQueueAsyncRes(SSqlObj *pSql) {
     return;
   }
 
+  assert(pSql->res.code != TSDB_CODE_SUCCESS);
   tscError("%p add into queued async res, code:%s", pSql, tstrerror(pSql->res.code));
 
   SSqlRes *pRes = &pSql->res;
-
   if (pSql->fp == NULL || pSql->fetchFp == NULL){
     return;
   }

src/client/src/tscFunctionImpl.c

@@ -2597,14 +2597,23 @@ static void percentile_next_step(SQLFunctionCtx *pCtx) {
 }
 
 //////////////////////////////////////////////////////////////////////////////////
+static void buildHistogramInfo(SAPercentileInfo* pInfo) {
+  pInfo->pHisto = (SHistogramInfo*) ((char*) pInfo + sizeof(SAPercentileInfo));
+  pInfo->pHisto->elems = (SHistBin*) ((char*)pInfo->pHisto + sizeof(SHistogramInfo));
+}
+
 static SAPercentileInfo *getAPerctInfo(SQLFunctionCtx *pCtx) {
   SResultRowCellInfo *pResInfo = GET_RES_INFO(pCtx);
+  SAPercentileInfo* pInfo = NULL;
 
   if (pCtx->stableQuery && pCtx->currentStage != SECONDARY_STAGE_MERGE) {
-    return (SAPercentileInfo*) pCtx->aOutputBuf;
+    pInfo = (SAPercentileInfo*) pCtx->aOutputBuf;
   } else {
-    return GET_ROWCELL_INTERBUF(pResInfo);
+    pInfo = GET_ROWCELL_INTERBUF(pResInfo);
   }
+
+  buildHistogramInfo(pInfo);
+  return pInfo;
 }
 
 static bool apercentile_function_setup(SQLFunctionCtx *pCtx) {
@@ -2616,6 +2625,7 @@ static bool apercentile_function_setup(SQLFunctionCtx *pCtx) {
   
   char *tmp = (char *)pInfo + sizeof(SAPercentileInfo);
   pInfo->pHisto = tHistogramCreateFrom(tmp, MAX_HISTOGRAM_BIN);
+  printf("%p, %p\n", pInfo->pHisto, pInfo->pHisto->elems);
   return true;
 }
 
@@ -2625,6 +2635,8 @@ static void apercentile_function(SQLFunctionCtx *pCtx) {
   SResultRowCellInfo *     pResInfo = GET_RES_INFO(pCtx);
   SAPercentileInfo *pInfo = getAPerctInfo(pCtx);
 
+  assert(pInfo->pHisto->elems != NULL);
+  
   for (int32_t i = 0; i < pCtx->size; ++i) {
     char *data = GET_INPUT_CHAR_INDEX(pCtx, i);
     if (pCtx->hasNull && isNull(data, pCtx->inputType)) {

src/client/src/tscLocalMerge.c

@@ -911,6 +911,13 @@ static void genFinalResWithoutFill(SSqlRes* pRes, SLocalReducer *pLocalReducer,
     }
   }
 
+  if (pRes->numOfRowsGroup >= pQueryInfo->limit.limit && pQueryInfo->limit.limit > 0) {
+    pRes->numOfRows = 0;
+    pBeforeFillData->num = 0;
+    pLocalReducer->discard = true;
+    return;
+  }
+
   pRes->numOfRowsGroup += pRes->numOfRows;
 
   // impose the limitation of output rows on the final result

src/client/src/tscSql.c

@@ -284,9 +284,10 @@ void taos_close(TAOS *taos) {
     return;
   }
 
+  if (RID_VALID(pObj->hbrid)) {
     SSqlObj* pHb = (SSqlObj*)taosAcquireRef(tscObjRef, pObj->hbrid);
     if (pHb != NULL) {
-    if (pHb->rpcRid > 0) {  // wait for rsp from dnode
+      if (RID_VALID(pHb->rpcRid)) {  // wait for rsp from dnode
         rpcCancelRequest(pHb->rpcRid);
         pHb->rpcRid = -1;
       }
@@ -295,6 +296,7 @@ void taos_close(TAOS *taos) {
       taosReleaseRef(tscObjRef, pHb->self);
       taos_free_result(pHb);
     }
+  }
 
   tscDebug("%p all sqlObj are freed, free tscObj and close dnodeConn:%p", pObj, pObj->pDnodeConn);
   taosRemoveRef(tscRefId, pObj->rid);

src/common/inc/tglobal.h

@@ -101,7 +101,8 @@ extern int32_t tsAlternativeRole;
 extern int32_t tsBalanceInterval;
 extern int32_t tsOfflineThreshold;
 extern int32_t tsMnodeEqualVnodeNum;
-extern int32_t tsFlowCtrl;
+extern int32_t tsEnableFlowCtrl;
+extern int32_t tsEnableSlaveQuery;
 
 // restful
 extern int32_t  tsEnableHttpModule;

src/common/src/tglobal.c

@@ -138,7 +138,8 @@ int32_t tsAlternativeRole = 0;
 int32_t tsBalanceInterval = 300;  // seconds
 int32_t tsOfflineThreshold = 86400*100;   // seconds 10days
 int32_t tsMnodeEqualVnodeNum = 4;
-int32_t tsFlowCtrl = 1;
+int32_t tsEnableFlowCtrl = 1;
+int32_t tsEnableSlaveQuery = 1;
 
 // restful
 int32_t  tsEnableHttpModule = 1;
@@ -221,7 +222,7 @@ int32_t uDebugFlag = 131;
 int32_t debugFlag = 0;
 int32_t sDebugFlag = 135;
 int32_t wDebugFlag = 135;
-int32_t tsdbDebugFlag = 131;
+uint32_t tsdbDebugFlag = 131;
 int32_t cqDebugFlag = 131;
 
 int32_t (*monStartSystemFp)() = NULL;
@@ -542,7 +543,7 @@ static void doInitGlobalConfig(void) {
   cfg.ptr = &tsOfflineThreshold;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
-  cfg.minValue = 5;
+  cfg.minValue = 3;
   cfg.maxValue = 7200000;
   cfg.ptrLength = 0;
   cfg.unitType = TAOS_CFG_UTYPE_SECOND;
@@ -1004,7 +1005,17 @@ static void doInitGlobalConfig(void) {
 
     // module configs
   cfg.option = "flowctrl";
-  cfg.ptr = &tsFlowCtrl;
+  cfg.ptr = &tsEnableFlowCtrl;
+  cfg.valType = TAOS_CFG_VTYPE_INT32;
+  cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
+  cfg.minValue = 0;
+  cfg.maxValue = 1;
+  cfg.ptrLength = 0;
+  cfg.unitType = TAOS_CFG_UTYPE_NONE;
+  taosInitConfigOption(cfg);
+
+  cfg.option = "slaveQuery";
+  cfg.ptr = &tsEnableSlaveQuery;
   cfg.valType = TAOS_CFG_VTYPE_INT32;
   cfg.cfgType = TSDB_CFG_CTYPE_B_CONFIG | TSDB_CFG_CTYPE_B_SHOW;
   cfg.minValue = 0;

grafanaplugin @ ec77d904

-Subproject commit 32e2c97a4cf7bedaa99f5d6dd8cb036e7f4470df
+Subproject commit ec77d9049a719dabfd1a7c1122a209e201861944

src/connector/nodejs/nodetaos/cinterface.js

@@ -349,11 +349,12 @@ CTaosInterface.prototype.useResult = function useResult(result) {
   return fields;
 }
 CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
-  let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data
-  let num_of_rows = this.libtaos.taos_fetch_block(result, pblock)
-  if (num_of_rows == 0) {
+  //let pblock = ref.ref(ref.ref(ref.NULL)); // equal to our raw data
+  let pblock = this.libtaos.taos_fetch_row(result);
+  if (pblock == null) {
     return {block:null, num_of_rows:0};
   }
+
   var fieldL = this.libtaos.taos_fetch_lengths(result);
 
   let isMicro = (this.libtaos.taos_result_precision(result) == FieldTypes.C_TIMESTAMP_MICRO);
@@ -361,7 +362,6 @@ CTaosInterface.prototype.fetchBlock = function fetchBlock(result, fields) {
   var fieldlens = [];
   
   if (ref.isNull(fieldL) == false) {
-    
     for (let i = 0; i < fields.length; i ++) {
       let plen = ref.reinterpret(fieldL, 4, i*4);
       let len = plen.readInt32LE(0);

src/dnode/src/dnodeMain.c

@@ -113,6 +113,7 @@ static void dnodeCleanupTmr() {
 int32_t dnodeInitSystem() {
   dnodeSetRunStatus(TSDB_RUN_STATUS_INITIALIZE);
   tscEmbedded  = 1;
+  taosIgnSIGPIPE();
   taosBlockSIGPIPE();
   taosResolveCRC();
   taosInitGlobalCfg();
@@ -120,7 +121,6 @@ int32_t dnodeInitSystem() {
   taosSetCoreDump();
   taosInitNotes();
   dnodeInitTmr();
-  signal(SIGPIPE, SIG_IGN);
 
   if (dnodeCreateDir(tsLogDir) < 0) {
    printf("failed to create dir: %s, reason: %s\n", tsLogDir, strerror(errno));

src/dnode/src/dnodeVRead.c

@@ -54,6 +54,7 @@ void dnodeCleanupVRead() {
 void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
   int32_t queuedMsgNum = 0;
   int32_t leftLen = pMsg->contLen;
+  int32_t code = TSDB_CODE_VND_INVALID_VGROUP_ID;
   char *  pCont = pMsg->pCont;
 
   while (leftLen > 0) {
@@ -64,7 +65,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
     assert(pHead->contLen > 0);
     void *pVnode = vnodeAcquire(pHead->vgId);
     if (pVnode != NULL) {
-      int32_t code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
+      code = vnodeWriteToRQueue(pVnode, pCont, pHead->contLen, TAOS_QTYPE_RPC, pMsg);
       if (code == TSDB_CODE_SUCCESS) queuedMsgNum++;
       vnodeRelease(pVnode);
     }
@@ -74,7 +75,7 @@ void dnodeDispatchToVReadQueue(SRpcMsg *pMsg) {
   }
 
   if (queuedMsgNum == 0) {
-    SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = TSDB_CODE_VND_INVALID_VGROUP_ID};
+    SRpcMsg rpcRsp = {.handle = pMsg->handle, .code = code};
     rpcSendResponse(&rpcRsp);
   }
 

src/dnode/src/dnodeVWrite.c

@@ -188,6 +188,7 @@ static void *dnodeProcessVWriteQueue(void *wparam) {
   int32_t        numOfMsgs;
   int32_t        qtype;
 
+  taosBlockSIGPIPE();
   dDebug("dnode vwrite worker:%d is running", pWorker->workerId);
 
   while (1) {

src/inc/ttype.h

@@ -28,7 +28,7 @@ extern "C" {
     default:                                         \
       (_v) = (_finalType)GET_INT32_VAL(_data);       \
       break;                                         \
-  };
+  }
 
 #ifdef __cplusplus
 }

src/kit/taosdump/taosdump.c

@@ -332,6 +332,9 @@ static error_t parse_opt(int key, char *arg, struct argp_state *state) {
       break;
     case 'N':
       arguments->data_batch = atoi(arg);
+      if (arguments->data_batch >= INT16_MAX) {
+        arguments->data_batch = INT16_MAX - 1;
+      } 
       break;
     case 'L': 
     {

src/mnode/src/mnodeMnode.c

@@ -377,6 +377,24 @@ static int32_t mnodeCreateMnodeCb(SMnodeMsg *pMsg, int32_t code) {
   return code;
 }
 
+static bool mnodeAllOnline() {
+  void *pIter = NULL;
+  bool  allOnline = true;
+
+  while (1) {
+    SMnodeObj *pMnode = NULL;
+    pIter = mnodeGetNextMnode(pIter, &pMnode);
+    if (pMnode == NULL) break;
+    if (pMnode->role != TAOS_SYNC_ROLE_MASTER && pMnode->role != TAOS_SYNC_ROLE_SLAVE) {
+      allOnline = false;
+      mnodeDecMnodeRef(pMnode);
+    }
+  }
+  mnodeCancelGetNextMnode(pIter);
+
+  return allOnline;
+}
+
 void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
   SMnodeObj *pMnode = calloc(1, sizeof(SMnodeObj));
   pMnode->mnodeId = dnodeId;
@@ -389,6 +407,11 @@ void mnodeCreateMnode(int32_t dnodeId, char *dnodeEp, bool needConfirm) {
     .fpRsp   = mnodeCreateMnodeCb
   };
 
+  if (needConfirm && !mnodeAllOnline()) {
+    mDebug("wait all mnode online then create new mnode");
+    return;
+  }
+
   int32_t code = TSDB_CODE_SUCCESS;
   if (needConfirm) {
     code = mnodeSendCreateMnodeMsg(dnodeId, dnodeEp);

src/mnode/src/mnodeSdb.c

@@ -1081,6 +1081,8 @@ static void *sdbWorkerFp(void *pWorker) {
   int32_t  qtype;
   void *   unUsed;
 
+  taosBlockSIGPIPE();
+
   while (1) {
     int32_t numOfMsgs = taosReadAllQitemsFromQset(tsSdbWQset, tsSdbWQall, &unUsed);
     if (numOfMsgs == 0) {

src/mnode/src/mnodeVgroup.c

@@ -659,7 +659,7 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
 
   pShow->bytes[cols] = 4;
   pSchema[cols].type = TSDB_DATA_TYPE_INT;
-  strcpy(pSchema[cols].name, "onlineVnodes");
+  strcpy(pSchema[cols].name, "onlines");
   pSchema[cols].bytes = htons(pShow->bytes[cols]);
   cols++;
 
@@ -674,13 +674,13 @@ static int32_t mnodeGetVgroupMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *p
   for (int32_t i = 0; i < pShow->maxReplica; ++i) {
     pShow->bytes[cols] = 2;
     pSchema[cols].type = TSDB_DATA_TYPE_SMALLINT;
-    snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dDnode", i + 1);
+    snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_dnode", i + 1);
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
 
     pShow->bytes[cols] = 9 + VARSTR_HEADER_SIZE;
     pSchema[cols].type = TSDB_DATA_TYPE_BINARY;
-    snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%dStatus", i + 1);
+    snprintf(pSchema[cols].name, TSDB_COL_NAME_LEN, "v%d_status", i + 1);
     pSchema[cols].bytes = htons(pShow->bytes[cols]);
     cols++;
   }

src/os/inc/osSocket.h

@@ -59,6 +59,7 @@ extern "C" {
 
 // TAOS_OS_FUNC_SOCKET
 int32_t taosSetNonblocking(SOCKET sock, int32_t on);
+void    taosIgnSIGPIPE();
 void    taosBlockSIGPIPE();
 
 // TAOS_OS_FUNC_SOCKET_SETSOCKETOPT

src/os/src/detail/osSocket.c

@@ -39,6 +39,10 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
   return 0;
 }
 
+void taosIgnSIGPIPE() {
+  signal(SIGPIPE, SIG_IGN);
+}
+
 void taosBlockSIGPIPE() {
   sigset_t signal_mask;
   sigemptyset(&signal_mask);

src/os/src/windows/wSocket.c

@@ -46,6 +46,7 @@ int32_t taosSetNonblocking(SOCKET sock, int32_t on) {
   return 0;
 }
 
+void taosIgnSIGPIPE() {}
 void taosBlockSIGPIPE() {}
 
 int32_t taosSetSockOpt(SOCKET socketfd, int32_t level, int32_t optname, void *optval, int32_t optlen) {

src/query/inc/qExecutor.h

@@ -33,13 +33,6 @@ struct SColumnFilterElem;
 typedef bool (*__filter_func_t)(struct SColumnFilterElem* pFilter, char* val1, char* val2);
 typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order);
 
-typedef struct SGroupResInfo {
-  int32_t  groupId;
-  int32_t  numOfDataPages;
-  int32_t  pageId;
-  int32_t  rowId;
-} SGroupResInfo;
-
 typedef struct SResultRowPool {
   int32_t elemSize;
   int32_t blockSize;
@@ -72,6 +65,12 @@ typedef struct SResultRow {
   union {STimeWindow win; char* key;};  // start key of current time window
 } SResultRow;
 
+typedef struct SGroupResInfo {
+  int32_t rowId;
+  int32_t index;
+  SArray* pRows;      // SArray<SResultRow*>
+} SGroupResInfo;
+
 /**
  * If the number of generated results is greater than this value,
  * query query will be halt and return results to client immediate.
@@ -89,7 +88,6 @@ typedef struct SResultRowInfo {
   int32_t      size:24;    // number of result set
   int32_t      capacity;   // max capacity
   int32_t      curIndex;   // current start active index
-  int64_t      startTime;  // start time of the first time window for sliding query
   int64_t      prevSKey;   // previous (not completed) sliding window start key
 } SResultRowInfo;
 

src/query/inc/qHistogram.h

@@ -67,7 +67,7 @@ void tHistogramDestroy(SHistogramInfo** pHisto);
 
 void tHistogramPrint(SHistogramInfo* pHisto);
 
-int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val);
+int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val);
 
 SHeapEntry* tHeapCreate(int32_t numOfEntries);
 void tHeapSort(SHeapEntry* pEntry, int32_t len);

src/query/inc/qUtil.h

@@ -77,7 +77,6 @@ void* destroyResultRowPool(SResultRowPool* p);
 int32_t getNumOfAllocatedResultRows(SResultRowPool* p);
 int32_t getNumOfUsedResultRows(SResultRowPool* p);
 
-uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv);
 bool isPointInterpoQuery(SQuery *pQuery);
 
 

src/query/src/qExecutor.c

@@ -53,7 +53,7 @@
 #define TIME_WINDOW_COPY(_dst, _src)  do {\
    (_dst).skey = (_src).skey;\
    (_dst).ekey = (_src).ekey;\
-} while (0);
+} while (0)
 
 enum {
   // when query starts to execute, this status will set
@@ -178,11 +178,10 @@ static void getNextTimeWindow(SQuery* pQuery, STimeWindow* tw) {
 #define IS_STASBLE_QUERY_OVER(_q) ((_q)->tableIndex >= (int32_t)((_q)->tableqinfoGroupInfo.numOfTables))
 
 // todo move to utility
-static int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *group);
+static int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableList, SQInfo* pQInfo);
 
 static void setResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult);
 static void setResultRowOutputBufInitCtx(SQueryRuntimeEnv *pRuntimeEnv, SResultRow *pResult);
-static void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow);
 static bool functionNeedToExecute(SQueryRuntimeEnv *pRuntimeEnv, SQLFunctionCtx *pCtx, int32_t functionId);
 
 static void setExecParams(SQuery *pQuery, SQLFunctionCtx *pCtx, void* inputData, TSKEY *tsCol, SDataBlockInfo* pBlockInfo,
@@ -195,7 +194,6 @@ static bool hasMainOutput(SQuery *pQuery);
 static void buildTagQueryResult(SQInfo *pQInfo);
 
 static int32_t setAdditionalInfo(SQInfo *pQInfo, void *pTable, STableQueryInfo *pTableQueryInfo);
-static int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo);
 static int32_t checkForQueryBuf(size_t numOfTables);
 static void releaseQueryBuf(size_t numOfTables);
 static int32_t binarySearchForKey(char *pValue, int num, TSKEY key, int order);
@@ -291,7 +289,7 @@ void updateNumOfResult(SQueryRuntimeEnv *pRuntimeEnv, int32_t numOfRes) {
   }
 }
 
-static int32_t getMergeResultGroupId(int32_t groupIndex) {
+static UNUSED_FUNC int32_t getMergeResultGroupId(int32_t groupIndex) {
   int32_t base = 50000000;
   return base + (groupIndex * 10000);
 }
@@ -466,16 +464,34 @@ static bool hasNullValue(SColIndex* pColIndex, SDataStatis *pStatis, SDataStatis
 
 static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, char *pData,
                                              int16_t bytes, bool masterscan, uint64_t uid) {
+  bool existed = false;
   SET_RES_WINDOW_KEY(pRuntimeEnv->keyBuf, pData, bytes, uid);
-  int32_t *p1 =
-      (int32_t *)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
+
+  SResultRow **p1 =
+      (SResultRow **)taosHashGet(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes));
+
+  // in case of repeat scan/reverse scan, no new time window added.
+  if (QUERY_IS_INTERVAL_QUERY(pRuntimeEnv->pQuery)) {
+    if (!masterscan) {  // the *p1 may be NULL in case of sliding+offset exists.
+      return (p1 != NULL)? *p1:NULL;
+    }
+
     if (p1 != NULL) {
-    pResultRowInfo->curIndex = *p1;
+      for(int32_t i = pResultRowInfo->size - 1; i >= 0; --i) {
+        if (pResultRowInfo->pResult[i] == (*p1)) {
+          pResultRowInfo->curIndex = i;
+          existed = true;
+          break;
+        }
+      }
+    }
   } else {
-    if (!masterscan) {  // not master scan, do not add new timewindow
-      return NULL;
+    if (p1 != NULL) {  // group by column query
+      return *p1;
+    }
   }
 
+  if (!existed) {
     // TODO refactor
     // more than the capacity, reallocate the resources
     if (pResultRowInfo->size >= pResultRowInfo->capacity) {
@@ -499,17 +515,23 @@ static SResultRow *doPrepareResultRowFromKey(SQueryRuntimeEnv *pRuntimeEnv, SRes
       pResultRowInfo->capacity = (int32_t)newCapacity;
     }
 
-    SResultRow *pResult = getNewResultRow(pRuntimeEnv->pool);
-    pResultRowInfo->pResult[pResultRowInfo->size] = pResult;
+    SResultRow *pResult = NULL;
+
+    if (p1 == NULL) {
+      pResult = getNewResultRow(pRuntimeEnv->pool);
       int32_t ret = initResultRow(pResult);
       if (ret != TSDB_CODE_SUCCESS) {
         longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
       }
 
       // add a new result set for a new group
+      taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pResult, POINTER_BYTES);
+    } else {
+      pResult = *p1;
+    }
+
+    pResultRowInfo->pResult[pResultRowInfo->size] = pResult;
     pResultRowInfo->curIndex = pResultRowInfo->size++;
-    taosHashPut(pRuntimeEnv->pResultRowHashTable, pRuntimeEnv->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes),
-                (char *)&pResultRowInfo->curIndex, sizeof(int32_t));
   }
 
   // too many time window in query
@@ -591,7 +613,6 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf
     if (pData->num >= numOfRowsPerPage) {
       // release current page first, and prepare the next one
       releaseResBufPageInfo(pResultBuf, pi);
-
       pData = getNewDataBuf(pResultBuf, tid, &pageId);
       if (pData != NULL) {
         assert(pData->num == 0);  // number of elements must be 0 for new allocated buffer
@@ -614,24 +635,20 @@ static int32_t addNewWindowResultBuf(SResultRow *pWindowRes, SDiskbasedResultBuf
   return 0;
 }
 
-static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, SDataBlockInfo* pBockInfo,
-                                       STimeWindow *win, bool masterscan, bool* newWind, SResultRow** pResult) {
+static int32_t setWindowOutputBufByKey(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRowInfo, STimeWindow *win,
+    bool masterscan, SResultRow** pResult, int64_t groupId) {
   assert(win->skey <= win->ekey);
   SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
 
-  // todo refactor
-  int64_t uid = getResultInfoUId(pRuntimeEnv);
-  SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, uid);
+  SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pResultRowInfo, (char *)&win->skey, TSDB_KEYSIZE, masterscan, groupId);
   if (pResultRow == NULL) {
-    *newWind = false;
-    return masterscan? -1:0;   // no master scan, no result generated means error occurs
+    *pResult = NULL;
+    return TSDB_CODE_SUCCESS;
   }
 
-  *newWind = true;
-
   // not assign result buffer yet, add new result buffer
   if (pResultRow->pageId == -1) {
-    int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, pBockInfo->tid, pRuntimeEnv->numOfRowsPerPage);
+    int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, (int32_t) groupId, pRuntimeEnv->numOfRowsPerPage);
     if (ret != TSDB_CODE_SUCCESS) {
       return -1;
     }
@@ -701,81 +718,47 @@ static FORCE_INLINE int32_t getForwardStepsInBlock(int32_t numOfRows, __block_se
   return forwardStep;
 }
 
-static int32_t updateResultRowCurrentIndex(SResultRowInfo* pWindowResInfo, TSKEY lastKey, bool ascQuery) {
-  int32_t i = 0;
+static void doUpdateResultRowIndex(SResultRowInfo*pResultRowInfo, TSKEY lastKey, bool ascQuery) {
   int64_t skey = TSKEY_INITIAL_VAL;
-
-  int32_t numOfClosed = 0;
-  for (i = 0; i < pWindowResInfo->size; ++i) {
-    SResultRow *pResult = pWindowResInfo->pResult[i];
+  int32_t i = 0;
+  for (i = pResultRowInfo->size - 1; i >= 0; --i) {
+    SResultRow *pResult = pResultRowInfo->pResult[i];
     if (pResult->closed) {
-      numOfClosed += 1;
-      continue;
+      break;
     }
 
-    TSKEY ekey = pResult->win.ekey;
-    if ((ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) {
-      closeResultRow(pWindowResInfo, i);
+    // new closed result rows
+    if ((pResult->win.ekey <= lastKey && ascQuery) || (pResult->win.skey >= lastKey && !ascQuery)) {
+      closeResultRow(pResultRowInfo, i);
     } else {
       skey = pResult->win.skey;
-      break;
     }
   }
 
-  // all windows are closed, set the last one to be the skey
+  // all result rows are closed, set the last one to be the skey
   if (skey == TSKEY_INITIAL_VAL) {
-    assert(i == pWindowResInfo->size);
-    pWindowResInfo->curIndex = pWindowResInfo->size - 1;
+    pResultRowInfo->curIndex = pResultRowInfo->size - 1;
   } else {
-    pWindowResInfo->curIndex = i;
-    pWindowResInfo->prevSKey = pWindowResInfo->pResult[pWindowResInfo->curIndex]->win.skey;
-  }
-
-  return numOfClosed;
-}
-
-/**
- * NOTE: the query status only set for the first scan of master scan.
- */
-static int32_t doCheckQueryCompleted(SQueryRuntimeEnv *pRuntimeEnv, TSKEY lastKey, SResultRowInfo *pWindowResInfo) {
-  SQuery *pQuery = pRuntimeEnv->pQuery;
-  if (pRuntimeEnv->scanFlag != MASTER_SCAN || pWindowResInfo->size == 0) {
-    return pWindowResInfo->size;
-  }
-
-  // no qualified results exist, abort check
-  int32_t numOfClosed = 0;
-  bool ascQuery = QUERY_IS_ASC_QUERY(pQuery);
-
-  // query completed
-  if ((lastKey >= pQuery->current->win.ekey && ascQuery) || (lastKey <= pQuery->current->win.ekey && (!ascQuery))) {
-    closeAllResultRows(pWindowResInfo);
-
-    pWindowResInfo->curIndex = pWindowResInfo->size - 1;
-    setQueryStatus(pQuery, QUERY_COMPLETED | QUERY_RESBUF_FULL);
-  } else {  // set the current index to be the last unclosed window
-    numOfClosed = updateResultRowCurrentIndex(pWindowResInfo, lastKey, ascQuery);
-
-    // the number of completed slots are larger than the threshold, return current generated results to client.
-    if (numOfClosed > pQuery->rec.threshold) {
-      qDebug("QInfo:%p total result window:%d closed:%d, reached the output threshold %d, return",
-          GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size, numOfClosed, pQuery->rec.threshold);
 
-      setQueryStatus(pQuery, QUERY_RESBUF_FULL);
-    } else {
-      qDebug("QInfo:%p total result window:%d already closed:%d", GET_QINFO_ADDR(pRuntimeEnv), pWindowResInfo->size,
-             numOfClosed);
+    for (i = pResultRowInfo->size - 1; i >= 0; --i) {
+      SResultRow *pResult = pResultRowInfo->pResult[i];
+      if (pResult->closed) {
+        break;
       }
     }
 
-  // output has reached the limitation, set query completed
-  if (pQuery->limit.limit > 0 && (pQuery->limit.limit + pQuery->limit.offset) <= numOfClosed &&
-      pRuntimeEnv->scanFlag == MASTER_SCAN) {
-    setQueryStatus(pQuery, QUERY_COMPLETED);
+    pResultRowInfo->curIndex = i + 1;  // current not closed result object
+    pResultRowInfo->prevSKey = pResultRowInfo->pResult[pResultRowInfo->curIndex]->win.skey;
   }
+}
 
-  assert(pWindowResInfo->prevSKey != TSKEY_INITIAL_VAL);
-  return numOfClosed;
+static void updateResultRowIndex(SResultRowInfo* pResultRowInfo, STableQueryInfo* pTableQueryInfo, bool ascQuery) {
+  if ((pTableQueryInfo->lastKey >= pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey <= pTableQueryInfo->win.ekey && (!ascQuery))) {
+    closeAllResultRows(pResultRowInfo);
+    pResultRowInfo->curIndex = pResultRowInfo->size - 1;
+  } else {
+    doUpdateResultRowIndex(pResultRowInfo, pTableQueryInfo->lastKey, ascQuery);
+  }
 }
 
 static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo, TSKEY *pPrimaryColumn,
@@ -818,14 +801,12 @@ static int32_t getNumOfRowsInTimeWindow(SQuery *pQuery, SDataBlockInfo *pDataBlo
   return num;
 }
 
-static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset,
-                                      int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) {
-  SQuery *        pQuery = pRuntimeEnv->pQuery;
+static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset, int32_t forwardStep, TSKEY *tsCol, int32_t numOfTotal) {
+  SQuery         *pQuery = pRuntimeEnv->pQuery;
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
   bool hasPrev = pCtx[0].preAggVals.isSet;
 
-  if (IS_MASTER_SCAN(pRuntimeEnv) || closed) {
   for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
     pCtx[k].nStartQueryTimestamp = pWin->skey;
     pCtx[k].size = forwardStep;
@@ -849,14 +830,12 @@ static void doBlockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed
     // restore it
     pCtx[k].preAggVals.isSet = hasPrev;
   }
-  }
 }
 
-static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed, STimeWindow *pWin, int32_t offset) {
-  SQuery *        pQuery = pRuntimeEnv->pQuery;
+static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pWin, int32_t offset) {
+  SQuery         *pQuery = pRuntimeEnv->pQuery;
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
-  if (IS_MASTER_SCAN(pRuntimeEnv) || closed) {
   for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
     pCtx[k].nStartQueryTimestamp = pWin->skey;
 
@@ -865,7 +844,6 @@ static void doRowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, bool closed,
       aAggs[functionId].xFunctionF(&pCtx[k], offset);
     }
   }
-  }
 }
 
 static int32_t getNextQualifiedWindow(SQueryRuntimeEnv *pRuntimeEnv, STimeWindow *pNext, SDataBlockInfo *pDataBlockInfo,
@@ -1124,6 +1102,45 @@ static TSKEY getStartTsKey(SQuery* pQuery, SDataBlockInfo* pDataBlockInfo, TSKEY
   return ts;
 }
 
+static void doWindowBorderInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo* pDataBlockInfo, SArray *pDataBlock,
+    SResultRow* pResult, STimeWindow* win, int32_t startPos, int32_t forwardStep) {
+  if (!pRuntimeEnv->timeWindowInterpo) {
+    return;
+  }
+
+  assert(pDataBlock != NULL);
+
+  SQuery* pQuery = pRuntimeEnv->pQuery;
+  int32_t step = GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
+
+  SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0);
+
+  TSKEY  *tsCols = (TSKEY *)(pColInfo->pData);
+  bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
+  if (!done) {
+    int32_t startRowIndex = startPos;
+    bool    interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, win);
+    if (interp) {
+      setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
+    }
+  } else {
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
+  }
+
+  done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
+  if (!done) {
+    int32_t endRowIndex = startPos + (forwardStep - 1) * step;
+
+    TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey;
+    bool  interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, win);
+    if (interp) {
+      setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
+    }
+  } else {
+    setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP);
+  }
+}
+
 /**
  * todo set the last value for pQueryTableInfo as in rowwiseapplyfunctions
  * @param pRuntimeEnv
@@ -1140,6 +1157,8 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
   bool            masterScan = IS_MASTER_SCAN(pRuntimeEnv);
 
   SQuery *pQuery  = pRuntimeEnv->pQuery;
+  int64_t groupId = pQuery->current->groupIndex;
+
   TSKEY  *tsCols = NULL;
   if (pDataBlock != NULL) {
     SColumnInfoData *pColInfo = taosArrayGet(pDataBlock, 0);
@@ -1164,19 +1183,15 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
     TSKEY ts = getStartTsKey(pQuery, pDataBlockInfo, tsCols, step);
     STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
 
-    bool hasTimeWindow  = false;
     SResultRow* pResult = NULL;
-    int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult);
-    if (ret != TSDB_CODE_SUCCESS) {
-      tfree(sasArray);
-      return;
+    int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId);
+    if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {
+      goto _end;
     }
 
     int32_t forwardStep = 0;
     int32_t startPos = pQuery->pos;
 
-    // in case of repeat scan/reverse scan, no new time window added.
-    if (hasTimeWindow) {
     TSKEY ekey = reviseWindowEkey(pQuery, &win);
     forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, pQuery->pos, ekey, searchFn, true);
 
@@ -1187,7 +1202,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
         SResultRow *pRes = pWindowResInfo->pResult[j];
 
         STimeWindow w = pRes->win;
-          ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &w, masterScan, &hasTimeWindow, &pResult);
+        ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &w, masterScan, &pResult, groupId);
         assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP));
 
         int32_t p = QUERY_IS_ASC_QUERY(pQuery)? 0:pDataBlockInfo->rows-1;
@@ -1195,49 +1210,19 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
         setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
         setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
 
-          bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
-          doBlockwiseApplyFunctions(pRuntimeEnv, closed, &w, startPos, 0, tsCols, pDataBlockInfo->rows);
+        doBlockwiseApplyFunctions(pRuntimeEnv, &w, startPos, 0, tsCols, pDataBlockInfo->rows);
       }
 
       // restore current time window
-        ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult);
-        assert (ret == TSDB_CODE_SUCCESS);  // null data, too many state code
+      ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId);
+      assert (ret == TSDB_CODE_SUCCESS);
     }
 
     // window start key interpolation
-      if (pRuntimeEnv->timeWindowInterpo) {
-        bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
-        if (!done) {
-          int32_t startRowIndex = pQuery->pos;
-          bool    interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &win);
-          if (interp) {
-            setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
-          }
-        } else {
-          setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
-        }
-
-        done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
-        if (!done) {
-          int32_t endRowIndex = pQuery->pos + (forwardStep - 1) * step;
-
-          TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey;
-          bool  interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &win);
-          if (interp) {
-            setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
-          }
-        } else {
-          setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP);
-        }
-      }
-
-      bool pStatus = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
-      doBlockwiseApplyFunctions(pRuntimeEnv, pStatus, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows);
-    }
+    doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &win, pQuery->pos, forwardStep);
+    doBlockwiseApplyFunctions(pRuntimeEnv, &win, startPos, forwardStep, tsCols, pDataBlockInfo->rows);
 
-    int32_t     index = pWindowResInfo->curIndex;
     STimeWindow nextWin = win;
-
     while (1) {
       int32_t prevEndPos = (forwardStep - 1) * step + startPos;
       startPos = getNextQualifiedWindow(pRuntimeEnv, &nextWin, pDataBlockInfo, tsCols, searchFn, prevEndPos);
@@ -1246,50 +1231,19 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
       }
 
       // null data, failed to allocate more memory buffer
-      hasTimeWindow = false;
-      if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) !=
-          TSDB_CODE_SUCCESS) {
+      int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId);
+      if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
         break;
       }
 
-      if (!hasTimeWindow) {
-        continue;
-      }
-
-      TSKEY ekey = reviseWindowEkey(pQuery, &nextWin);
+      ekey = reviseWindowEkey(pQuery, &nextWin);
       forwardStep = getNumOfRowsInTimeWindow(pQuery, pDataBlockInfo, tsCols, startPos, ekey, searchFn, true);
 
       // window start(end) key interpolation
-      if (pRuntimeEnv->timeWindowInterpo) {
-        bool done = resultRowInterpolated(pResult, RESULT_ROW_START_INTERP);
-        if (!done) {
-          int32_t startRowIndex = startPos;
-          bool    interp = setTimeWindowInterpolationStartTs(pRuntimeEnv, startRowIndex, pDataBlockInfo->rows, pDataBlock, tsCols, &nextWin);
-          if (interp) {
-            setResultRowInterpo(pResult, RESULT_ROW_START_INTERP);
-          }
-        } else {
-          setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_START_INTERP);
+      doWindowBorderInterpolation(pRuntimeEnv, pDataBlockInfo, pDataBlock, pResult, &nextWin, startPos, forwardStep);
+      doBlockwiseApplyFunctions(pRuntimeEnv, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows);
     }
 
-        done = resultRowInterpolated(pResult, RESULT_ROW_END_INTERP);
-        if (!done) {
-          int32_t endRowIndex = startPos + (forwardStep - 1)*step;
-          TSKEY endKey = QUERY_IS_ASC_QUERY(pQuery)? pDataBlockInfo->window.ekey:pDataBlockInfo->window.skey;
-          bool  interp = setTimeWindowInterpolationEndTs(pRuntimeEnv, endRowIndex, pDataBlock, tsCols, endKey, &nextWin);
-          if (interp) {
-            setResultRowInterpo(pResult, RESULT_ROW_END_INTERP);
-          }
-        } else {
-          setNotInterpoWindowKey(pRuntimeEnv->pCtx, pQuery->numOfOutput, RESULT_ROW_END_INTERP);
-        }
-      }
-
-      bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
-      doBlockwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, startPos, forwardStep, tsCols, pDataBlockInfo->rows);
-    }
-
-    pWindowResInfo->curIndex = index;
   } else {
     /*
      * the sqlfunctionCtx parameters should be set done before all functions are invoked,
@@ -1305,6 +1259,7 @@ static void blockwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *
     }
   }
 
+  _end:
   if (pRuntimeEnv->timeWindowInterpo) {
     saveDataBlockLastRow(pRuntimeEnv, pDataBlockInfo, pDataBlock);
   }
@@ -1325,8 +1280,6 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
     return -1;
   }
 
-  int32_t GROUPRESULTID = 1;
-
   SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
 
   // not assign result buffer yet, add new result buffer, TODO remove it
@@ -1342,11 +1295,8 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
     longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_APP_ERROR);
   }
 
-  uint64_t uid = groupIndex;
-  SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, uid);
-  if (pResultRow == NULL) {
-    return -1;
-  }
+  SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, d, len, true, groupIndex);
+  assert (pResultRow != NULL);
 
   int64_t v = -1;
   GET_TYPED_DATA(v, int64_t, type, pData);
@@ -1363,7 +1313,7 @@ static int32_t setGroupResultOutputBuf(SQueryRuntimeEnv *pRuntimeEnv, char *pDat
   }
 
   if (pResultRow->pageId == -1) {
-    int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, GROUPRESULTID, pRuntimeEnv->numOfRowsPerPage);
+    int32_t ret = addNewWindowResultBuf(pResultRow, pResultBuf, groupIndex, pRuntimeEnv->numOfRowsPerPage);
     if (ret != 0) {
       return -1;
     }
@@ -1498,7 +1448,7 @@ void doRowwiseTimeWindowInterpolation(SQueryRuntimeEnv* pRuntimeEnv, SArray* pDa
     double v1 = 0, v2 = 0, v = 0;
 
     if (prevRowIndex == -1) {
-      GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[k]);
+      GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pRuntimeEnv->prevRow[index]);
     } else {
       GET_TYPED_DATA(v1, double, pColInfo->info.type, (char *)pColInfo->pData + prevRowIndex * pColInfo->info.bytes);
     }
@@ -1565,6 +1515,8 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
   SQuery *pQuery = pRuntimeEnv->pQuery;
   STableQueryInfo* item = pQuery->current;
 
+  int64_t groupId = item->groupIndex;
+
   SColumnInfoData* pColumnInfoData = (SColumnInfoData *)taosArrayGet(pDataBlock, 0);
 
   TSKEY  *tsCols = (pColumnInfoData->info.type == TSDB_DATA_TYPE_TIMESTAMP)? (TSKEY*) pColumnInfoData->pData:NULL;
@@ -1635,15 +1587,10 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
 
       STimeWindow win = getActiveTimeWindow(pWindowResInfo, ts, pQuery);
 
-      bool hasTimeWindow  = false;
       SResultRow* pResult = NULL;
-      int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow, &pResult);
-      if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
-        continue;
-      }
-
-      if (!hasTimeWindow) {
-        continue;
+      int32_t ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId);
+      if (ret != TSDB_CODE_SUCCESS || pResult == NULL) {  // null data, too many state code
+        goto _end;
       }
 
       // window start key interpolation
@@ -1654,18 +1601,15 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
           for (int32_t k = prevWindowIndex; k < curIndex; ++k) {
             SResultRow *pRes = pWindowResInfo->pResult[k];
 
-            ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &pRes->win, masterScan, &hasTimeWindow, &pResult);
+            ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &pRes->win, masterScan, &pResult, groupId);
             assert(ret == TSDB_CODE_SUCCESS && !resultRowInterpolated(pResult, RESULT_ROW_END_INTERP));
 
             setTimeWindowEKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &pRes->win);
-
-            bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
-            doRowwiseApplyFunctions(pRuntimeEnv, closed, &pRes->win, offset);
+            doRowwiseApplyFunctions(pRuntimeEnv, &pRes->win, offset);
           }
 
           // restore current time window
-          ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &win, masterScan, &hasTimeWindow,
-                                        &pResult);
+          ret = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId);
           if (ret != TSDB_CODE_SUCCESS) {  // null data, too many state code
             continue;
           }
@@ -1674,8 +1618,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
         setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &win);
       }
 
-      bool closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
-      doRowwiseApplyFunctions(pRuntimeEnv, closed, &win, offset);
+      doRowwiseApplyFunctions(pRuntimeEnv, &win, offset);
 
       STimeWindow nextWin = win;
       int32_t     index = pWindowResInfo->curIndex;
@@ -1692,16 +1635,13 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
         }
 
         // null data, failed to allocate more memory buffer
-        hasTimeWindow = false;
-        if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pDataBlockInfo, &nextWin, masterScan, &hasTimeWindow, &pResult) != TSDB_CODE_SUCCESS) {
+        int32_t code = setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &nextWin, masterScan, &pResult, groupId);
+        if (code != TSDB_CODE_SUCCESS || pResult == NULL) {
           break;
         }
 
-        if (hasTimeWindow) {
         setTimeWindowSKeyInterp(pRuntimeEnv, pDataBlock, prevTs, prevRowIndex, ts, offset, pResult, &nextWin);
-          closed = getResultRowStatus(pWindowResInfo, curTimeWindowIndex(pWindowResInfo));
-          doRowwiseApplyFunctions(pRuntimeEnv, closed, &nextWin, offset);
-        }
+        doRowwiseApplyFunctions(pRuntimeEnv, &nextWin, offset);
       }
 
       pWindowResInfo->curIndex = index;
@@ -1736,6 +1676,7 @@ static void rowwiseApplyFunctions(SQueryRuntimeEnv *pRuntimeEnv, SDataStatis *pS
     }
   }
 
+  _end:
   assert(offset >= 0);
   if (tsCols != NULL) {
     item->lastKey = tsCols[offset] + step;
@@ -1763,28 +1704,26 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
                                           SDataStatis *pStatis, __block_search_fn_t searchFn, SArray *pDataBlock) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
-  STableQueryInfo* pTableQInfo = pQuery->current;
-  SResultRowInfo*  pWindowResInfo = &pRuntimeEnv->windowResInfo;
+  STableQueryInfo* pTableQueryInfo = pQuery->current;
+  SResultRowInfo*  pResultRowInfo = &pRuntimeEnv->windowResInfo;
 
   if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyNormalCol) {
-    rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock);
+    rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock);
   } else {
-    blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock);
+    blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock);
   }
 
   // update the lastkey of current table
   TSKEY lastKey = QUERY_IS_ASC_QUERY(pQuery) ? pDataBlockInfo->window.ekey : pDataBlockInfo->window.skey;
-  pTableQInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
+  pTableQueryInfo->lastKey = lastKey + GET_FORWARD_DIRECTION_FACTOR(pQuery->order.order);
 
   // interval query with limit applied
   int32_t numOfRes = 0;
-  if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
-    numOfRes = doCheckQueryCompleted(pRuntimeEnv, lastKey, pWindowResInfo);
-  } else if (pRuntimeEnv->groupbyNormalCol) {
-    closeAllResultRows(pWindowResInfo);
-    numOfRes = pWindowResInfo->size;
+  if (QUERY_IS_INTERVAL_QUERY(pQuery) || pRuntimeEnv->groupbyNormalCol) {
+    numOfRes = pResultRowInfo->size;
+    updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery));
   } else { // projection query
-    numOfRes = (int32_t)getNumOfResult(pRuntimeEnv);
+    numOfRes = (int32_t) getNumOfResult(pRuntimeEnv);
 
     // update the number of output result
     if (numOfRes > 0 && pQuery->checkBuffer == 1) {
@@ -1799,8 +1738,8 @@ static int32_t tableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBl
         setQueryStatus(pQuery, QUERY_COMPLETED);
       }
 
-      if (((pTableQInfo->lastKey > pTableQInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) ||
-          ((pTableQInfo->lastKey < pTableQInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) {
+      if (((pTableQueryInfo->lastKey > pTableQueryInfo->win.ekey) && QUERY_IS_ASC_QUERY(pQuery)) ||
+          ((pTableQueryInfo->lastKey < pTableQueryInfo->win.ekey) && (!QUERY_IS_ASC_QUERY(pQuery)))) {
         setQueryStatus(pQuery, QUERY_COMPLETED);
       }
     }
@@ -2182,8 +2121,8 @@ static bool isFixedOutputQuery(SQueryRuntimeEnv* pRuntimeEnv) {
 // todo refactor with isLastRowQuery
 bool isPointInterpoQuery(SQuery *pQuery) {
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-    int32_t functionID = pQuery->pExpr1[i].base.functionId;
-    if (functionID == TSDB_FUNC_INTERP) {
+    int32_t functionId = pQuery->pExpr1[i].base.functionId;
+    if (functionId == TSDB_FUNC_INTERP) {
       return true;
     }
   }
@@ -2618,6 +2557,8 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW
   *status = BLK_DATA_NO_NEEDED;
 
   SQuery *pQuery = pRuntimeEnv->pQuery;
+  int64_t groupId = pQuery->current->groupIndex;
+
   SQueryCostInfo* pCost = &pRuntimeEnv->summary;
 
   if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf > 0) {
@@ -2634,15 +2575,13 @@ int32_t loadDataBlockOnDemand(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo * pW
       // the pCtx[i] result is belonged to previous time window since the outputBuf has not been set yet,
       // the filter result may be incorrect. So in case of interval query, we need to set the correct time output buffer
       if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
-        bool hasTimeWindow = false;
         SResultRow* pResult = NULL;
 
         bool masterScan = IS_MASTER_SCAN(pRuntimeEnv);
 
         TSKEY k = QUERY_IS_ASC_QUERY(pQuery)? pBlockInfo->window.skey:pBlockInfo->window.ekey;
         STimeWindow win = getActiveTimeWindow(pWindowResInfo, k, pQuery);
-        if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, pBlockInfo, &win, masterScan, &hasTimeWindow, &pResult) !=
-            TSDB_CODE_SUCCESS) {
+        if (setWindowOutputBufByKey(pRuntimeEnv, pWindowResInfo, &win, masterScan, &pResult, groupId) != TSDB_CODE_SUCCESS) {
           // todo handle error in set result for timewindow
         }
       }
@@ -2840,13 +2779,9 @@ static void doSetInitialTimewindow(SQueryRuntimeEnv* pRuntimeEnv, SDataBlockInfo
 
     if (QUERY_IS_ASC_QUERY(pQuery)) {
       getAlignQueryTimeWindow(pQuery, pBlockInfo->window.skey, pBlockInfo->window.skey, pQuery->window.ekey, &w);
-      pWindowResInfo->startTime = w.skey;
       pWindowResInfo->prevSKey = w.skey;
-    } else {
-      // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
+    } else { // the start position of the first time window in the endpoint that spreads beyond the queried last timestamp
       getAlignQueryTimeWindow(pQuery, pBlockInfo->window.ekey, pQuery->window.ekey, pBlockInfo->window.ekey, &w);
-
-      pWindowResInfo->startTime = pQuery->window.skey;
       pWindowResInfo->prevSKey = w.skey;
     }
   }
@@ -2916,13 +2851,9 @@ static int64_t doScanAllDataBlocks(SQueryRuntimeEnv *pRuntimeEnv) {
     setQueryStatus(pQuery, QUERY_COMPLETED);
   }
 
-  if (QUERY_IS_INTERVAL_QUERY(pQuery) && (IS_MASTER_SCAN(pRuntimeEnv)|| pRuntimeEnv->scanFlag == REPEAT_SCAN)) {
-    if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
+  if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
     closeAllResultRows(&pRuntimeEnv->windowResInfo);
     pRuntimeEnv->windowResInfo.curIndex = pRuntimeEnv->windowResInfo.size - 1;  // point to the last time window
-    } else {
-      assert(Q_STATUS_EQUAL(pQuery->status, QUERY_RESBUF_FULL));
-    }
   }
 
   return 0;
@@ -3027,7 +2958,7 @@ void setTagVal(SQueryRuntimeEnv *pRuntimeEnv, void *pTable, void *tsdb) {
   }
 }
 
-static void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) {
+static UNUSED_FUNC void doMerge(SQueryRuntimeEnv *pRuntimeEnv, int64_t timestamp, SResultRow *pWindowRes, bool mergeFlag) {
   SQuery *        pQuery = pRuntimeEnv->pQuery;
   SQLFunctionCtx *pCtx = pRuntimeEnv->pCtx;
 
@@ -3176,8 +3107,8 @@ void UNUSED_FUNC displayInterResult(tFilePage **pdata, SQueryRuntimeEnv* pRuntim
 
 typedef struct SCompSupporter {
   STableQueryInfo **pTableQueryInfo;
-  int32_t *         position;
-  SQInfo *          pQInfo;
+  int32_t          *rowIndex;
+  int32_t           order;
 } SCompSupporter;
 
 int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param) {
@@ -3185,10 +3116,9 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param)
   int32_t right = *(int32_t *)pRight;
 
   SCompSupporter *  supporter = (SCompSupporter *)param;
-  SQueryRuntimeEnv *pRuntimeEnv = &supporter->pQInfo->runtimeEnv;
 
-  int32_t leftPos = supporter->position[left];
-  int32_t rightPos = supporter->position[right];
+  int32_t leftPos  = supporter->rowIndex[left];
+  int32_t rightPos = supporter->rowIndex[right];
 
   /* left source is exhausted */
   if (leftPos == -1) {
@@ -3200,53 +3130,55 @@ int32_t tableResultComparFn(const void *pLeft, const void *pRight, void *param)
     return -1;
   }
 
-  SResultRowInfo *pWindowResInfo1 = &supporter->pTableQueryInfo[left]->windowResInfo;
-  SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos);
-  tFilePage *page1 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes1->pageId);
+  STableQueryInfo** pList = supporter->pTableQueryInfo;
 
-  char *b1 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes1, page1);
-  TSKEY leftTimestamp = GET_INT64_VAL(b1);
+  SResultRowInfo *pWindowResInfo1 = &(pList[left]->windowResInfo);
+  SResultRow * pWindowRes1 = getResultRow(pWindowResInfo1, leftPos);
+  TSKEY leftTimestamp = pWindowRes1->win.skey;
 
-  SResultRowInfo *pWindowResInfo2 = &supporter->pTableQueryInfo[right]->windowResInfo;
+  SResultRowInfo *pWindowResInfo2 = &(pList[right]->windowResInfo);
   SResultRow * pWindowRes2 = getResultRow(pWindowResInfo2, rightPos);
-  tFilePage *page2 = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes2->pageId);
-
-  char *b2 = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes2, page2);
-  TSKEY rightTimestamp = GET_INT64_VAL(b2);
+  TSKEY rightTimestamp = pWindowRes2->win.skey;
 
   if (leftTimestamp == rightTimestamp) {
     return 0;
   }
 
-  return leftTimestamp > rightTimestamp ? 1 : -1;
+  if (supporter->order == TSDB_ORDER_ASC) {
+    return (leftTimestamp > rightTimestamp)? 1:-1;
+  } else {
+    return (leftTimestamp < rightTimestamp)? 1:-1;
+  }
 }
 
-int32_t mergeIntoGroupResult(SQInfo *pQInfo) {
+int32_t mergeGroupResult(SQInfo *pQInfo) {
   int64_t st = taosGetTimestampUs();
-  int32_t ret = TSDB_CODE_SUCCESS;
 
-  int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo));
+  SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
 
+  int32_t numOfGroups = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo));
   while (pQInfo->groupIndex < numOfGroups) {
     SArray *group = GET_TABLEGROUP(pQInfo, pQInfo->groupIndex);
-    ret = mergeIntoGroupResultImpl(pQInfo, group);
-    if (ret < 0) {  // not enough disk space to save the data into disk
+
+    int32_t ret = mergeIntoGroupResultImpl(pGroupResInfo, group, pQInfo);
+    if (ret < 0) {
       return -1;
     }
 
-    pQInfo->groupIndex += 1;
-
     // this group generates at least one result, return results
-    if (ret > 0) {
+    pQInfo->groupIndex += 1;
+    if (taosArrayGetSize(pGroupResInfo->pRows) > 0) {
       break;
     }
 
-    assert(pQInfo->groupResInfo.numOfDataPages == 0);
     qDebug("QInfo:%p no result in group %d, continue", pQInfo, pQInfo->groupIndex - 1);
+    taosArrayClear(pGroupResInfo->pRows);
+
+    pGroupResInfo->index = 0;
+    pGroupResInfo->rowId = 0;
   }
 
-  SGroupResInfo* info = &pQInfo->groupResInfo;
-  if (pQInfo->groupIndex == numOfGroups && info->pageId == info->numOfDataPages) {
+  if (pQInfo->groupIndex == numOfGroups && taosArrayGetSize(pGroupResInfo->pRows) == 0) {
     SET_STABLE_QUERY_OVER(pQInfo);
   }
 
@@ -3258,89 +3190,28 @@ int32_t mergeIntoGroupResult(SQInfo *pQInfo) {
   return TSDB_CODE_SUCCESS;
 }
 
+static int32_t doCopyToSData(SQInfo *pQInfo, SResultRow **pRows, int32_t numOfRows, int32_t* index, int32_t orderType);
+
 void copyResToQueryResultBuf(SQInfo *pQInfo, SQuery *pQuery) {
   SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
 
-  // all results have been return to client, try next group
-  if (pGroupResInfo->pageId == pGroupResInfo->numOfDataPages) {
-    pGroupResInfo->numOfDataPages = 0;
-    pGroupResInfo->pageId = 0;
-    pGroupResInfo->rowId = 0;
-
+  // all results in current group have been returned to client, try next group
+  if (pGroupResInfo->index >= taosArrayGetSize(pGroupResInfo->pRows)) {
     // current results of group has been sent to client, try next group
-    if (mergeIntoGroupResult(pQInfo) != TSDB_CODE_SUCCESS) {
+    if (mergeGroupResult(pQInfo) != TSDB_CODE_SUCCESS) {
       return;  // failed to save data in the disk
     }
 
     // check if all results has been sent to client
     int32_t numOfGroup = (int32_t)(GET_NUM_OF_TABLEGROUP(pQInfo));
-    if (pGroupResInfo->numOfDataPages == 0 && pQInfo->groupIndex == numOfGroup) {
+    if (taosArrayGetSize(pGroupResInfo->pRows) == 0 && pQInfo->groupIndex == numOfGroup) {
       SET_STABLE_QUERY_OVER(pQInfo);
       return;
     }
   }
 
-  SQueryRuntimeEnv *   pRuntimeEnv = &pQInfo->runtimeEnv;
-  SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
-
-  int32_t id = pQInfo->groupResInfo.groupId;
-  SIDList list = getDataBufPagesIdList(pResultBuf, id);
-
-  int32_t offset = 0;
-  int32_t numOfCopiedRows = 0;
-
-  size_t size = taosArrayGetSize(list);
-  assert(size == pGroupResInfo->numOfDataPages);
-
-  bool done = false;
-
-  //TODO add API for release none-dirty pages
-//  SPageInfo* prev = NULL;
-
-  for (int32_t j = pGroupResInfo->pageId; j < size; ++j) {
-    SPageInfo* pi = *(SPageInfo**) taosArrayGet(list, j);
-    tFilePage* pData = getResBufPage(pResultBuf, pi->pageId);
-
-    // release previous buffer pages
-//    if (prev == NULL) {
-//      prev = pi;
-//    } else {
-//      if (prev->pageId != pi->pageId) {
-//        releaseResBufPageInfo(pResultBuf, prev);
-//        prev = pi;
-//      }
-//    }
-
-    assert(pData->num > 0 && pData->num <= pRuntimeEnv->numOfRowsPerPage && pGroupResInfo->rowId < pData->num);
-    int32_t numOfRes = (int32_t)(pData->num - pGroupResInfo->rowId);
-
-    if (numOfRes > pQuery->rec.capacity - offset) {
-      numOfCopiedRows = (int32_t)(pQuery->rec.capacity - offset);
-      pGroupResInfo->rowId += numOfCopiedRows;
-      done = true;
-    } else {
-      numOfCopiedRows = (int32_t)pData->num;
-
-      pGroupResInfo->pageId += 1;
-      pGroupResInfo->rowId = 0;
-    }
-
-    for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-      int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
-      char *  pDest = pQuery->sdata[i]->data;
-
-      memcpy(pDest + offset * bytes, pData->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage,
-          (size_t)bytes * numOfCopiedRows);
-    }
-
-    offset += numOfCopiedRows;
-    if (done) {
-      break;
-    }
-  }
-
-  assert(pQuery->rec.rows == 0);
-  pQuery->rec.rows += offset;
+  int32_t size = (int32_t) taosArrayGetSize(pGroupResInfo->pRows);
+  pQuery->rec.rows = doCopyToSData(pQInfo, pGroupResInfo->pRows->pData, (int32_t) size, &pGroupResInfo->index, TSDB_ORDER_ASC);
 }
 
 int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResultRow) {
@@ -3368,155 +3239,99 @@ int64_t getNumOfResultWindowRes(SQueryRuntimeEnv* pRuntimeEnv, SResultRow *pResu
   return 0;
 }
 
-int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
+int32_t mergeIntoGroupResultImpl(SGroupResInfo* pGroupResInfo, SArray *pTableList, SQInfo* pQInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
-  SQuery *          pQuery = pRuntimeEnv->pQuery;
+  bool ascQuery = QUERY_IS_ASC_QUERY(pRuntimeEnv->pQuery);
+
+  int32_t code = TSDB_CODE_SUCCESS;
 
-  size_t size = taosArrayGetSize(pGroup);
-  tFilePage **buffer = pQuery->sdata;
+  int32_t *posList = NULL;
+  SLoserTreeInfo *pTree = NULL;
+  STableQueryInfo **pTableQueryInfoList = NULL;
 
-  int32_t *posList = calloc(size, sizeof(int32_t));
-  STableQueryInfo **pTableList = malloc(POINTER_BYTES * size);
+  size_t size = taosArrayGetSize(pTableList);
+  if (pGroupResInfo->pRows == NULL) {
+    pGroupResInfo->pRows = taosArrayInit(100, POINTER_BYTES);
+  }
 
-  if (pTableList == NULL || posList == NULL) {
-    tfree(posList);
-    tfree(pTableList);
+  posList = calloc(size, sizeof(int32_t));
+  pTableQueryInfoList = malloc(POINTER_BYTES * size);
 
+  if (pTableQueryInfoList == NULL || posList == NULL) {
     qError("QInfo:%p failed alloc memory", pQInfo);
-    longjmp(pRuntimeEnv->env, TSDB_CODE_QRY_OUT_OF_MEMORY);
+    code = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    goto _end;
   }
 
-  // todo opt for the case of one table per group
   int32_t numOfTables = 0;
-  SIDList pageList = NULL;
-  int32_t tid = -1;
-
   for (int32_t i = 0; i < size; ++i) {
-    STableQueryInfo *item = taosArrayGetP(pGroup, i);
-
-    SIDList list = getDataBufPagesIdList(pRuntimeEnv->pResultBuf, TSDB_TABLEID(item->pTable)->tid);
-    if (taosArrayGetSize(list) > 0 && item->windowResInfo.size > 0) {
-      pTableList[numOfTables++] = item;
-      tid = TSDB_TABLEID(item->pTable)->tid;
-      pageList = list;
+    STableQueryInfo *item = taosArrayGetP(pTableList, i);
+    if (item->windowResInfo.size > 0) {
+      pTableQueryInfoList[numOfTables++] = item;
     }
   }
 
   // there is no data in current group
+  // no need to merge results since only one table in each group
   if (numOfTables == 0) {
-    tfree(posList);
-    tfree(pTableList);
-    return 0;
-  } else if (numOfTables == 1) { // no need to merge results since only one table in each group
-    tfree(posList);
-    tfree(pTableList);
-
-    SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
-
-    pGroupResInfo->numOfDataPages = (int32_t)taosArrayGetSize(pageList);
-    pGroupResInfo->groupId = tid;
-    pGroupResInfo->pageId = 0;
-    pGroupResInfo->rowId = 0;
-
-    return pGroupResInfo->numOfDataPages;
+    goto _end;
   }
 
-  SCompSupporter cs = {pTableList, posList, pQInfo};
-
-  SLoserTreeInfo *pTree = NULL;
-  tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn);
-
-  SResultRow* pRow = getNewResultRow(pRuntimeEnv->pool);
-  resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow);
+  SCompSupporter cs = {pTableQueryInfoList, posList, pRuntimeEnv->pQuery->order.order};
 
-  pQInfo->groupResInfo.groupId = getMergeResultGroupId(pQInfo->groupIndex);
+  int32_t ret = tLoserTreeCreate(&pTree, numOfTables, &cs, tableResultComparFn);
+  if (ret != TSDB_CODE_SUCCESS) {
+    code = TSDB_CODE_QRY_OUT_OF_MEMORY;
+    goto _end;
+  }
 
-  // todo add windowRes iterator
-  int64_t lastTimestamp = -1;
+  int64_t lastTimestamp = ascQuery? INT64_MIN:INT64_MAX;
   int64_t startt = taosGetTimestampMs();
 
   while (1) {
     if (isQueryKilled(pQInfo)) {
       qDebug("QInfo:%p it is already killed, abort", pQInfo);
-
-      tfree(pTableList);
-      tfree(posList);
-      tfree(pTree);
-      longjmp(pRuntimeEnv->env, TSDB_CODE_TSC_QUERY_CANCELLED);
+      code = TSDB_CODE_TSC_QUERY_CANCELLED;
+      goto _end;
     }
 
-    int32_t pos = pTree->pNode[0].index;
-
-    SResultRowInfo *pWindowResInfo = &pTableList[pos]->windowResInfo;
-    SResultRow  *pWindowRes = getResultRow(pWindowResInfo, cs.position[pos]);
-    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pWindowRes->pageId);
+    int32_t tableIndex = pTree->pNode[0].index;
 
-    char *b = getPosInResultPage(pRuntimeEnv, PRIMARYKEY_TIMESTAMP_COL_INDEX, pWindowRes, page);
-    TSKEY ts = GET_INT64_VAL(b);
+    SResultRowInfo *pWindowResInfo = &pTableQueryInfoList[tableIndex]->windowResInfo;
+    SResultRow  *pWindowRes = getResultRow(pWindowResInfo, cs.rowIndex[tableIndex]);
 
-    assert(ts == pWindowRes->win.skey);
     int64_t num = getNumOfResultWindowRes(pRuntimeEnv, pWindowRes);
     if (num <= 0) {
-      cs.position[pos] += 1;
+      cs.rowIndex[tableIndex] += 1;
 
-      if (cs.position[pos] >= pWindowResInfo->size) {
-        cs.position[pos] = -1;
-
-        // all input sources are exhausted
-        if (--numOfTables == 0) {
+      if (cs.rowIndex[tableIndex] >= pWindowResInfo->size) {
+        cs.rowIndex[tableIndex] = -1;
+        if (--numOfTables == 0) { // all input sources are exhausted
           break;
         }
       }
     } else {
-      if (ts == lastTimestamp) {  // merge with the last one
-        doMerge(pRuntimeEnv, ts, pWindowRes, true);
-      } else {  // copy data to disk buffer
-        if (buffer[0]->num == pQuery->rec.capacity) {
-          if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) {
-            return -1;
-          }
-
-          resetMergeResultBuf(pRuntimeEnv, pRuntimeEnv->pCtx, pRow);
-        }
+      assert((pWindowRes->win.skey >= lastTimestamp && ascQuery) || (pWindowRes->win.skey <= lastTimestamp && !ascQuery));
 
-        doMerge(pRuntimeEnv, ts, pWindowRes, false);
-        buffer[0]->num += 1;
+      if (pWindowRes->win.skey != lastTimestamp) {
+        taosArrayPush(pGroupResInfo->pRows, &pWindowRes);
+        pWindowRes->numOfRows = (uint32_t) num;
       }
 
-      lastTimestamp = ts;
+      lastTimestamp = pWindowRes->win.skey;
 
-      // move to the next element of current entry
-      int32_t currentPageId = pWindowRes->pageId;
-
-      cs.position[pos] += 1;
-      if (cs.position[pos] >= pWindowResInfo->size) {
-        cs.position[pos] = -1;
+      // move to the next row of current entry
+      if ((++cs.rowIndex[tableIndex]) >= pWindowResInfo->size) {
+        cs.rowIndex[tableIndex] = -1;
 
         // all input sources are exhausted
-        if (--numOfTables == 0) {
+        if ((--numOfTables) == 0) {
           break;
         }
-      } else {
-        // current page is not needed anymore
-        SResultRow  *pNextWindowRes = getResultRow(pWindowResInfo, cs.position[pos]);
-        if (pNextWindowRes->pageId != currentPageId) {
-          releaseResBufPage(pRuntimeEnv->pResultBuf, page);
-        }
       }
     }
 
-    tLoserTreeAdjust(pTree, pos + pTree->numOfEntries);
-  }
-
-  if (buffer[0]->num != 0) {  // there are data in buffer
-    if (flushFromResultBuf(pRuntimeEnv, &pQInfo->groupResInfo) != TSDB_CODE_SUCCESS) {
-      qError("QInfo:%p failed to flush data into temp file, abort query", pQInfo);
-
-      tfree(pTree);
-      tfree(pTableList);
-      tfree(posList);
-      return -1;
-    }
+    tLoserTreeAdjust(pTree, tableIndex + pTree->numOfEntries);
   }
 
   int64_t endt = taosGetTimestampMs();
@@ -3527,65 +3342,16 @@ int32_t mergeIntoGroupResultImpl(SQInfo *pQInfo, SArray *pGroup) {
 
   qDebug("QInfo:%p result merge completed for group:%d, elapsed time:%" PRId64 " ms", pQInfo, pQInfo->groupIndex, endt - startt);
 
-  tfree(pTableList);
+  _end:
+  tfree(pTableQueryInfoList);
   tfree(posList);
   tfree(pTree);
 
-//  tfree(pResultInfo);
-//  tfree(buf);
-
-  return pQInfo->groupResInfo.numOfDataPages;
-}
-
-int32_t flushFromResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SGroupResInfo* pGroupResInfo) {
-  SQuery *pQuery = pRuntimeEnv->pQuery;
-
-  SDiskbasedResultBuf *pResultBuf = pRuntimeEnv->pResultBuf;
-
-  // the base value for group result, since the maximum number of table for each vnode will not exceed 100,000.
-  int32_t pageId = -1;
-  int32_t capacity = pResultBuf->numOfRowsPerPage;
-
-  int32_t remain = (int32_t) pQuery->sdata[0]->num;
-  int32_t offset = 0;
-
-  while (remain > 0) {
-    int32_t rows = (remain > capacity)? capacity:remain;
-    assert(rows > 0);
-
-    // get the output buffer page
-    tFilePage *buf = getNewDataBuf(pResultBuf, pGroupResInfo->groupId, &pageId);
-    buf->num = rows;
-
-    // pagewisely copy to dest buffer
-    for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
-      int32_t bytes = pRuntimeEnv->pCtx[i].outputBytes;
-
-      char* output = buf->data + pRuntimeEnv->offset[i] * pRuntimeEnv->numOfRowsPerPage;
-      char* src = ((char *) pQuery->sdata[i]->data) + offset * bytes;
-      memcpy(output, src, (size_t)(buf->num * bytes));
-    }
-
-    offset += rows;
-    remain -= rows;
-
-    pGroupResInfo->numOfDataPages += 1;
+  if (code != TSDB_CODE_SUCCESS) {
+    longjmp(pRuntimeEnv->env, code);
   }
 
-  return TSDB_CODE_SUCCESS;
-}
-
-void resetMergeResultBuf(SQueryRuntimeEnv* pRuntimeEnv, SQLFunctionCtx *pCtx, SResultRow *pRow) {
-  SQuery* pQuery = pRuntimeEnv->pQuery;
-
-  for (int32_t k = 0; k < pQuery->numOfOutput; ++k) {
-    pCtx[k].aOutputBuf = pQuery->sdata[k]->data - pCtx[k].outputBytes;
-    pCtx[k].size = 1;
-    pCtx[k].startOffset = 0;
-    pCtx[k].resultInfo = getResultCell(pRuntimeEnv, pRow, k);
-
-    pQuery->sdata[k]->num = 0;
-  }
+  return code;
 }
 
 static void updateTableQueryInfoForReverseScan(SQuery *pQuery, STableQueryInfo *pTableQueryInfo) {
@@ -3706,7 +3472,7 @@ void resetDefaultResInfoOutputBuf(SQueryRuntimeEnv *pRuntimeEnv) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
 
   int32_t tid = 0;
-  int64_t uid = getResultInfoUId(pRuntimeEnv);
+  int64_t uid = 0;
   SResultRow* pRow = doPrepareResultRowFromKey(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (char *)&tid, sizeof(tid), true, uid);
 
   for (int32_t i = 0; i < pQuery->numOfOutput; ++i) {
@@ -3838,12 +3604,8 @@ bool needScanDataBlocksAgain(SQueryRuntimeEnv *pRuntimeEnv) {
 
     for (int32_t i = 0; i < pWindowResInfo->size; ++i) {
       SResultRow *pResult = getResultRow(pWindowResInfo, i);
-      if (!pResult->closed) {
-        continue;
-      }
 
       setResultOutputBuf(pRuntimeEnv, pResult);
-
       for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
         int16_t functId = pQuery->pExpr1[j].base.functionId;
         if (functId == TSDB_FUNC_TS) {
@@ -4142,12 +3904,10 @@ void setExecutionContext(SQInfo *pQInfo, int32_t groupIndex, TSKEY nextKey) {
     return;
   }
 
-  uint64_t uid = getResultInfoUId(pRuntimeEnv);
+  int64_t uid = 0;
   SResultRow *pResultRow = doPrepareResultRowFromKey(pRuntimeEnv, pWindowResInfo, (char *)&groupIndex,
       sizeof(groupIndex), true, uid);
-  if (pResultRow == NULL) {
-    return;
-  }
+  assert (pResultRow != NULL);
 
   /*
    * not assign result buffer yet, add new result buffer
@@ -4293,7 +4053,7 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
     /**
      * In handling the both ascending and descending order super table query, we need to find the first qualified
      * timestamp of this table, and then set the first qualified start timestamp.
-     * In ascending query, key is the first qualified timestamp. However, in the descending order query, additional
+     * In ascending query, the key is the first qualified timestamp. However, in the descending order query, additional
      * operations involve.
      */
     STimeWindow     w = TSWINDOW_INITIALIZER;
@@ -4302,7 +4062,6 @@ void setIntervalQueryRange(SQInfo *pQInfo, TSKEY key) {
     TSKEY sk = MIN(win.skey, win.ekey);
     TSKEY ek = MAX(win.skey, win.ekey);
     getAlignQueryTimeWindow(pQuery, win.skey, sk, ek, &w);
-    pWindowResInfo->startTime = pTableQueryInfo->win.skey;  // windowSKey may be 0 in case of 1970 timestamp
 
     if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
       if (!QUERY_IS_ASC_QUERY(pQuery)) {
@@ -4341,36 +4100,33 @@ bool needPrimaryTimestampCol(SQuery *pQuery, SDataBlockInfo *pDataBlockInfo) {
   return loadPrimaryTS;
 }
 
-static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_t orderType) {
+static int32_t doCopyToSData(SQInfo *pQInfo, SResultRow **pRows, int32_t numOfRows, int32_t *index, int32_t orderType) {
   SQueryRuntimeEnv *pRuntimeEnv = &pQInfo->runtimeEnv;
   SQuery *          pQuery = pRuntimeEnv->pQuery;
 
   int32_t numOfResult = 0;
-  int32_t startIdx = 0;
+  int32_t start = 0;
   int32_t step = -1;
 
   qDebug("QInfo:%p start to copy data from windowResInfo to query buf", pQInfo);
-  int32_t totalSet = numOfClosedResultRows(pResultInfo);
-  SResultRow** result = pResultInfo->pResult;
-
   if (orderType == TSDB_ORDER_ASC) {
-    startIdx = pQInfo->groupIndex;
+    start = (*index);
     step = 1;
   } else {  // desc order copy all data
-    startIdx = totalSet - pQInfo->groupIndex - 1;
+    start = numOfRows - (*index) - 1;
     step = -1;
   }
 
   SGroupResInfo* pGroupResInfo = &pQInfo->groupResInfo;
 
-  for (int32_t i = startIdx; (i < totalSet) && (i >= 0); i += step) {
-    if (result[i]->numOfRows == 0) {
-      pQInfo->groupIndex += 1;
+  for (int32_t i = start; (i < numOfRows) && (i >= 0); i += step) {
+    if (pRows[i]->numOfRows == 0) {
+      (*index) += 1;
       pGroupResInfo->rowId = 0;
       continue;
     }
 
-    int32_t numOfRowsToCopy = result[i]->numOfRows - pGroupResInfo->rowId;
+    int32_t numOfRowsToCopy = pRows[i]->numOfRows - pGroupResInfo->rowId;
     int32_t oldOffset = pGroupResInfo->rowId;
 
     /*
@@ -4382,16 +4138,16 @@ static int32_t doCopyToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo, int32_
       pGroupResInfo->rowId += numOfRowsToCopy;
     } else {
       pGroupResInfo->rowId = 0;
-      pQInfo->groupIndex += 1;
+      (*index) += 1;
     }
 
-    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, result[i]->pageId);
+    tFilePage *page = getResBufPage(pRuntimeEnv->pResultBuf, pRows[i]->pageId);
 
     for (int32_t j = 0; j < pQuery->numOfOutput; ++j) {
       int32_t size = pRuntimeEnv->pCtx[j].outputBytes;
 
       char *out = pQuery->sdata[j]->data + numOfResult * size;
-      char *in = getPosInResultPage(pRuntimeEnv, j, result[i], page);
+      char *in  = getPosInResultPage(pRuntimeEnv, j, pRows[i], page);
       memcpy(out, in + oldOffset * size, size * numOfRowsToCopy);
     }
 
@@ -4422,10 +4178,9 @@ void copyFromWindowResToSData(SQInfo *pQInfo, SResultRowInfo *pResultInfo) {
   SQuery *pQuery = pQInfo->runtimeEnv.pQuery;
 
   int32_t orderType = (pQuery->pGroupbyExpr != NULL) ? pQuery->pGroupbyExpr->orderType : TSDB_ORDER_ASC;
-  int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo, orderType);
+  int32_t numOfResult = doCopyToSData(pQInfo, pResultInfo->pResult, pResultInfo->size, &pQInfo->groupIndex, orderType);
 
   pQuery->rec.rows += numOfResult;
-
   assert(pQuery->rec.rows <= pQuery->rec.capacity);
 }
 
@@ -4457,25 +4212,17 @@ static void stableApplyFunctionsOnBlock(SQueryRuntimeEnv *pRuntimeEnv, SDataBloc
   SQuery *         pQuery = pRuntimeEnv->pQuery;
   STableQueryInfo* pTableQueryInfo = pQuery->current;
 
-  SResultRowInfo * pWindowResInfo = &pTableQueryInfo->windowResInfo;
+  SResultRowInfo * pResultRowInfo = &pTableQueryInfo->windowResInfo;
   pQuery->pos = QUERY_IS_ASC_QUERY(pQuery)? 0 : pDataBlockInfo->rows - 1;
 
   if (pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL || pRuntimeEnv->groupbyNormalCol) {
-    rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, pDataBlock);
+    rowwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, pDataBlock);
   } else {
-    blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pWindowResInfo, searchFn, pDataBlock);
+    blockwiseApplyFunctions(pRuntimeEnv, pStatis, pDataBlockInfo, pResultRowInfo, searchFn, pDataBlock);
   }
 
   if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
-    bool ascQuery = QUERY_IS_ASC_QUERY(pQuery);
-
-    // TODO refactor
-    if ((pTableQueryInfo->lastKey >= pTableQueryInfo->win.ekey && ascQuery) || (pTableQueryInfo->lastKey <= pTableQueryInfo->win.ekey && (!ascQuery))) {
-      closeAllResultRows(pWindowResInfo);
-      pWindowResInfo->curIndex = pWindowResInfo->size - 1;
-    } else {
-      updateResultRowCurrentIndex(pWindowResInfo, pTableQueryInfo->lastKey, ascQuery);
-    }
+    updateResultRowIndex(pResultRowInfo, pTableQueryInfo, QUERY_IS_ASC_QUERY(pQuery));
   }
 }
 
@@ -4809,13 +4556,10 @@ static bool skipTimeInterval(SQueryRuntimeEnv *pRuntimeEnv, TSKEY* start) {
     if (QUERY_IS_ASC_QUERY(pQuery)) {
       if (pWindowResInfo->prevSKey == TSKEY_INITIAL_VAL) {
         getAlignQueryTimeWindow(pQuery, blockInfo.window.skey, blockInfo.window.skey, pQuery->window.ekey, &w);
-        pWindowResInfo->startTime = w.skey;
         pWindowResInfo->prevSKey = w.skey;
       }
     } else {
       getAlignQueryTimeWindow(pQuery, blockInfo.window.ekey, pQuery->window.ekey, blockInfo.window.ekey, &w);
-
-      pWindowResInfo->startTime = pQuery->window.skey;
       pWindowResInfo->prevSKey = w.skey;
     }
 
@@ -5766,6 +5510,7 @@ static void doRestoreContext(SQInfo *pQInfo) {
   SQuery *          pQuery = pRuntimeEnv->pQuery;
 
   SWAP(pQuery->window.skey, pQuery->window.ekey, TSKEY);
+  SWITCH_ORDER(pQuery->order.order);
 
   if (pRuntimeEnv->pTsBuf != NULL) {
     SWITCH_ORDER(pRuntimeEnv->pTsBuf->cur.order);
@@ -5805,9 +5550,6 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
      */
     if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
       copyResToQueryResultBuf(pQInfo, pQuery);
-#ifdef _DEBUG_VIEW
-      displayInterResult(pQuery->sdata, pRuntimeEnv, pQuery->sdata[0]->num);
-#endif
     } else {
       copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo);
     }
@@ -5852,7 +5594,7 @@ static void multiTableQueryProcess(SQInfo *pQInfo) {
   }
 
   if (QUERY_IS_INTERVAL_QUERY(pQuery) || isSumAvgRateQuery(pQuery)) {
-    if (mergeIntoGroupResult(pQInfo) == TSDB_CODE_SUCCESS) {
+    if (mergeGroupResult(pQInfo) == TSDB_CODE_SUCCESS) {
       copyResToQueryResultBuf(pQInfo, pQuery);
 
 #ifdef _DEBUG_VIEW
@@ -6024,33 +5766,6 @@ static void tableMultiOutputProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo)
   }
 }
 
-static void tableIntervalProcessImpl(SQueryRuntimeEnv *pRuntimeEnv, TSKEY start) {
-  SQuery *pQuery = pRuntimeEnv->pQuery;
-
-  while (1) {
-    scanOneTableDataBlocks(pRuntimeEnv, start);
-
-    assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED));
-    finalizeQueryResult(pRuntimeEnv);
-
-    // here we can ignore the records in case of no interpolation
-    // todo handle offset, in case of top/bottom interval query
-    if ((pQuery->numOfFilterCols > 0 || pRuntimeEnv->pTsBuf != NULL) && pQuery->limit.offset > 0 &&
-        pQuery->fillType == TSDB_FILL_NONE) {
-      // maxOutput <= 0, means current query does not generate any results
-      int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo);
-
-      int32_t c = (int32_t)(MIN(numOfClosed, pQuery->limit.offset));
-      popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, c);
-      pQuery->limit.offset -= c;
-    }
-
-    if (Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED | QUERY_RESBUF_FULL)) {
-      break;
-    }
-  }
-}
-
 // handle time interval query on table
 static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
   SQueryRuntimeEnv *pRuntimeEnv = &(pQInfo->runtimeEnv);
@@ -6058,7 +5773,6 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
   SQuery *pQuery = pRuntimeEnv->pQuery;
   pQuery->current = pTableInfo;
 
-  int32_t numOfFilled = 0;
   TSKEY newStartKey = TSKEY_INITIAL_VAL;
 
   // skip blocks without load the actual data block from file if no filter condition present
@@ -6070,59 +5784,39 @@ static void tableIntervalProcess(SQInfo *pQInfo, STableQueryInfo* pTableInfo) {
     }
   }
 
-  while (1) {
-    tableIntervalProcessImpl(pRuntimeEnv, newStartKey);
+  scanOneTableDataBlocks(pRuntimeEnv, newStartKey);
+  assert(!Q_STATUS_EQUAL(pQuery->status, QUERY_NOT_COMPLETED));
 
-    if (QUERY_IS_INTERVAL_QUERY(pQuery)) {
-      pQInfo->groupIndex = 0;  // always start from 0
+  finalizeQueryResult(pRuntimeEnv);
+
+  // skip offset result rows
   pQuery->rec.rows = 0;
-      copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo);
 
-      popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQInfo->groupIndex);
+  if (pQuery->fillType == TSDB_FILL_NONE) {
+    // all data scanned, the group by normal column can return
+    int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo);
+    if (pQuery->limit.offset > numOfClosed) {
+      return;
     }
 
-    // no result generated, abort
-    if (pQuery->rec.rows == 0 || pRuntimeEnv->groupbyNormalCol) {
-      break;
-    }
+    pQInfo->groupIndex = (int32_t) pQuery->limit.offset;
 
+    copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo);
     doSecondaryArithmeticProcess(pQuery);
 
-    // the offset is handled at prepare stage if no interpolation involved
-    if (pQuery->fillType == TSDB_FILL_NONE) {
     limitResults(pRuntimeEnv);
-      break;
   } else {
-      taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey);
-      taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage**) pQuery->sdata);
-      numOfFilled = 0;
-
-      pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled);
-      if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
-        limitResults(pRuntimeEnv);
-        break;
-      }
 
-      // no result generated yet, continue retrieve data
-      pQuery->rec.rows = 0;
-    }
-  }
-
-  // all data scanned, the group by normal column can return
-  if (pRuntimeEnv->groupbyNormalCol) {  // todo refactor with merge interval time result
-    // maxOutput <= 0, means current query does not generate any results
-    int32_t numOfClosed = numOfClosedResultRows(&pRuntimeEnv->windowResInfo);
+    copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo);
+    doSecondaryArithmeticProcess(pQuery);
 
-    if ((pQuery->limit.offset > 0 && pQuery->limit.offset < numOfClosed) || pQuery->limit.offset == 0) {
-      // skip offset result rows
-      popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, (int32_t) pQuery->limit.offset);
+    taosFillSetStartInfo(pRuntimeEnv->pFillInfo, (int32_t)pQuery->rec.rows, pQuery->window.ekey);
+    taosFillCopyInputDataFromFilePage(pRuntimeEnv->pFillInfo, (const tFilePage **)pQuery->sdata);
 
-      pQuery->rec.rows   = 0;
-      pQInfo->groupIndex = 0;
-      copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo);
-      popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQInfo->groupIndex);
+    int32_t numOfFilled = 0;
+    pQuery->rec.rows = doFillGapsInResults(pRuntimeEnv, (tFilePage **)pQuery->sdata, &numOfFilled);
 
-      doSecondaryArithmeticProcess(pQuery);
+    if (pQuery->rec.rows > 0 || Q_STATUS_EQUAL(pQuery->status, QUERY_COMPLETED)) {
       limitResults(pRuntimeEnv);
     }
   }
@@ -6133,7 +5827,6 @@ static void tableQueryImpl(SQInfo *pQInfo) {
   SQuery *          pQuery = pRuntimeEnv->pQuery;
 
   if (queryHasRemainResForTableQuery(pRuntimeEnv)) {
-
     if (pQuery->fillType != TSDB_FILL_NONE) {
       /*
        * There are remain results that are not returned due to result interpolation
@@ -6150,25 +5843,25 @@ static void tableQueryImpl(SQInfo *pQInfo) {
       return;
     } else {
       pQuery->rec.rows = 0;
-      pQInfo->groupIndex = 0;  // always start from 0
+      assert(pRuntimeEnv->windowResInfo.size > 0);
 
-      if (pRuntimeEnv->windowResInfo.size > 0) {
+      if (pQInfo->groupIndex < pRuntimeEnv->windowResInfo.size) {
         copyFromWindowResToSData(pQInfo, &pRuntimeEnv->windowResInfo);
-        popFrontResultRow(pRuntimeEnv, &pRuntimeEnv->windowResInfo, pQInfo->groupIndex);
+      }
 
       if (pQuery->rec.rows > 0) {
-          qDebug("QInfo:%p %"PRId64" rows returned from group results, total:%"PRId64"", pQInfo, pQuery->rec.rows, pQuery->rec.total);
+        qDebug("QInfo:%p %" PRId64 " rows returned from group results, total:%" PRId64 "", pQInfo, pQuery->rec.rows,
+               pQuery->rec.total);
+      }
 
       // there are not data remains
-          if (pRuntimeEnv->windowResInfo.size <= 0) {
-            qDebug("QInfo:%p query over, %"PRId64" rows are returned", pQInfo, pQuery->rec.total);
+      if (pQuery->rec.rows <= 0 || pRuntimeEnv->windowResInfo.size <= pQInfo->groupIndex) {
+        qDebug("QInfo:%p query over, %" PRId64 " rows are returned", pQInfo, pQuery->rec.total);
       }
 
       return;
     }
   }
-    }
-  }
 
   // number of points returned during this query
   pQuery->rec.rows = 0;
@@ -7284,6 +6977,8 @@ static void freeQInfo(SQInfo *pQInfo) {
   tsdbDestroyTableGroup(&pQInfo->tableGroupInfo);
   taosHashCleanup(pQInfo->arrTableIdInfo);
 
+  taosArrayDestroy(pQInfo->groupResInfo.pRows);
+
   pQInfo->signature = 0;
 
   qDebug("QInfo:%p QInfo is freed", pQInfo);
@@ -7703,7 +7398,7 @@ int32_t qDumpRetrieveResult(qinfo_t qinfo, SRetrieveTableRsp **pRsp, int32_t *co
     (*pRsp)->completed = 1;  // notify no more result to client
   } else {
     *continueExec = true;
-    qDebug("QInfo:%p has more results waits for client retrieve", pQInfo);
+    qDebug("QInfo:%p has more results to retrieve", pQInfo);
   }
 
   return pQInfo->code;
@@ -7981,9 +7676,9 @@ void qQueryMgmtNotifyClosed(void* pQMgmt) {
   SQueryMgmt* pQueryMgmt = pQMgmt;
   qDebug("vgId:%d, set querymgmt closed, wait for all queries cancelled", pQueryMgmt->vgId);
 
-//  pthread_mutex_lock(&pQueryMgmt->lock);
+  pthread_mutex_lock(&pQueryMgmt->lock);
   pQueryMgmt->closed = true;
-//  pthread_mutex_unlock(&pQueryMgmt->lock);
+  pthread_mutex_unlock(&pQueryMgmt->lock);
 
   taosCacheRefresh(pQueryMgmt->qinfoPool, queryMgmtKillQueryFn);
 }
@@ -8021,9 +7716,9 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {
     return NULL;
   }
 
-//  pthread_mutex_lock(&pQueryMgmt->lock);
+  pthread_mutex_lock(&pQueryMgmt->lock);
   if (pQueryMgmt->closed) {
-//    pthread_mutex_unlock(&pQueryMgmt->lock);
+    pthread_mutex_unlock(&pQueryMgmt->lock);
     qError("QInfo:%p failed to add qhandle into cache, since qMgmt is colsing", (void *)qInfo);
     terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;
     return NULL;
@@ -8031,7 +7726,7 @@ void** qRegisterQInfo(void* pMgmt, uint64_t qInfo) {
     TSDB_CACHE_PTR_TYPE handleVal = (TSDB_CACHE_PTR_TYPE) qInfo;
     void** handle = taosCachePut(pQueryMgmt->qinfoPool, &handleVal, sizeof(TSDB_CACHE_PTR_TYPE), &qInfo, sizeof(TSDB_CACHE_PTR_TYPE),
         (getMaximumIdleDurationSec()*1000));
-//    pthread_mutex_unlock(&pQueryMgmt->lock);
+    pthread_mutex_unlock(&pQueryMgmt->lock);
 
     return handle;
   }
@@ -8069,5 +7764,3 @@ void** qReleaseQInfo(void* pMgmt, void* pQInfo, bool freeHandle) {
   taosCacheRelease(pQueryMgmt->qinfoPool, pQInfo, freeHandle);
   return 0;
 }
\ No newline at end of file
-
-

src/query/src/qHistogram.c

@@ -158,8 +158,8 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
   }
 
 #if defined(USE_ARRAYLIST)
-  int32_t idx = vnodeHistobinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
-  assert(idx >= 0 && idx <= (*pHisto)->maxEntries);
+  int32_t idx = histoBinarySearch((*pHisto)->elems, (*pHisto)->numOfEntries, val);
+  assert(idx >= 0 && idx <= (*pHisto)->maxEntries && (*pHisto)->elems != NULL);
 
   if ((*pHisto)->elems[idx].val == val && idx >= 0) {
     (*pHisto)->elems[idx].num += 1;
@@ -356,7 +356,7 @@ int32_t tHistogramAdd(SHistogramInfo** pHisto, double val) {
   return 0;
 }
 
-int32_t vnodeHistobinarySearch(SHistBin* pEntry, int32_t len, double val) {
+int32_t histoBinarySearch(SHistBin* pEntry, int32_t len, double val) {
   int32_t end = len - 1;
   int32_t start = 0;
 
@@ -466,7 +466,7 @@ void tHistogramPrint(SHistogramInfo* pHisto) {
  */
 int64_t tHistogramSum(SHistogramInfo* pHisto, double v) {
 #if defined(USE_ARRAYLIST)
-  int32_t slotIdx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, v);
+  int32_t slotIdx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, v);
   if (pHisto->elems[slotIdx].val != v) {
     slotIdx -= 1;
 

src/query/src/qUtil.c

@@ -96,8 +96,6 @@ void resetResultRowInfo(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRo
   
   pResultRowInfo->curIndex = -1;
   pResultRowInfo->size = 0;
-  
-  pResultRowInfo->startTime = TSKEY_INITIAL_VAL;
   pResultRowInfo->prevSKey = TSKEY_INITIAL_VAL;
 }
 
@@ -110,7 +108,7 @@ void popFrontResultRow(SQueryRuntimeEnv *pRuntimeEnv, SResultRowInfo *pResultRow
   assert(num >= 0 && num <= numOfClosed);
 
   int16_t type = pResultRowInfo->type;
-  int64_t uid = getResultInfoUId(pRuntimeEnv);
+  int64_t uid  = 0;
 
   char    *key  = NULL;
   int16_t  bytes = -1;
@@ -181,11 +179,12 @@ void closeAllResultRows(SResultRowInfo *pResultRowInfo) {
   assert(pResultRowInfo->size >= 0 && pResultRowInfo->capacity >= pResultRowInfo->size);
   
   for (int32_t i = 0; i < pResultRowInfo->size; ++i) {
-    if (pResultRowInfo->pResult[i]->closed) {
+    SResultRow* pRow = pResultRowInfo->pResult[i];
+    if (pRow->closed) {
       continue;
     }
     
-    pResultRowInfo->pResult[i]->closed = true;
+    pRow->closed = true;
   }
 }
 
@@ -384,17 +383,3 @@ void* destroyResultRowPool(SResultRowPool* p) {
   tfree(p);
   return NULL;
 }
\ No newline at end of file
-
-uint64_t getResultInfoUId(SQueryRuntimeEnv* pRuntimeEnv) {
-  if (!pRuntimeEnv->stableQuery) {
-    return 0;  // for simple table query, the uid is always set to be 0;
-  }
-
-  SQuery* pQuery = pRuntimeEnv->pQuery;
-  if (pQuery->interval.interval == 0 || isPointInterpoQuery(pQuery) || pRuntimeEnv->groupbyNormalCol) {
-    return 0;
-  }
-
-  STableId* id = TSDB_TABLEID(pRuntimeEnv->pQuery->current->pTable);
-  return id->uid;
-}
\ No newline at end of file

src/query/tests/histogramTest.cpp

@@ -21,19 +21,19 @@ TEST(testCase, histogram_binary_search) {
     pHisto->elems[i].val = i;
   }
 
-  int32_t idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 1);
+  int32_t idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 1);
   assert(idx == 1);
 
-  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 9);
+  idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 9);
   assert(idx == 9);
 
-  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 20);
+  idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 20);
   assert(idx == 10);
 
-  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, -1);
+  idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, -1);
   assert(idx == 0);
 
-  idx = vnodeHistobinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9);
+  idx = histoBinarySearch(pHisto->elems, pHisto->numOfEntries, 3.9);
   assert(idx == 4);
 
   free(pHisto);

src/sync/inc/syncInt.h

@@ -38,7 +38,7 @@ extern "C" {
 #define SYNC_MAX_FWDS 512
 #define SYNC_FWD_TIMER 300
 #define SYNC_ROLE_TIMER 15000             // ms
-#define SYNC_CHECK_INTERVAL 1             // ms
+#define SYNC_CHECK_INTERVAL 1000          // ms
 #define SYNC_WAIT_AFTER_CHOOSE_MASTER 10  // ms
 
 #define nodeRole    pNode->peerInfo[pNode->selfIndex]->role
@@ -86,9 +86,10 @@ typedef struct SsyncPeer {
   int32_t  peerFd;          // forward FD
   int32_t  numOfRetrieves;  // number of retrieves tried
   int32_t  fileChanged;     // a flag to indicate file is changed during retrieving process
+  int32_t  refCount;
+  int64_t  rid;
   void *   timer;
   void *   pConn;
-  int32_t  refCount;  // reference count
   struct   SSyncNode *pSyncNode;
 } SSyncPeer;
 
@@ -98,6 +99,7 @@ typedef struct SSyncNode {
   int8_t       quorum;
   int8_t       selfIndex;
   uint32_t     vgId;
+  int32_t      refCount;
   int64_t      rid;
   SSyncPeer *  peerInfo[TAOS_SYNC_MAX_REPLICA + 1];  // extra one for arbitrator
   SSyncPeer *  pMaster;
@@ -121,13 +123,13 @@ extern int32_t tsSyncNum;
 extern char    tsNodeFqdn[TSDB_FQDN_LEN];
 extern char *  syncStatus[];
 
-void *syncRetrieveData(void *param);
-void *syncRestoreData(void *param);
+void *     syncRetrieveData(void *param);
+void *     syncRestoreData(void *param);
 int32_t    syncSaveIntoBuffer(SSyncPeer *pPeer, SWalHead *pHead);
 void       syncRestartConnection(SSyncPeer *pPeer);
 void       syncBroadcastStatus(SSyncNode *pNode);
-void    syncAddPeerRef(SSyncPeer *pPeer);
-int32_t syncDecPeerRef(SSyncPeer *pPeer);
+SSyncPeer *syncAcquirePeer(int64_t rid);
+void       syncReleasePeer(SSyncPeer *pPeer);
 
 #ifdef __cplusplus
 }

src/sync/inc/syncTcp.h

@@ -25,14 +25,14 @@ typedef struct {
   uint32_t  serverIp;
   int16_t   port;
   int32_t   bufferSize;
-  void    (*processBrokenLink)(void *ahandle);
-  int32_t (*processIncomingMsg)(void *ahandle, void *buffer);
+  void    (*processBrokenLink)(int64_t handleId);
+  int32_t (*processIncomingMsg)(int64_t handleId, void *buffer);
   void    (*processIncomingConn)(int32_t fd, uint32_t ip);
 } SPoolInfo;
 
 void *syncOpenTcpThreadPool(SPoolInfo *pInfo);
 void  syncCloseTcpThreadPool(void *);
-void *syncAllocateTcpConn(void *, void *ahandle, int32_t connFd);
+void *syncAllocateTcpConn(void *, int64_t rid, int32_t connFd);
 void  syncFreeTcpConn(void *);
 
 #ifdef __cplusplus

src/sync/src/syncArbitrator.c

@@ -29,8 +29,8 @@
 
 static void    arbSignalHandler(int32_t signum, siginfo_t *sigInfo, void *context);
 static void    arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
-static void    arbProcessBrokenLink(void *param);
-static int32_t arbProcessPeerMsg(void *param, void *buffer);
+static void    arbProcessBrokenLink(int64_t rid);
+static int32_t arbProcessPeerMsg(int64_t rid, void *buffer);
 static tsem_t  tsArbSem;
 static void *  tsArbTcpPool;
 
@@ -138,20 +138,20 @@ static void arbProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
 
   sDebug("%s, arbitrator request is accepted", pNode->id);
   pNode->nodeFd = connFd;
-  pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, pNode, connFd);
+  pNode->pConn = syncAllocateTcpConn(tsArbTcpPool, (int64_t)pNode, connFd);
 
   return;
 }
 
-static void arbProcessBrokenLink(void *param) {
-  SNodeConn *pNode = param;
+static void arbProcessBrokenLink(int64_t rid) {
+  SNodeConn *pNode = (SNodeConn *)rid;
 
   sDebug("%s, TCP link is broken since %s, close connection", pNode->id, strerror(errno));
   tfree(pNode);
 }
 
-static int32_t arbProcessPeerMsg(void *param, void *buffer) {
-  SNodeConn *pNode = param;
+static int32_t arbProcessPeerMsg(int64_t rid, void *buffer) {
+  SNodeConn *pNode = (SNodeConn *)rid;
   SSyncHead  head;
   int32_t    bytes = 0;
   char *     cont = (char *)buffer;

src/sync/src/syncMain.c

@@ -35,19 +35,21 @@ char    tsNodeFqdn[TSDB_FQDN_LEN] = {0};
 static void *  tsTcpPool = NULL;
 static void *  tsSyncTmrCtrl = NULL;
 static void *  tsVgIdHash = NULL;
-static int32_t tsSyncRefId = -1;
+static int32_t tsNodeRefId = -1;
+static int32_t tsPeerRefId = -1;
 
 // local functions
 static void    syncProcessSyncRequest(char *pMsg, SSyncPeer *pPeer);
 static void    syncRecoverFromMaster(SSyncPeer *pPeer);
 static void    syncCheckPeerConnection(void *param, void *tmrId);
-static void    syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId);
-static void    syncProcessBrokenLink(void *param);
-static int32_t syncProcessPeerMsg(void *param, void *buffer);
+static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId);
+static void    syncProcessBrokenLink(int64_t rid);
+static int32_t syncProcessPeerMsg(int64_t rid, void *buffer);
 static void    syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp);
 static void    syncRemovePeer(SSyncPeer *pPeer);
 static void    syncAddArbitrator(SSyncNode *pNode);
 static void    syncFreeNode(void *);
+static void    syncFreePeer(void *);
 static void    syncRemoveConfirmedFwdInfo(SSyncNode *pNode);
 static void    syncMonitorFwdInfos(void *param, void *tmrId);
 static void    syncMonitorNodeRole(void *param, void *tmrId);
@@ -55,7 +57,12 @@ static void    syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t c
 static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle);
 static void    syncRestartPeer(SSyncPeer *pPeer);
 static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtyp);
+
 static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo);
+static void       syncStartCheckPeerConn(SSyncPeer *pPeer);
+static void       syncStopCheckPeerConn(SSyncPeer *pPeer);
+static SSyncNode *syncAcquireNode(int64_t rid);
+static void       syncReleaseNode(SSyncNode *pNode);
 
 char* syncRole[] = {
   "offline",
@@ -87,29 +94,34 @@ int32_t syncInit() {
   tsTcpPool = syncOpenTcpThreadPool(&info);
   if (tsTcpPool == NULL) {
     sError("failed to init tcpPool");
+    syncCleanUp();
     return -1;
   }
 
   tsSyncTmrCtrl = taosTmrInit(1000, 50, 10000, "SYNC");
   if (tsSyncTmrCtrl == NULL) {
     sError("failed to init tmrCtrl");
-    syncCloseTcpThreadPool(tsTcpPool);
-    tsTcpPool = NULL;
+    syncCleanUp();
     return -1;
   }
 
   tsVgIdHash = taosHashInit(TSDB_MIN_VNODES, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK);
   if (tsVgIdHash == NULL) {
     sError("failed to init vgIdHash");
-    taosTmrCleanUp(tsSyncTmrCtrl);
-    syncCloseTcpThreadPool(tsTcpPool);
-    tsTcpPool = NULL;
-    tsSyncTmrCtrl = NULL;
+    syncCleanUp();
+    return -1;
+  }
+
+  tsNodeRefId = taosOpenRef(200, syncFreeNode);
+  if (tsNodeRefId < 0) {
+    sError("failed to init node ref");
+    syncCleanUp();
     return -1;
   }
 
-  tsSyncRefId = taosOpenRef(200, syncFreeNode);
-  if (tsSyncRefId < 0) {
+  tsPeerRefId = taosOpenRef(1000, syncFreePeer);
+  if (tsPeerRefId < 0) {
+    sError("failed to init peer ref");
     syncCleanUp();
     return -1;
   }
@@ -121,12 +133,12 @@ int32_t syncInit() {
 }
 
 void syncCleanUp() {
-  if (tsTcpPool) {
+  if (tsTcpPool != NULL) {
     syncCloseTcpThreadPool(tsTcpPool);
     tsTcpPool = NULL;
   }
 
-  if (tsSyncTmrCtrl) {
+  if (tsSyncTmrCtrl != NULL) {
     taosTmrCleanUp(tsSyncTmrCtrl);
     tsSyncTmrCtrl = NULL;
   }
@@ -136,8 +148,15 @@ void syncCleanUp() {
     tsVgIdHash = NULL;
   }
 
-  taosCloseRef(tsSyncRefId);
-  tsSyncRefId = -1;
+  if (tsNodeRefId != -1) {
+    taosCloseRef(tsNodeRefId);
+    tsNodeRefId = -1;
+  }
+
+  if (tsPeerRefId != -1) {
+    taosCloseRef(tsPeerRefId);
+    tsPeerRefId = -1;
+  }
 
   sInfo("sync module is cleaned up");
 }
@@ -170,7 +189,8 @@ int64_t syncStart(const SSyncInfo *pInfo) {
   pNode->quorum = pCfg->quorum;
   if (pNode->quorum > pNode->replica) pNode->quorum = pNode->replica;
 
-  pNode->rid = taosAddRef(tsSyncRefId, pNode);
+  pNode->refCount = 1;
+  pNode->rid = taosAddRef(tsNodeRefId, pNode);
   if (pNode->rid < 0) {
     syncFreeNode(pNode);
     return -1;
@@ -232,13 +252,18 @@ int64_t syncStart(const SSyncInfo *pInfo) {
     (*pNode->notifyRole)(pNode->vgId, nodeRole);
   }
 
+  syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]);  // arb
+  for (int32_t index = 0; index < pNode->replica; ++index) {
+    syncStartCheckPeerConn(pNode->peerInfo[index]);
+  }
+
   return pNode->rid;
 }
 
 void syncStop(int64_t rid) {
   SSyncPeer *pPeer;
 
-  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  SSyncNode *pNode = syncAcquireNode(rid);
   if (pNode == NULL) return;
 
   sInfo("vgId:%d, cleanup sync", pNode->vgId);
@@ -259,14 +284,14 @@ void syncStop(int64_t rid) {
 
   pthread_mutex_unlock(&pNode->mutex);
 
-  taosReleaseRef(tsSyncRefId, rid);
-  taosRemoveRef(tsSyncRefId, rid);
+  syncReleaseNode(pNode);
+  taosRemoveRef(tsNodeRefId, rid);
 }
 
 int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
   int32_t i, j;
 
-  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  SSyncNode *pNode = syncAcquireNode(rid);
   if (pNode == NULL) return TSDB_CODE_SYN_INVALID_CONFIG;
 
   sInfo("vgId:%d, reconfig, role:%s replica:%d old:%d", pNode->vgId, syncRole[nodeRole], pNewCfg->replica,
@@ -274,6 +299,11 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
 
   pthread_mutex_lock(&pNode->mutex);
 
+  syncStopCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]);  // arb
+  for (int32_t index = 0; index < pNode->replica; ++index) {
+    syncStopCheckPeerConn(pNode->peerInfo[index]);
+  }
+
   for (i = 0; i < pNode->replica; ++i) {
     for (j = 0; j < pNewCfg->replica; ++j) {
       if ((strcmp(pNode->peerInfo[i]->fqdn, pNewCfg->nodeInfo[j].nodeFqdn) == 0) &&
@@ -330,28 +360,32 @@ int32_t syncReconfig(int64_t rid, const SSyncCfg *pNewCfg) {
     (*pNode->notifyRole)(pNode->vgId, nodeRole);
   }
 
+  syncStartCheckPeerConn(pNode->peerInfo[TAOS_SYNC_MAX_REPLICA]);  // arb
+  for (int32_t index = 0; index < pNode->replica; ++index) {
+    syncStartCheckPeerConn(pNode->peerInfo[index]);
+  }
+
   pthread_mutex_unlock(&pNode->mutex);
 
   sInfo("vgId:%d, %d replicas are configured, quorum:%d", pNode->vgId, pNode->replica, pNode->quorum);
   syncBroadcastStatus(pNode);
 
-  taosReleaseRef(tsSyncRefId, rid);
+  syncReleaseNode(pNode);
   return 0;
 }
 
 int32_t syncForwardToPeer(int64_t rid, void *data, void *mhandle, int32_t qtype) {
-  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  SSyncNode *pNode = syncAcquireNode(rid);
   if (pNode == NULL) return 0;
 
   int32_t code = syncForwardToPeerImpl(pNode, data, mhandle, qtype);
 
-  taosReleaseRef(tsSyncRefId, rid);
-
+  syncReleaseNode(pNode);
   return code;
 }
 
 void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
-  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  SSyncNode *pNode = syncAcquireNode(rid);
   if (pNode == NULL) return;
 
   SSyncPeer *pPeer = pNode->pMaster;
@@ -367,14 +401,14 @@ void syncConfirmForward(int64_t rid, uint64_t version, int32_t code) {
     }
   }
 
-  taosReleaseRef(tsSyncRefId, rid);
+  syncReleaseNode(pNode);
 }
 
 #if 0
 void syncRecover(int64_t rid) {
   SSyncPeer *pPeer;
 
-  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  SSyncNode *pNode = syncAcquireNode(rid);
   if (pNode == NULL) return;
 
   // to do: add a few lines to check if recover is OK
@@ -395,12 +429,12 @@ void syncRecover(int64_t rid) {
 
   pthread_mutex_unlock(&pNode->mutex);
 
-  taosReleaseRef(tsSyncRefId, rid);
+  syncReleaseNode(pNode);
 }
 #endif
 
 int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
-  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  SSyncNode *pNode = syncAcquireNode(rid);
   if (pNode == NULL) return -1;
 
   pNodesRole->selfIndex = pNode->selfIndex;
@@ -409,8 +443,7 @@ int32_t syncGetNodesRole(int64_t rid, SNodesRole *pNodesRole) {
     pNodesRole->role[i] = pNode->peerInfo[i]->role;
   }
 
-  taosReleaseRef(tsSyncRefId, rid);
-
+  syncReleaseNode(pNode);
   return 0;
 }
 
@@ -446,24 +479,61 @@ static void syncAddArbitrator(SSyncNode *pNode) {
 static void syncFreeNode(void *param) {
   SSyncNode *pNode = param;
 
+  int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1);
+  sDebug("vgId:%d, syncnode is freed, refCount:%d", pNode->vgId, refCount);
+
   pthread_mutex_destroy(&pNode->mutex);
   tfree(pNode->pRecv);
   tfree(pNode->pSyncFwds);
   tfree(pNode);
 }
 
-void syncAddPeerRef(SSyncPeer *pPeer) { atomic_add_fetch_32(&pPeer->refCount, 1); }
+SSyncNode *syncAcquireNode(int64_t rid) {
+  SSyncNode *pNode = taosAcquireRef(tsNodeRefId, rid);
+  if (pNode == NULL) {
+    sDebug("failed to acquire syncnode from refId:%" PRId64, rid);
+  } else {
+    int32_t refCount = atomic_add_fetch_32(&pNode->refCount, 1);
+    sTrace("vgId:%d, acquire syncnode refId:%" PRId64 ", refCount:%d", pNode->vgId, rid, refCount);
+  }
+
+  return pNode;
+}
+
+void syncReleaseNode(SSyncNode *pNode) {
+  int32_t refCount = atomic_sub_fetch_32(&pNode->refCount, 1);
+  sTrace("vgId:%d, dec syncnode refId:%" PRId64 " refCount:%d", pNode->vgId, pNode->rid, refCount);
+
+  taosReleaseRef(tsNodeRefId, pNode->rid);
+}
+
+static void syncFreePeer(void *param) {
+  SSyncPeer *pPeer = param;
 
-int32_t syncDecPeerRef(SSyncPeer *pPeer) {
-  if (atomic_sub_fetch_32(&pPeer->refCount, 1) == 0) {
-    taosReleaseRef(tsSyncRefId, pPeer->pSyncNode->rid);
+  int32_t refCount = atomic_sub_fetch_32(&pPeer->refCount, 1);
+  sDebug("%s, peer is freed, refCount:%d", pPeer->id, refCount);
 
-    sDebug("%s, resource is freed", pPeer->id);
+  syncReleaseNode(pPeer->pSyncNode);
   tfree(pPeer);
-    return 0;
+}
+
+SSyncPeer *syncAcquirePeer(int64_t rid) {
+  SSyncPeer *pPeer = taosAcquireRef(tsPeerRefId, rid);
+  if (pPeer == NULL) {
+    sDebug("failed to acquire peer from refId:%" PRId64, rid);
+  } else {
+    int32_t refCount = atomic_add_fetch_32(&pPeer->refCount, 1);
+    sTrace("%s, acquire peer refId:%" PRId64 ", refCount:%d", pPeer->id, rid, refCount);
   }
 
-  return 1;
+  return pPeer;
+}
+
+void syncReleasePeer(SSyncPeer *pPeer) {
+  int32_t refCount = atomic_sub_fetch_32(&pPeer->refCount, 1);
+  sTrace("%s, dec peer refId:%" PRId64 ", refCount:%d", pPeer->id, pPeer->rid, refCount);
+
+  taosReleaseRef(tsPeerRefId, pPeer->rid);
 }
 
 static void syncClosePeerConn(SSyncPeer *pPeer) {
@@ -473,7 +543,8 @@ static void syncClosePeerConn(SSyncPeer *pPeer) {
   taosClose(pPeer->syncFd);
   if (pPeer->peerFd >= 0) {
     pPeer->peerFd = -1;
-    syncFreeTcpConn(pPeer->pConn);
+    void *pConn = pPeer->pConn;
+    if (pConn != NULL) syncFreeTcpConn(pPeer->pConn);
   }
 }
 
@@ -482,7 +553,28 @@ static void syncRemovePeer(SSyncPeer *pPeer) {
 
   pPeer->ip = 0;
   syncClosePeerConn(pPeer);
-  syncDecPeerRef(pPeer);
+  //taosRemoveRef(tsPeerRefId, pPeer->rid);
+  syncReleasePeer(pPeer);
+}
+
+static void syncStartCheckPeerConn(SSyncPeer *pPeer) {
+  if (pPeer == NULL) return;
+  SSyncNode *pNode = pPeer->pSyncNode;
+
+  int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
+  if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
+    int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
+    if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs;
+    sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs);
+    taosTmrReset(syncCheckPeerConnection, checkMs, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
+  }
+}
+
+static void syncStopCheckPeerConn(SSyncPeer *pPeer) {
+  if (pPeer == NULL) return;
+
+  taosTmrStopA(&pPeer->timer);
+  sDebug("%s, stop check peer connection", pPeer->id);
 }
 
 static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
@@ -508,17 +600,11 @@ static SSyncPeer *syncAddPeer(SSyncNode *pNode, const SNodeInfo *pInfo) {
   pPeer->role = TAOS_SYNC_ROLE_OFFLINE;
   pPeer->pSyncNode = pNode;
   pPeer->refCount = 1;
+  pPeer->rid = taosAddRef(tsPeerRefId, pPeer);
 
-  sInfo("%s, it is configured, ep:%s:%u", pPeer->id, pPeer->fqdn, pPeer->port);
-  int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
-  if (pPeer->nodeId == 0 || (ret > 0) || (ret == 0 && pPeer->port > tsSyncPort)) {
-    int32_t checkMs = 100 + (pNode->vgId * 10) % 100;
-    if (pNode->vgId > 1) checkMs = tsStatusInterval * 1000 + checkMs;
-    sDebug("%s, check peer connection after %d ms", pPeer->id, checkMs);
-    taosTmrReset(syncCheckPeerConnection, checkMs, pPeer, tsSyncTmrCtrl, &pPeer->timer);
-  }
+  sInfo("%s, %p it is configured, ep:%s:%u rid:%" PRId64, pPeer->id, pPeer, pPeer->fqdn, pPeer->port, pPeer->rid);
 
-  taosAcquireRef(tsSyncRefId, pNode->rid);  
+  (void)syncAcquireNode(pNode->rid);
   return pPeer;
 }
 
@@ -560,6 +646,9 @@ static void syncChooseMaster(SSyncNode *pNode) {
         index = i;
       }
     }
+    sDebug("vgId:%d, master:%s may be choosed, index:%d", pNode->vgId, pNode->peerInfo[index]->id, index);
+  } else {
+    sDebug("vgId:%d, no master election since onlineNum:%d replica:%d", pNode->vgId, onlineNum, pNode->replica);
   }
 
   // add arbitrator connection
@@ -580,6 +669,11 @@ static void syncChooseMaster(SSyncNode *pNode) {
         }
       }
     }
+
+    if (index >= 0) {
+      sDebug("vgId:%d, master:%s may be choosed, index:%d onlineNum(arb):%d replica:%d", pNode->vgId,
+             pNode->peerInfo[index]->id, index, onlineNum, replica);
+    }
   }
 
   if (index >= 0) {
@@ -621,10 +715,14 @@ static SSyncPeer *syncCheckMaster(SSyncNode *pNode) {
 
   if (onlineNum <= replica * 0.5) {
     if (nodeRole != TAOS_SYNC_ROLE_UNSYNCED) {
+       if (nodeRole == TAOS_SYNC_ROLE_MASTER && onlineNum == replica * 0.5 && onlineNum >= 1) {
+         sInfo("vgId:%d, self keep work as master, online:%d replica:%d", pNode->vgId, onlineNum, replica);
+       } else {
         nodeRole = TAOS_SYNC_ROLE_UNSYNCED;
-      (*pNode->notifyRole)(pNode->vgId, nodeRole);
         sInfo("vgId:%d, self change to unsynced state, online:%d replica:%d", pNode->vgId, onlineNum, replica);
        }
+      (*pNode->notifyRole)(pNode->vgId, nodeRole);
+    }
   } else {
     for (int32_t index = 0; index < pNode->replica; ++index) {
       SSyncPeer *pTemp = pNode->peerInfo[index];
@@ -678,7 +776,7 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new
   if (pMaster) {
     // master is there
     pNode->pMaster = pMaster;
-    sDebug("%s, it is the master, sver:%" PRIu64, pMaster->id, pMaster->version);
+    sDebug("%s, it is the master, replica:%d sver:%" PRIu64, pMaster->id, pNode->replica, pMaster->version);
 
     if (syncValidateMaster(pPeer) < 0) return;
 
@@ -711,10 +809,10 @@ static void syncCheckRole(SSyncPeer *pPeer, SPeerStatus* peersStatus, int8_t new
     }
 
     if (consistent) {
-      sDebug("vgId:%d, choose master", pNode->vgId);
+      sDebug("vgId:%d, choose master, replica:%d", pNode->vgId, pNode->replica);
       syncChooseMaster(pNode);
     } else {
-      sDebug("vgId:%d, cannot choose master since roles inequality", pNode->vgId);
+      sDebug("vgId:%d, cannot choose master since roles inequality, replica:%d", pNode->vgId, pNode->replica);
     }
   }
 
@@ -743,7 +841,7 @@ static void syncRestartPeer(SSyncPeer *pPeer) {
   int32_t ret = strcmp(pPeer->fqdn, tsNodeFqdn);
   if (ret > 0 || (ret == 0 && pPeer->port > tsSyncPort)) {
     sDebug("%s, check peer connection in 1000 ms", pPeer->id);
-    taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer);
+    taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
   }
 }
 
@@ -772,25 +870,30 @@ static void syncProcessSyncRequest(char *msg, SSyncPeer *pPeer) {
   }
 
   // start a new thread to retrieve the data
-  syncAddPeerRef(pPeer);
+  (void)syncAcquirePeer(pPeer->rid);
+
   pthread_attr_t thattr;
   pthread_t      thread;
   pthread_attr_init(&thattr);
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
-  int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, pPeer);
+  int32_t ret = pthread_create(&thread, &thattr, syncRetrieveData, (void *)pPeer->rid);
   pthread_attr_destroy(&thattr);
 
   if (ret != 0) {
     sError("%s, failed to create sync thread since %s", pPeer->id, strerror(errno));
-    syncDecPeerRef(pPeer);
   } else {
     pPeer->sstatus = TAOS_SYNC_STATUS_START;
     sDebug("%s, thread is created to retrieve data, set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
   }
+
+  syncReleasePeer(pPeer);
 }
 
 static void syncNotStarted(void *param, void *tmrId) {
-  SSyncPeer *pPeer = param;
+  int64_t    rid = (int64_t)param;
+  SSyncPeer *pPeer = syncAcquirePeer(rid);
+  if (pPeer == NULL) return;
+
   SSyncNode *pNode = pPeer->pSyncNode;
 
   pthread_mutex_lock(&pNode->mutex);
@@ -799,15 +902,22 @@ static void syncNotStarted(void *param, void *tmrId) {
   sInfo("%s, sync conn is still not up, restart and set sstatus:%s", pPeer->id, syncStatus[pPeer->sstatus]);
   syncRestartConnection(pPeer);
   pthread_mutex_unlock(&pNode->mutex);
+
+  syncReleasePeer(pPeer);
 }
 
 static void syncTryRecoverFromMaster(void *param, void *tmrId) {
-  SSyncPeer *pPeer = param;
+  int64_t    rid = (int64_t)param;
+  SSyncPeer *pPeer = syncAcquirePeer(rid);
+  if (pPeer == NULL) return;
+
   SSyncNode *pNode = pPeer->pSyncNode;
 
   pthread_mutex_lock(&pNode->mutex);
   syncRecoverFromMaster(pPeer);
   pthread_mutex_unlock(&pNode->mutex);
+
+  syncReleasePeer(pPeer);
 }
 
 static void syncRecoverFromMaster(SSyncPeer *pPeer) {
@@ -823,7 +933,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
   // Ensure the sync of mnode not interrupted
   if (pNode->vgId != 1 && tsSyncNum >= SYNC_MAX_NUM) {
     sInfo("%s, %d syncs are in process, try later", pPeer->id, tsSyncNum);
-    taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, pPeer, tsSyncTmrCtrl, &pPeer->timer);
+    taosTmrReset(syncTryRecoverFromMaster, 500 + (pNode->vgId * 10) % 200, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
     return;
   }
 
@@ -832,7 +942,7 @@ static void syncRecoverFromMaster(SSyncPeer *pPeer) {
   SSyncMsg msg;
   syncBuildSyncReqMsg(&msg, pNode->vgId);
 
-  taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer);
+  taosTmrReset(syncNotStarted, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
 
   if (taosWriteMsg(pPeer->peerFd, &msg, sizeof(SSyncMsg)) != sizeof(SSyncMsg)) {
     sError("%s, failed to send sync-req to peer", pPeer->id);
@@ -920,8 +1030,10 @@ static int32_t syncReadPeerMsg(SSyncPeer *pPeer, SSyncHead *pHead) {
   return 0;
 }
 
-static int32_t syncProcessPeerMsg(void *param, void *buffer) {
-  SSyncPeer *pPeer = param;
+static int32_t syncProcessPeerMsg(int64_t rid, void *buffer) {
+  SSyncPeer *pPeer = syncAcquirePeer(rid);
+  if (pPeer == NULL) return -1;
+
   SSyncHead *pHead = buffer;
   SSyncNode *pNode = pPeer->pSyncNode;
   
@@ -942,12 +1054,17 @@ static int32_t syncProcessPeerMsg(void *param, void *buffer) {
   }
 
   pthread_mutex_unlock(&pNode->mutex);
+  syncReleasePeer(pPeer);
 
   return code;
 }
 
-static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) {
-  if (pPeer->peerFd < 0 || pPeer->ip == 0) return;
+static int32_t syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type, uint16_t tranId) {
+  if (pPeer->peerFd < 0 || pPeer->ip == 0) {
+    sDebug("%s, failed to send status msg, restart fd:%d", pPeer->id, pPeer->peerFd);
+    syncRestartConnection(pPeer);
+    return -1;
+  }
 
   SSyncNode *  pNode = pPeer->pSyncNode;
   SPeersStatus msg;
@@ -970,9 +1087,11 @@ static void syncSendPeersStatusMsgToPeer(SSyncPeer *pPeer, char ack, int8_t type
     sDebug("%s, status is sent, self:%s:%s:%" PRIu64 ", peer:%s:%s:%" PRIu64 ", ack:%d tranId:%u type:%s pfd:%d",
            pPeer->id, syncRole[nodeRole], syncStatus[nodeSStatus], nodeVersion, syncRole[pPeer->role],
            syncStatus[pPeer->sstatus], pPeer->version, ack, tranId, statusType[type], pPeer->peerFd);
+    return 0;
   } else {
     sDebug("%s, failed to send status msg, restart", pPeer->id);
     syncRestartConnection(pPeer);
+    return -1;
   }
 }
 
@@ -989,7 +1108,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
   int32_t connFd = taosOpenTcpClientSocket(pPeer->ip, pPeer->port, 0);
   if (connFd < 0) {
     sDebug("%s, failed to open tcp socket since %s", pPeer->id, strerror(errno));
-    taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer);
+    taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
     return;
   }
 
@@ -1000,17 +1119,19 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
     sDebug("%s, connection to peer server is setup, pfd:%d sfd:%d tranId:%u", pPeer->id, connFd, pPeer->syncFd, msg.tranId);
     pPeer->peerFd = connFd;
     pPeer->role = TAOS_SYNC_ROLE_UNSYNCED;
-    pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer, connFd);
-    syncAddPeerRef(pPeer);
+    pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd);
   } else {
     sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno));
     taosClose(connFd);
-    taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, pPeer, tsSyncTmrCtrl, &pPeer->timer);
+    taosTmrReset(syncCheckPeerConnection, SYNC_CHECK_INTERVAL, (void *)pPeer->rid, tsSyncTmrCtrl, &pPeer->timer);
   }
 }
 
 static void syncCheckPeerConnection(void *param, void *tmrId) {
-  SSyncPeer *pPeer = param;
+  int64_t    rid = (int64_t)param;
+  SSyncPeer *pPeer = syncAcquirePeer(rid);
+  if (pPeer == NULL) return;
+
   SSyncNode *pNode = pPeer->pSyncNode;
 
   pthread_mutex_lock(&pNode->mutex);
@@ -1019,6 +1140,8 @@ static void syncCheckPeerConnection(void *param, void *tmrId) {
   syncSetupPeerConnection(pPeer);
 
   pthread_mutex_unlock(&pNode->mutex);
+
+  syncReleasePeer(pPeer);
 }
 
 static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
@@ -1029,8 +1152,9 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
   pthread_attr_init(&thattr);
   pthread_attr_setdetachstate(&thattr, PTHREAD_CREATE_DETACHED);
 
-  syncAddPeerRef(pPeer);
-  int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, pPeer);
+  (void)syncAcquirePeer(pPeer->rid);
+
+  int32_t ret = pthread_create(&(thread), &thattr, (void *)syncRestoreData, (void *)pPeer->rid);
   pthread_attr_destroy(&thattr);
 
   if (ret < 0) {
@@ -1038,10 +1162,11 @@ static void syncCreateRestoreDataThread(SSyncPeer *pPeer) {
     nodeSStatus = TAOS_SYNC_STATUS_INIT;
     sError("%s, failed to create sync thread, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
     taosClose(pPeer->syncFd);
-    syncDecPeerRef(pPeer);
   } else {
     sInfo("%s, sync connection is up", pPeer->id);
   }
+
+  syncReleasePeer(pPeer);
 }
 
 static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
@@ -1073,7 +1198,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
     return;
   }
 
-  sDebug("vgId:%d, sync msg is received, tranId:%u", vgId, msg.tranId);
+  sDebug("vgId:%d, sync connection is incomming, tranId:%u", vgId, msg.tranId);
 
   SSyncNode *pNode = *ppNode;
   pthread_mutex_lock(&pNode->mutex);
@@ -1101,8 +1226,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
       sDebug("%s, TCP connection is up, pfd:%d sfd:%d, old pfd:%d", pPeer->id, connFd, pPeer->syncFd, pPeer->peerFd);
       syncClosePeerConn(pPeer);
       pPeer->peerFd = connFd;
-      pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer, connFd);
-      syncAddPeerRef(pPeer);
+      pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd);
       sDebug("%s, ready to exchange data", pPeer->id);
       syncSendPeersStatusMsgToPeer(pPeer, 1, SYNC_STATUS_EXCHANGE_DATA, syncGenTranId());
     }
@@ -1111,23 +1235,21 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
   pthread_mutex_unlock(&pNode->mutex);
 }
 
-static void syncProcessBrokenLink(void *param) {
-  if (param == NULL) return;  // the connection for arbitrator
-  SSyncPeer *pPeer = param;
+static void syncProcessBrokenLink(int64_t rid) {
+  SSyncPeer *pPeer = syncAcquirePeer(rid);
+  if (pPeer == NULL) return;
+
   SSyncNode *pNode = pPeer->pSyncNode;
 
-  if (taosAcquireRef(tsSyncRefId, pNode->rid) == NULL) return;
   pthread_mutex_lock(&pNode->mutex);
 
   sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd);
   pPeer->peerFd = -1;
 
-  if (syncDecPeerRef(pPeer) != 0) {
   syncRestartConnection(pPeer);
-  }
-
   pthread_mutex_unlock(&pNode->mutex);
-  taosReleaseRef(tsSyncRefId, pNode->rid);
+
+  syncReleasePeer(pPeer);
 }
 
 static int32_t syncSaveFwdInfo(SSyncNode *pNode, uint64_t version, void *mhandle) {
@@ -1198,7 +1320,7 @@ static void syncProcessFwdAck(SSyncNode *pNode, SFwdInfo *pFwdInfo, int32_t code
 
 static void syncMonitorNodeRole(void *param, void *tmrId) {
   int64_t    rid = (int64_t)param;
-  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  SSyncNode *pNode = syncAcquireNode(rid);
   if (pNode == NULL) return;
 
   for (int32_t index = 0; index < pNode->replica; index++) {
@@ -1215,12 +1337,12 @@ static void syncMonitorNodeRole(void *param, void *tmrId) {
   }
 
   pNode->pRoleTimer = taosTmrStart(syncMonitorNodeRole, SYNC_ROLE_TIMER, (void *)pNode->rid, tsSyncTmrCtrl);
-  taosReleaseRef(tsSyncRefId, rid);
+  syncReleaseNode(pNode);
 }
 
 static void syncMonitorFwdInfos(void *param, void *tmrId) {
   int64_t    rid = (int64_t)param;
-  SSyncNode *pNode = taosAcquireRef(tsSyncRefId, rid);
+  SSyncNode *pNode = syncAcquireNode(rid);
   if (pNode == NULL) return;
 
   SSyncFwds *pSyncFwds = pNode->pSyncFwds;
@@ -1246,7 +1368,7 @@ static void syncMonitorFwdInfos(void *param, void *tmrId) {
     pNode->pFwdTimer = taosTmrStart(syncMonitorFwdInfos, SYNC_FWD_TIMER, (void *)pNode->rid, tsSyncTmrCtrl);
   }
 
-  taosReleaseRef(tsSyncRefId, rid);
+  syncReleaseNode(pNode);
 }
 
 static int32_t syncForwardToPeerImpl(SSyncNode *pNode, void *data, void *mhandle, int32_t qtype) {

src/sync/src/syncRestore.c

@@ -90,15 +90,18 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
       break;
     }
 
+    sDebug("%s, file:%s info is received from master, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id,
+           minfo.name, minfo.index, minfo.size, minfo.fversion, minfo.magic);
+
     // remove extra files on slave between the current and last index
     syncRemoveExtraFile(pPeer, pindex + 1, minfo.index - 1);
     pindex = minfo.index;
 
     // check the file info
     sinfo = minfo;
-    sDebug("%s, get file:%s info size:%" PRId64, pPeer->id, minfo.name, minfo.size);
-    sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size,
-                                        &sinfo.fversion);
+    sinfo.magic = (*pNode->getFileInfo)(pNode->vgId, sinfo.name, &sinfo.index, TAOS_SYNC_MAX_INDEX, &sinfo.size, &sinfo.fversion);
+    sDebug("%s, local file:%s info, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, sinfo.name,
+           sinfo.index, sinfo.size, sinfo.fversion, sinfo.magic);
 
     // if file not there or magic is not the same, file shall be synced
     memset(&fileAck, 0, sizeof(SFileAck));
@@ -116,6 +119,8 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
     if (fileAck.sync == 0) {
       sDebug("%s, %s is the same", pPeer->id, minfo.name);
       continue;
+    } else {
+      sDebug("%s, %s will be received, size:%" PRId64, pPeer->id, minfo.name, minfo.size);
     }
 
     // if sync is required, open file, receive from master, and write to file
@@ -155,7 +160,7 @@ static int32_t syncRestoreFile(SSyncPeer *pPeer, uint64_t *fversion) {
   return code;
 }
 
-static int32_t syncRestoreWal(SSyncPeer *pPeer) {
+static int32_t syncRestoreWal(SSyncPeer *pPeer, uint64_t *wver) {
   SSyncNode *pNode = pPeer->pSyncNode;
   int32_t    ret, code = -1;
   uint64_t   lastVer = 0;
@@ -198,6 +203,7 @@ static int32_t syncRestoreWal(SSyncPeer *pPeer) {
   }
 
   free(pHead);
+  *wver = lastVer;
   return code;
 }
 
@@ -321,12 +327,19 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
 
   nodeVersion = fversion;
 
-  sInfo("%s, start to restore wal", pPeer->id);
-  if (syncRestoreWal(pPeer) < 0) {
-    sError("%s, failed to restore wal", pPeer->id);
+  sInfo("%s, start to restore wal, fver:%" PRIu64, pPeer->id, nodeVersion);
+  uint64_t wver = 0;
+  code = syncRestoreWal(pPeer, &wver);  // lastwar
+  if (code < 0) {
+    sError("%s, failed to restore wal, code:%d", pPeer->id, code);
     return -1;
   }
 
+  if (wver != 0) {
+    nodeVersion = wver;
+    sDebug("%s, restore wal finished, set sver:%" PRIu64, pPeer->id, nodeVersion);
+  }
+
   nodeSStatus = TAOS_SYNC_STATUS_CACHE;
   sInfo("%s, start to insert buffered points, set sstatus:%s", pPeer->id, syncStatus[nodeSStatus]);
   if (syncProcessBufferedFwd(pPeer) < 0) {
@@ -338,7 +351,10 @@ static int32_t syncRestoreDataStepByStep(SSyncPeer *pPeer) {
 }
 
 void *syncRestoreData(void *param) {
-  SSyncPeer *pPeer = param;
+  int64_t    rid = (int64_t)param;
+  SSyncPeer *pPeer = syncAcquirePeer(rid);
+  if (pPeer == NULL) return NULL;
+
   SSyncNode *pNode = pPeer->pSyncNode;
 
   taosBlockSIGPIPE();
@@ -369,7 +385,7 @@ void *syncRestoreData(void *param) {
   taosClose(pPeer->syncFd);
   syncCloseRecvBuffer(pNode);
   __sync_fetch_and_sub(&tsSyncNum, 1);
-  syncDecPeerRef(pPeer);
+  syncReleasePeer(pPeer);
 
   return NULL;
 }

src/sync/src/syncRetrieve.c

@@ -104,7 +104,8 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
     fileInfo.magic = (*pNode->getFileInfo)(pNode->vgId, fileInfo.name, &fileInfo.index, TAOS_SYNC_MAX_INDEX,
                                            &fileInfo.size, &fileInfo.fversion);
     syncBuildFileInfo(&fileInfo, pNode->vgId);
-    sDebug("%s, file:%s info is sent, size:%" PRId64, pPeer->id, fileInfo.name, fileInfo.size);
+    sDebug("%s, file:%s info is sent, index:%d size:%" PRId64 " fver:%" PRIu64 " magic:%d", pPeer->id, fileInfo.name,
+           fileInfo.index, fileInfo.size, fileInfo.fversion, fileInfo.magic);
 
     // send the file info
     int32_t ret = taosWriteMsg(pPeer->syncFd, &(fileInfo), sizeof(SFileInfo));
@@ -144,6 +145,8 @@ static int32_t syncRetrieveFile(SSyncPeer *pPeer) {
       fileInfo.index++;
       sDebug("%s, %s is the same", pPeer->id, fileInfo.name);
       continue;
+    } else {
+      sDebug("%s, %s will be sent", pPeer->id, fileInfo.name);
     }
 
     // get the full path to file
@@ -461,7 +464,10 @@ static int32_t syncRetrieveDataStepByStep(SSyncPeer *pPeer) {
 }
 
 void *syncRetrieveData(void *param) {
-  SSyncPeer *pPeer = (SSyncPeer *)param;
+  int64_t    rid = (int64_t)param;
+  SSyncPeer *pPeer = syncAcquirePeer(rid);
+  if (pPeer == NULL) return NULL;
+
   SSyncNode *pNode = pPeer->pSyncNode;
   taosBlockSIGPIPE();
 
@@ -490,7 +496,7 @@ void *syncRetrieveData(void *param) {
 
   pPeer->fileChanged = 0;
   taosClose(pPeer->syncFd);
-  syncDecPeerRef(pPeer);
+  syncReleasePeer(pPeer);
 
   return NULL;
 }

src/sync/src/syncTcp.c

@@ -42,7 +42,7 @@ typedef struct SPoolObj {
 
 typedef struct {
   SThreadObj *pThread;
-  void *      ahandle;
+  int64_t     handleId;
   int32_t     fd;
   int32_t     closedByApp;
 } SConnObj;
@@ -112,7 +112,7 @@ void syncCloseTcpThreadPool(void *param) {
   tfree(pPool);
 }
 
-void *syncAllocateTcpConn(void *param, void *pPeer, int32_t connFd) {
+void *syncAllocateTcpConn(void *param, int64_t rid, int32_t connFd) {
   struct epoll_event event;
   SPoolObj *pPool = param;
 
@@ -130,7 +130,7 @@ void *syncAllocateTcpConn(void *param, void *pPeer, int32_t connFd) {
 
   pConn->fd = connFd;
   pConn->pThread = pThread;
-  pConn->ahandle = pPeer;
+  pConn->handleId = rid;
   pConn->closedByApp = 0;
 
   event.events = EPOLLIN | EPOLLRDHUP;
@@ -164,7 +164,7 @@ static void taosProcessBrokenLink(SConnObj *pConn) {
   SPoolInfo * pInfo = &pPool->info;
 
   if (pConn->closedByApp == 0) shutdown(pConn->fd, SHUT_WR);
-  (*pInfo->processBrokenLink)(pConn->ahandle);
+  (*pInfo->processBrokenLink)(pConn->handleId);
 
   pThread->numOfFds--;
   epoll_ctl(pThread->pollFd, EPOLL_CTL_DEL, pConn->fd, NULL);
@@ -221,7 +221,7 @@ static void *syncProcessTcpData(void *param) {
       }
 
       if (pConn->closedByApp == 0) {
-        if ((*pInfo->processIncomingMsg)(pConn->ahandle, buffer) < 0) {
+        if ((*pInfo->processIncomingMsg)(pConn->handleId, buffer) < 0) {
           syncFreeTcpConn(pConn);
           continue;
         }

src/tsdb/inc/tsdbMain.h

@@ -31,14 +31,14 @@
 extern "C" {
 #endif
 
-extern int tsdbDebugFlag;
-
-#define tsdbFatal(...) { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}
-#define tsdbError(...) { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}
-#define tsdbWarn(...)  { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}
-#define tsdbInfo(...)  { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}
-#define tsdbDebug(...) { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
-#define tsdbTrace(...) { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }}
+extern uint32_t tsdbDebugFlag;
+
+#define tsdbFatal(...) do { if (tsdbDebugFlag & DEBUG_FATAL) { taosPrintLog("TDB FATAL ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbError(...) do { if (tsdbDebugFlag & DEBUG_ERROR) { taosPrintLog("TDB ERROR ", 255, __VA_ARGS__); }}     while(0)
+#define tsdbWarn(...)  do { if (tsdbDebugFlag & DEBUG_WARN)  { taosPrintLog("TDB WARN ", 255, __VA_ARGS__); }}      while(0)
+#define tsdbInfo(...)  do { if (tsdbDebugFlag & DEBUG_INFO)  { taosPrintLog("TDB ", 255, __VA_ARGS__); }}           while(0)
+#define tsdbDebug(...) do { if (tsdbDebugFlag & DEBUG_DEBUG) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
+#define tsdbTrace(...) do { if (tsdbDebugFlag & DEBUG_TRACE) { taosPrintLog("TDB ", tsdbDebugFlag, __VA_ARGS__); }} while(0)
 
 #define TSDB_MAX_TABLE_SCHEMAS 16
 #define TSDB_FILE_HEAD_SIZE    512

src/tsdb/src/tsdbCommit.c

@@ -161,6 +161,11 @@ _err:
 
 static void tsdbEndCommit(STsdbRepo *pRepo, int eno) {
   if (pRepo->appH.notifyStatus) pRepo->appH.notifyStatus(pRepo->appH.appH, TSDB_STATUS_COMMIT_OVER, eno);
+  SMemTable *pIMem = pRepo->imem;
+  tsdbLockRepo(pRepo);
+  pRepo->imem = NULL;
+  tsdbUnlockRepo(pRepo);
+  tsdbUnRefMemTable(pRepo, pIMem);
   sem_post(&(pRepo->readyToCommit));
 }
 

src/tsdb/src/tsdbMemTable.c

@@ -17,6 +17,7 @@
 #include "tsdbMain.h"
 
 #define TSDB_DATA_SKIPLIST_LEVEL 5
+#define TSDB_MAX_INSERT_BATCH 512
 
 static SMemTable * tsdbNewMemTable(STsdbRepo *pRepo);
 static void        tsdbFreeMemTable(SMemTable *pMemTable);
@@ -205,7 +206,7 @@ void *tsdbAllocBytes(STsdbRepo *pRepo, int bytes) {
 int tsdbAsyncCommit(STsdbRepo *pRepo) {
   if (pRepo->mem == NULL) return 0;
 
-  SMemTable *pIMem = pRepo->imem;
+  ASSERT(pRepo->imem == NULL);
 
   sem_wait(&(pRepo->readyToCommit));
 
@@ -220,8 +221,6 @@ int tsdbAsyncCommit(STsdbRepo *pRepo) {
   tsdbScheduleCommit(pRepo);
   if (tsdbUnlockRepo(pRepo) < 0) return -1;
 
-  if (tsdbUnRefMemTable(pRepo, pIMem) < 0) return -1;
-
   return 0;
 }
 
@@ -606,19 +605,13 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *
   STable *       pTable = NULL;
   SSubmitBlkIter blkIter = {0};
   SDataRow       row = NULL;
-  void **        rows = NULL;
+  void *         rows[TSDB_MAX_INSERT_BATCH] = {0};
   int            rowCounter = 0;
 
   ASSERT(pBlock->tid < pMeta->maxTables);
   pTable = pMeta->tables[pBlock->tid];
   ASSERT(pTable != NULL && TABLE_UID(pTable) == pBlock->uid);
 
-  rows = (void **)calloc(pBlock->numOfRows, sizeof(void *));
-  if (rows == NULL) {
-    terrno = TSDB_CODE_TDB_OUT_OF_MEMORY;
-    return -1;
-  }
-
   tsdbInitSubmitBlkIter(pBlock, &blkIter);
   while ((row = tsdbGetSubmitBlkNext(&blkIter)) != NULL) {
     if (tsdbCopyRowToMem(pRepo, row, pTable, &(rows[rowCounter])) < 0) {
@@ -632,21 +625,28 @@ static int tsdbInsertDataToTable(STsdbRepo *pRepo, SSubmitBlk *pBlock, int32_t *
     if (rows[rowCounter] != NULL) {
       rowCounter++;
     }
-  }
 
+    if (rowCounter == TSDB_MAX_INSERT_BATCH) {
       if (tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
         goto _err;
       }
 
+      rowCounter = 0;
+      memset(rows, 0, sizeof(rows));
+    }
+  }
+
+  if (rowCounter > 0 && tsdbInsertDataToTableImpl(pRepo, pTable, rows, rowCounter) < 0) {
+    goto _err;
+  }
+
   STSchema *pSchema = tsdbGetTableSchemaByVersion(pTable, pBlock->sversion);
   pRepo->stat.pointsWritten += points * schemaNCols(pSchema);
   pRepo->stat.totalStorage += points * schemaVLen(pSchema);
 
-  free(rows);
   return 0;
 
 _err:
-  free(rows);
   return -1;
 }
 

src/tsdb/src/tsdbRead.c

@@ -956,9 +956,9 @@ static int32_t loadFileDataBlock(STsdbQueryHandle* pQueryHandle, SCompBlock* pBl
         return code;
       }
 
-      SDataCols* pTSCol = pQueryHandle->rhelper.pDataCols[0];
+      SDataCols* pTsCol = pQueryHandle->rhelper.pDataCols[0];
       if (pCheckInfo->lastKey < pBlock->keyLast) {
-        cur->pos = binarySearchForKey(pTSCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
+        cur->pos = binarySearchForKey(pTsCol->cols[0].pData, pBlock->numOfRows, pCheckInfo->lastKey, pQueryHandle->order);
       } else {
         cur->pos = pBlock->numOfRows - 1;
       }
@@ -1704,7 +1704,32 @@ static int32_t createDataBlocksInfo(STsdbQueryHandle* pQueryHandle, int32_t numO
   return TSDB_CODE_SUCCESS;
 }
 
-static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* exists) {
+static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists);
+
+static int32_t getDataBlockRv(STsdbQueryHandle* pQueryHandle, STableBlockInfo* pNext, bool *exists) {
+  int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
+  SQueryFilePos* cur = &pQueryHandle->cur;
+
+  while(1) {
+    int32_t code = loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists);
+    if (code != TSDB_CODE_SUCCESS || *exists) {
+      return code;
+    }
+
+    if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
+        (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
+      // all data blocks in current file has been checked already, try next file if exists
+      return getFirstFileDataBlock(pQueryHandle, exists);
+    } else {  // next block of the same file
+      cur->slot += step;
+      cur->mixBlock = false;
+      cur->blockCompleted = false;
+      pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
+    }
+  }
+}
+
+static int32_t getFirstFileDataBlock(STsdbQueryHandle* pQueryHandle, bool* exists) {
   pQueryHandle->numOfBlocks = 0;
   SQueryFilePos* cur = &pQueryHandle->cur;
 
@@ -1789,7 +1814,23 @@ static int32_t getDataBlocksInFilesImpl(STsdbQueryHandle* pQueryHandle, bool* ex
   cur->fid = pQueryHandle->pFileGroup->fileId;
 
   STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
-  return loadFileDataBlock(pQueryHandle, pBlockInfo->compBlock, pBlockInfo->pTableCheckInfo, exists);
+  return getDataBlockRv(pQueryHandle, pBlockInfo, exists);
+}
+
+static bool isEndFileDataBlock(SQueryFilePos* cur, int32_t numOfBlocks, bool ascTrav) {
+  assert(cur != NULL && numOfBlocks > 0);
+  return (cur->slot == numOfBlocks - 1 && ascTrav) || (cur->slot == 0 && !ascTrav);
+}
+
+static void moveToNextDataBlockInCurrentFile(STsdbQueryHandle* pQueryHandle) {
+  int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order)? 1 : -1;
+
+  SQueryFilePos* cur = &pQueryHandle->cur;
+  assert(cur->slot < pQueryHandle->numOfBlocks && cur->slot >= 0);
+
+  cur->slot += step;
+  cur->mixBlock       = false;
+  cur->blockCompleted = false;
 }
 
 static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists) {
@@ -1807,7 +1848,7 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
     tsdbSeekFileGroupIter(&pQueryHandle->fileIter, fid);
     pthread_rwlock_unlock(&pQueryHandle->pTsdb->tsdbFileH->fhlock);
 
-    return getDataBlocksInFilesImpl(pQueryHandle, exists);
+    return getFirstFileDataBlock(pQueryHandle, exists);
   } else {
     // check if current file block is all consumed
     STableBlockInfo* pBlockInfo = &pQueryHandle->pDataBlockInfo[cur->slot];
@@ -1815,29 +1856,28 @@ static int32_t getDataBlocksInFiles(STsdbQueryHandle* pQueryHandle, bool* exists
 
     // current block is done, try next
     if ((!cur->mixBlock) || cur->blockCompleted) {
-      if ((cur->slot == pQueryHandle->numOfBlocks - 1 && ASCENDING_TRAVERSE(pQueryHandle->order)) ||
-          (cur->slot == 0 && !ASCENDING_TRAVERSE(pQueryHandle->order))) {
       // all data blocks in current file has been checked already, try next file if exists
-        return getDataBlocksInFilesImpl(pQueryHandle, exists);
-      } else {
-        // next block of the same file
-        int32_t step = ASCENDING_TRAVERSE(pQueryHandle->order) ? 1 : -1;
-        cur->slot += step;
-
-        cur->mixBlock = false;
-        cur->blockCompleted = false;
-
-        STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
-        return loadFileDataBlock(pQueryHandle, pNext->compBlock, pNext->pTableCheckInfo, exists);
-      }
     } else {
-      tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos, pQueryHandle->qinfo);
+      tsdbDebug("%p continue in current data block, index:%d, pos:%d, %p", pQueryHandle, cur->slot, cur->pos,
+                pQueryHandle->qinfo);
       int32_t code = handleDataMergeIfNeeded(pQueryHandle, pBlockInfo->compBlock, pCheckInfo);
-      *exists = pQueryHandle->realNumOfRows > 0;
+      *exists = (pQueryHandle->realNumOfRows > 0);
 
+      if (code != TSDB_CODE_SUCCESS || *exists) {
         return code;
       }
     }
+
+    // current block is empty, try next block in file
+    // all data blocks in current file has been checked already, try next file if exists
+    if (isEndFileDataBlock(cur, pQueryHandle->numOfBlocks, ASCENDING_TRAVERSE(pQueryHandle->order))) {
+      return getFirstFileDataBlock(pQueryHandle, exists);
+    } else {
+      moveToNextDataBlockInCurrentFile(pQueryHandle);
+      STableBlockInfo* pNext = &pQueryHandle->pDataBlockInfo[cur->slot];
+      return getDataBlockRv(pQueryHandle, pNext, exists);
+    }
+  }
 }
 
 static bool doHasDataInBuffer(STsdbQueryHandle* pQueryHandle) {

src/util/inc/tref.h

@@ -52,6 +52,8 @@ void *taosIterateRef(int rsetId, int64_t rid);
 // return the number of references in system
 int  taosListRef();  
 
+#define RID_VALID(x) ((x) > 0)
+
 /* sample code to iterate the refs 
 
 void demoIterateRefs(int rsetId) {

src/util/src/tcache.c

@@ -547,13 +547,14 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
     return;
   }
 
+  __cache_wr_lock(pCacheObj);
   STrashElem *pElem = calloc(1, sizeof(STrashElem));
   pElem->pData = pNode;
   pElem->prev = NULL;
+  pElem->next = NULL;
   pNode->inTrashcan = true;
   pNode->pTNodeHeader = pElem;
 
-  __cache_wr_lock(pCacheObj);
   pElem->next = pCacheObj->pTrash;
   if (pCacheObj->pTrash) {
     pCacheObj->pTrash->prev = pElem;
@@ -563,8 +564,8 @@ void taosAddToTrashcan(SCacheObj *pCacheObj, SCacheDataNode *pNode) {
   pCacheObj->numOfElemsInTrash++;
   __cache_unlock(pCacheObj);
 
-  uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name,
-      pNode->key, pNode->data, pElem, pCacheObj->numOfElemsInTrash);
+  uDebug("cache:%s key:%p, %p move to trashcan, pTrashElem:%p, numOfElem in trashcan:%d", pCacheObj->name, pNode->key,
+         pNode->data, pElem, pCacheObj->numOfElemsInTrash);
 }
 
 void taosTrashcanEmpty(SCacheObj *pCacheObj, bool force) {

src/util/src/tskiplist.c

@@ -280,10 +280,13 @@ bool tSkipListIterNext(SSkipListIterator *iter) {
   tSkipListRLock(pSkipList);
 
   if (iter->order == TSDB_ORDER_ASC) {
-    if (iter->cur == pSkipList->pTail) {
+    // no data in the skip list
+    if (iter->cur == pSkipList->pTail || iter->next == NULL) {
+      iter->cur = pSkipList->pTail;
       tSkipListUnlock(pSkipList);
       return false;
     }
+
     iter->cur = SL_NODE_GET_FORWARD_POINTER(iter->cur, 0);
 
     // a new node is inserted into between iter->cur and iter->next, ignore it
@@ -295,9 +298,11 @@ bool tSkipListIterNext(SSkipListIterator *iter) {
     iter->step++;
   } else {
     if (iter->cur == pSkipList->pHead) {
+      iter->cur = pSkipList->pHead;
       tSkipListUnlock(pSkipList);
       return false;
     }
+
     iter->cur = SL_NODE_GET_BACKWARD_POINTER(iter->cur, 0);
 
     // a new node is inserted into between iter->cur and iter->next, ignore it

src/util/src/tsocket.c

@@ -18,6 +18,10 @@
 #include "tsocket.h"
 #include "taoserror.h"
 
+#ifndef SIGPIPE
+  #define SIGPIPE EPIPE
+#endif
+
 int32_t taosGetFqdn(char *fqdn) {
   char hostname[1024];
   hostname[1023] = '\0';
@@ -115,6 +119,10 @@ int32_t taosWriteMsg(SOCKET fd, void *buf, int32_t nbytes) {
       nleft -= nwritten;
       ptr += nwritten;
     }
+
+    if (errno == SIGPIPE || errno == EPIPE) {
+      return -1;
+    }
   }
 
   return (nbytes - nleft);
@@ -142,6 +150,10 @@ int32_t taosReadMsg(SOCKET fd, void *buf, int32_t nbytes) {
       nleft -= nread;
       ptr += nread;
     }
+
+    if (errno == SIGPIPE || errno == EPIPE) {
+      return -1;
+    }
   }
 
   return (nbytes - nleft);

src/vnode/src/vnodeCfg.c

@@ -106,9 +106,10 @@ int32_t vnodeReadCfg(SVnodeObj *pVnode) {
   cJSON *vgCfgVersion = cJSON_GetObjectItem(root, "vgCfgVersion");
   if (!vgCfgVersion || vgCfgVersion->type != cJSON_Number) {
     vError("vgId:%d, failed to read %s, vgCfgVersion not found", pVnode->vgId, file);
-    goto PARSE_VCFG_ERROR;
-  }
+    vnodeMsg.cfg.vgCfgVersion = 0;
+  } else {
     vnodeMsg.cfg.vgCfgVersion = vgCfgVersion->valueint;
+  }
 
   cJSON *cacheBlockSize = cJSON_GetObjectItem(root, "cacheBlockSize");
   if (!cacheBlockSize || cacheBlockSize->type != cJSON_Number) {

src/vnode/src/vnodeMgmt.c

@@ -89,7 +89,10 @@ static void vnodeIncRef(void *ptNode) {
 }
 
 void *vnodeAcquire(int32_t vgId) {
-  SVnodeObj **ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
+  SVnodeObj **ppVnode = NULL;
+  if (tsVnodesHash != NULL) {
+    ppVnode = taosHashGetCB(tsVnodesHash, &vgId, sizeof(int32_t), vnodeIncRef, NULL, sizeof(void *));
+  }
 
   if (ppVnode == NULL || *ppVnode == NULL) {
     terrno = TSDB_CODE_VND_INVALID_VGROUP_ID;

src/vnode/src/vnodeRead.c

@@ -65,13 +65,17 @@ static int32_t vnodeCheckRead(SVnodeObj *pVnode) {
     return TSDB_CODE_APP_NOT_READY;
   }
 
-  if (pVnode->role != TAOS_SYNC_ROLE_SLAVE && pVnode->role != TAOS_SYNC_ROLE_MASTER) {
-    vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p", pVnode->vgId, pVnode->syncCfg.replica,
-           syncRole[pVnode->role], pVnode->refCount, pVnode);
-    return TSDB_CODE_APP_NOT_READY;
+  if (pVnode->role == TAOS_SYNC_ROLE_MASTER) {
+    return TSDB_CODE_SUCCESS;
   }
 
+  if (tsEnableSlaveQuery && pVnode->role == TAOS_SYNC_ROLE_SLAVE) {
     return TSDB_CODE_SUCCESS;
+  }
+
+  vDebug("vgId:%d, replica:%d role:%s, refCount:%d pVnode:%p, cant provide query service", pVnode->vgId, pVnode->syncCfg.replica,
+         syncRole[pVnode->role], pVnode->refCount, pVnode);
+  return TSDB_CODE_APP_NOT_READY;
 }
 
 void vnodeFreeFromRQueue(void *vparam, SVReadMsg *pRead) {
@@ -303,8 +307,11 @@ static int32_t vnodeProcessQueryMsg(SVnodeObj *pVnode, SVReadMsg *pRead) {
         // NOTE: set return code to be TSDB_CODE_QRY_HAS_RSP to notify dnode to return msg to client
         code = TSDB_CODE_QRY_HAS_RSP;
       } else {
-        void *h1 = qGetResultRetrieveMsg(*qhandle);
-        assert(h1 == NULL);
+        //void *h1 = qGetResultRetrieveMsg(*qhandle);
+
+        /* remove this assert, one possible case that will cause h1 not NULL: query thread unlock pQInfo->lock, and then FETCH thread execute twice before query thread reach here */
+        //assert(h1 == NULL);
+
         freehandle = qQueryCompleted(*qhandle);
       }
 

src/vnode/src/vnodeWrite.c

@@ -297,7 +297,7 @@ static int32_t vnodePerformFlowCtrl(SVWriteMsg *pWrite) {
   if (pWrite->qtype != TAOS_QTYPE_RPC) return 0;
   if (pVnode->queuedWMsg < MAX_QUEUED_MSG_NUM && pVnode->flowctrlLevel <= 0) return 0;
 
-  if (tsFlowCtrl == 0) {
+  if (tsEnableFlowCtrl == 0) {
     int32_t ms = pow(2, pVnode->flowctrlLevel + 2);
     if (ms > 100) ms = 100;
     vTrace("vgId:%d, msg:%p, app:%p, perform flowctrl for %d ms", pVnode->vgId, pWrite, pWrite->rpcMsg.ahandle, ms);

tests/Jenkinsfile

@@ -50,12 +50,7 @@ pipeline {
           agent{label 'master'}
           steps {
             pre_test()
-            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
-                sh '''
-                cd ${WKC}/tests/pytest
-                python3 concurrent_inquiry.py -c 1
-                '''
-            }
+            
             sh '''
             cd ${WKC}/tests
             ./test-all.sh b1
@@ -82,53 +77,26 @@ pipeline {
                 ./handle_crash_gen_val_log.sh
                 '''
             }
-            sh '''
-            cd ${WKC}/tests
-            ./test-all.sh b2
-            date
-            '''
-          }
-        }
-
-        stage('test_valgrind') {
-          agent{label "186"}
-
-          steps {
-            pre_test()
-            sh '''
-            cd ${WKC}/tests/pytest
-            ./valgrind-test.sh 2>&1 > mem-error-out.log
-            ./handle_val_log.sh
-          
-            date
-            cd ${WKC}/tests
-            ./test-all.sh b3
-            date'''
-          }
-        }
-       stage('connector'){
-         agent{label "release"}
-         steps{
             sh'''
-            cd ${WORKSPACE}
-            git checkout develop
+            systemctl start taosd 
+            sleep 10
             '''
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                 sh '''
-                cd ${WORKSPACE}/tests/gotest
+                cd ${WKC}/tests/gotest
                 bash batchtest.sh
                 '''
             }
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                 sh '''
-                cd ${WORKSPACE}/tests/examples/python/PYTHONConnectorChecker
+                cd ${WKC}/tests/examples/python/PYTHONConnectorChecker
                 python3 PythonChecker.py
                 '''
             }
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
                 sh '''
-                cd ${WORKSPACE}/tests/examples/JDBC/JDBCDemo/
-                mvn clean package assembly:single >/dev/null 
+                cd ${WKC}/tests/examples/JDBC/JDBCDemo/
+                mvn clean package assembly:single -DskipTests >/dev/null 
                 java -jar target/jdbcChecker-SNAPSHOT-jar-with-dependencies.jar -host 127.0.0.1
                 '''
             }
@@ -138,9 +106,41 @@ pipeline {
                 dotnet run
                 '''
             }
+            sh '''
+            systemctl stop taosd 
+            cd ${WKC}/tests
+            ./test-all.sh b2
+            date
+            '''
+          }
+        }
+
+        stage('test_valgrind') {
+          agent{label "186"}
 
+          steps {
+            pre_test()
+            catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
+                sh '''
+                cd ${WKC}/tests/pytest
+                nohup taosd >/dev/null &
+                sleep 10
+                python3 concurrent_inquiry.py -c 1
+                
+                '''
+            }
+            sh '''
+            cd ${WKC}/tests/pytest
+            ./valgrind-test.sh 2>&1 > mem-error-out.log
+            ./handle_val_log.sh
+          
+            date
+            cd ${WKC}/tests
+            ./test-all.sh b3
+            date'''
           }
         }
+       
        stage('arm64_build'){
                 agent{label 'arm64'}
                 steps{

tests/pytest/concurrent_inquiry.py

@@ -388,7 +388,9 @@ class ConcurrentInquiry:
                     print(
                 "Failure thread%d, sql: %s \nexception: %s" %
                 (threadID, str(sql),str(e)))
-                    #exit(-1)
+                    err_uec='Unable to establish connection'
+                    if err_uec in str(e) and loop >0:
+                        exit(-1)
                 loop -= 1
                 if loop == 0: break
                     
@@ -415,7 +417,9 @@ class ConcurrentInquiry:
                 print(
             "Failure thread%d, sql: %s \nexception: %s" %
             (threadID, str(sql),str(e)))
-                #exit(-1)
+                err_uec='Unable to establish connection'
+                if err_uec in str(e) and loop >0:
+                    exit(-1)
             loop -= 1    
             if loop == 0: break
                 

tests/pytest/crash_gen/crash_gen_main.py

@@ -2224,23 +2224,26 @@ class ClientManager:
         if svcMgr: # gConfig.auto_start_service:
             svcMgr.stopTaosServices()
             svcMgr = None
-        # Print exec status, etc., AFTER showing messages from the server
-        self.conclude()
-        # print("TC failed (2) = {}".format(self.tc.isFailed()))
-        # Linux return code: ref https://shapeshed.com/unix-exit-codes/
-        ret = 1 if self.tc.isFailed() else 0
-        self.tc.cleanup()
+        
 
         # Release global variables
         gConfig = None
         gSvcMgr = None
         logger = None
         
-        # Release variables here
-        self.tc = None
         thPool = None
+        dbManager.cleanUp() # destructor wouldn't run in time
         dbManager = None
 
+        # Print exec status, etc., AFTER showing messages from the server
+        self.conclude()
+        # print("TC failed (2) = {}".format(self.tc.isFailed()))
+        # Linux return code: ref https://shapeshed.com/unix-exit-codes/
+        ret = 1 if self.tc.isFailed() else 0
+        self.tc.cleanup()
+        # Release variables here
+        self.tc = None
+
         gc.collect() # force garbage collection
         # h = hpy()
         # print("\n----- Final Python Heap -----\n")        

tests/pytest/crash_gen/db.py

@@ -394,6 +394,7 @@ class DbManager():
             cType == 'native') else DbConn.createRest(dbTarget)
         try:
             self._dbConn.open()  # may throw taos.error.ProgrammingError: disconnected
+            Logging.debug("DbManager opened DB connection...")
         except taos.error.ProgrammingError as err:
             # print("Error type: {}, msg: {}, value: {}".format(type(err), err.msg, err))
             if (err.msg == 'client disconnected'):  # cannot open DB connection
@@ -412,6 +413,10 @@ class DbManager():
         # Moved to Database()
         # self._stateMachine = StateMechine(self._dbConn)
 
+    def __del__(self):
+        ''' Release the underlying DB connection upon deletion of DbManager '''
+        self.cleanUp()
+
     def getDbConn(self):
         return self._dbConn
 
@@ -438,5 +443,8 @@ class DbManager():
         return "table_{}".format(tblNum)
 
     def cleanUp(self):
+        if self._dbConn:
             self._dbConn.close()
+            self._dbConn = None
+            Logging.debug("DbManager closed DB connection...")
 

tests/pytest/crash_gen/misc.py

@@ -2,6 +2,7 @@ import threading
 import random
 import logging
 import os
+import sys
 
 import taos
 
@@ -53,7 +54,7 @@ class Logging:
         # global misc.logger
         _logger = logging.getLogger('CrashGen')  # real logger
         _logger.addFilter(LoggingFilter())
-        ch = logging.StreamHandler()
+        ch = logging.StreamHandler(sys.stdout) # Ref: https://stackoverflow.com/questions/14058453/making-python-loggers-output-all-messages-to-stdout-in-addition-to-log-file
         _logger.addHandler(ch)
 
         # Logging adapter, to be used as a logger

tests/pytest/crash_gen_bootstrap.py

@@ -19,5 +19,5 @@ if __name__ == "__main__":
     mExec.init()
     exitCode = mExec.run()
 
-    print("Exiting with code: {}".format(exitCode))
+    print("\nCrash_Gen is now exiting with status code: {}".format(exitCode))
     sys.exit(exitCode)

tests/pytest/handle_crash_gen_val_log.sh

@@ -5,9 +5,10 @@ GREEN='\033[1;32m'
 GREEN_DARK='\033[0;32m'
 GREEN_UNDERLINE='\033[4;32m'
 NC='\033[0m'
-nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null &
+#nohup /var/lib/jenkins/workspace/TDinternal/debug/build/bin/taosd -c /var/lib/jenkins/workspace/TDinternal/community/sim/dnode1/cfg >/dev/null &
+nohup /root/TDinternal/debug/build/bin/taosd -c /root/TDinternal/community/sim/dnode1/cfg >/dev/null &
 ./crash_gen.sh --valgrind -p -t 10 -s 250 -b 4
-pidof taosd|xargs kill  
+pidof taosd|xargs kill -9
 grep 'start to execute\|ERROR SUMMARY' valgrind.err|grep -v 'grep'|uniq|tee crash_gen_mem_err.log
 
 for memError in `grep 'ERROR SUMMARY' crash_gen_mem_err.log | awk '{print $4}'`

tests/pytest/import_merge/importCacheFileH.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileHO.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileHPO.py

@@ -57,7 +57,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileS.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileSub.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileT.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileTO.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importCacheFileTPO.py

@@ -57,7 +57,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataH2.py

@@ -59,7 +59,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataHO.py

@@ -60,7 +60,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataHO2.py

@@ -60,7 +60,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataHPO.py

@@ -62,7 +62,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastH.py

@@ -59,7 +59,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastHO.py

@@ -59,7 +59,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastHPO.py

@@ -61,7 +61,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastS.py

@@ -59,7 +59,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastSub.py

@@ -32,7 +32,7 @@ class TDTestCase:
         tdDnodes.stop(1)
         tdDnodes.deploy(1)
         tdDnodes.start(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
 
         tdSql.execute('reset query cache')
         tdSql.execute('drop database if exists db')
@@ -60,9 +60,9 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
 
         tdLog.info("================= step5")
         tdLog.info("import 10 data totally repetitive")

tests/pytest/import_merge/importDataLastT.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastTO.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataLastTPO.py

@@ -57,7 +57,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataS.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataSub.py

@@ -60,7 +60,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataT.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataTO.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importDataTPO.py

@@ -57,9 +57,9 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
 
         tdLog.info("================= step5")
         tdLog.info("import 20 data later with partly overlap")

tests/pytest/import_merge/importHORestart.py

@@ -57,7 +57,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importHPORestart.py

@@ -62,7 +62,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importHRestart.py

@@ -54,7 +54,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importInsertThenImport.py

@@ -61,7 +61,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.stop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdLog.info("import 100 sequential data again")

tests/pytest/import_merge/importLastH.py

@@ -53,7 +53,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastHO.py

@@ -53,7 +53,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastHPO.py

@@ -55,7 +55,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastS.py

@@ -53,7 +53,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastSub.py

@@ -53,7 +53,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastT.py

@@ -57,7 +57,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastTO.py

@@ -57,7 +57,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importLastTPO.py

@@ -59,7 +59,7 @@ class TDTestCase:
 
         tdLog.info("================= step4")
         tdDnodes.stop(1)
-        tdLog.sleep(5)
+        #tdLog.sleep(5)
         tdDnodes.start(1)
 
         tdLog.info("================= step5")

tests/pytest/import_merge/importSRestart.py

@@ -64,7 +64,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importSubRestart.py

@@ -64,7 +64,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importTORestart.py

@@ -64,7 +64,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importTPORestart.py

@@ -68,7 +68,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/import_merge/importTRestart.py

@@ -61,7 +61,7 @@ class TDTestCase:
         tdLog.info("================= step5")
         tdDnodes.forcestop(1)
         tdDnodes.start(1)
-        tdLog.sleep(10)
+        #tdLog.sleep(10)
 
         tdLog.info("================= step6")
         tdSql.query('select * from tb1')

tests/pytest/query/query.py

@@ -16,7 +16,7 @@ import taos
 from util.log import tdLog
 from util.cases import tdCases
 from util.sql import tdSql
-
+from util.dnodes import tdDnodes
 
 class TDTestCase:
     def init(self, conn, logSql):
@@ -44,6 +44,25 @@ class TDTestCase:
         tdSql.query("select * from db.st where ts='2020-05-13 10:00:00.000'")
         tdSql.checkRows(1)
 
+        ## test case for https://jira.taosdata.com:18080/browse/TD-2488
+        tdSql.execute("create table m1(ts timestamp, k int) tags(a int)")
+        tdSql.execute("create table t1 using m1 tags(1)")
+        tdSql.execute("create table t2 using m1 tags(2)")
+        tdSql.execute("insert into t1 values('2020-1-1 1:1:1', 1)")
+        tdSql.execute("insert into t1 values('2020-1-1 1:10:1', 2)")
+        tdSql.execute("insert into t2 values('2020-1-1 1:5:1', 99)")
+        
+        tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
+        tdDnodes.stop(1)
+        tdDnodes.start(1)
+
+        tdSql.query("select count(*) from m1 where ts = '2020-1-1 1:5:1' ")
+        tdSql.checkRows(1)
+        tdSql.checkData(0, 0, 1)
+
     def stop(self):
         tdSql.close()
         tdLog.success("%s successfully executed" % __file__)

tests/pytest/table/alter_wal0.py

@@ -43,7 +43,8 @@ class TDTestCase:
 
         print("==============step2")
         tdDnodes.stopAll()
-        filename = '/var/lib/taos/mnode/wal/wal0'
+        path = tdDnodes.getDnodesRootDir()        
+        filename = path + '/dnode1/data/mnode/wal/wal0'
 
         with open(filename, 'rb') as f1:
             temp = f1.read()
@@ -57,12 +58,12 @@ class TDTestCase:
         print("==============step3")
         tdSql.execute("use demo;")
         tdSql.query('show tables;')
-        tdSql.checkRows(10)
+        tdSql.checkRows(9)
         for i in range(11,21):
             tdSql.execute("CREATE table if not exists test{num} using meters tags({num});".format(num=i))
 
         tdSql.query('show tables;')
-        tdSql.checkRows(20)
+        tdSql.checkRows(19)
         print("==============check table numbers and create 10 tables")
 
 

tests/script/general/db/alter_tables_d2.sim

@@ -11,11 +11,26 @@ system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 5
 
 system sh/exec.sh -n dnode1 -s start
 system sh/exec.sh -n dnode2 -s start
-
-sleep 3000
 sql connect
 sql create dnode $hostname2
-sleep 1000
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
 
 print ============================ step1
 
@@ -80,10 +95,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10
 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 10
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step1
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step2
+endi
 
 sql create table db.t100 using db.st tags(0)
 sql create table db.t101 using db.st tags(1)
@@ -110,7 +138,6 @@ if $rows != 40 then
   return -1
 endi
 
-
 sql insert into db.t100 values(now, 1)
 sql insert into db.t101 values(now, 1)
 sql insert into db.t102 values(now, 1)
@@ -144,10 +171,24 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15
 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 15
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
+
+$x = 0
+step3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step3
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step3
+endi
 
 sql create table db.t200 using db.st tags(0)
 sql create table db.t201 using db.st tags(1)
@@ -198,7 +239,7 @@ sql insert into db.t219 values(now, 1)
 print ============================ step6
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select * from db.t000
 if $rows != 1 then 
@@ -268,10 +309,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20
 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 20
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
+$x = 0
+step9: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step9
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step9
+endi
 
 sql create table db.t300 using db.st tags(0)
 sql create table db.t301 using db.st tags(1)
@@ -321,7 +375,7 @@ if $rows != 80 then
 endi
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select * from db.t000
 if $rows != 1 then 
@@ -351,13 +405,26 @@ endi
 print ============================ step10
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
+$x = 0
+step10: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step10
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step10
+endi
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql show db.tables
 if $rows != 80 then 
@@ -401,10 +468,23 @@ system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25
 system sh/cfg.sh -n dnode2 -c maxTablesPerVnode -v 25
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
+$x = 0
+step1xx: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step1xx
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1xx
+endi
 
 sql create table db.t400 using db.st tags(0)
 sql create table db.t401 using db.st tags(1)
@@ -454,7 +534,7 @@ if $rows != 100 then
 endi
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select * from db.t000
 if $rows != 1 then 

tests/script/general/db/alter_tables_v1.sim

@@ -5,8 +5,6 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 1
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10
 
 system sh/exec.sh -n dnode1 -s start
-
-sleep 3000
 sql connect
 
 print ============================ step1
@@ -51,9 +49,22 @@ print ============================ step3
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step2
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step2
+endi
 
 sql create table db.t10 using db.st tags(0)
 sql create table db.t11 using db.st tags(1)
@@ -91,9 +102,22 @@ endi
 print ============================ step5
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 30
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step5
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step5
+endi
 
 sql create table db.t20 using db.st tags(0)
 sql create table db.t21 using db.st tags(1)
@@ -124,7 +148,7 @@ sql insert into db.t29 values(now, 1)
 print ============================ step6
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select * from db.t0
 if $rows != 1 then 
@@ -148,9 +172,22 @@ endi
 
 print ============================ step7
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step7: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step7
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step7
+endi
 
 sql reset query cache
 sleep 1000
@@ -190,9 +227,22 @@ endi
 print ============================ step9
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 40
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step9: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step9
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step9
+endi
 
 sql create table db.t30 using db.st tags(0)
 sql create table db.t31 using db.st tags(1)
@@ -249,12 +299,24 @@ if $rows != 40 then
   return -1
 endi
 
-
 print ============================ step10
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step10: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step10
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step10
+endi
 
 sql reset query cache
 sleep 1000
@@ -294,9 +356,22 @@ endi
 print ============================ step12
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 50
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step12: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step12
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step12
+endi
 
 sql create table db.t40 using db.st tags(0)
 sql create table db.t41 using db.st tags(1)

tests/script/general/db/alter_tables_v4.sim

@@ -5,8 +5,6 @@ system sh/cfg.sh -n dnode1 -c maxVgroupsPerDb -v 4
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 5
 
 system sh/exec.sh -n dnode1 -s start
-
-sleep 3000
 sql connect
 
 print ============================ step1
@@ -70,9 +68,22 @@ endi
 print ============================ step3
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 10
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step3
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step3
+endi
 
 sql create table db.t100 using db.st tags(0)
 sql create table db.t101 using db.st tags(1)
@@ -131,9 +142,22 @@ endi
 print ============================ step5
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 15
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step5
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step5
+endi
 
 sql create table db.t200 using db.st tags(0)
 sql create table db.t201 using db.st tags(1)
@@ -184,7 +208,7 @@ sql insert into db.t219 values(now, 1)
 print ============================ step6
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select * from db.t000
 if $rows != 1 then 
@@ -250,9 +274,22 @@ endi
 print ============================ step9
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 20
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step9: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step9
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step9
+endi
 
 sql create table db.t300 using db.st tags(0)
 sql create table db.t301 using db.st tags(1)
@@ -331,9 +368,22 @@ endi
 
 print ============================ step10
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step10: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step10
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step10
+endi
 
 sql reset query cache
 sleep 1000
@@ -378,9 +428,24 @@ endi
 print ============================ step12
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
 system sh/cfg.sh -n dnode1 -c maxTablesPerVnode -v 25
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+step12: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show mnodes -x step12
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step12
+endi
+
+
 sql create table db.t400 using db.st tags(0)
 sql create table db.t401 using db.st tags(1)
 sql create table db.t402 using db.st tags(2)

tests/script/general/db/delete_reusevnode.sim

@@ -5,7 +5,6 @@ system sh/deploy.sh -n dnode1 -i 1
 
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 
 print ======== step1 
@@ -38,7 +37,6 @@ if $rows != 0 then
 endi 
 
 system sh/stop_dnodes.sh
-sleep 3000
 
 system sh/deploy.sh -n dnode1 -i 1
 print ========= start dnodes

tests/script/general/db/delete_writing1.sim

@@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
 
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 
 sql create database db 

tests/script/general/parser/mixed_blocks.sim

@@ -144,4 +144,20 @@ if $data03 != 319 then
   return -1
 endi
 
+print ===================> TD-2488
+sql create table m1(ts timestamp, k int) tags(a int);
+sql create table t1 using m1 tags(1);
+sql create table t2 using m1 tags(2);
+sql insert into t1 values('2020-1-1 1:1:1', 1);
+sql insert into t1 values('2020-1-1 1:10:1', 2);
+sql insert into t2 values('2020-1-1 1:5:1', 99);
+
+print ================== restart server to commit data into disk
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
+sleep 3000
+system sh/exec.sh -n dnode1 -s start
+print ================== server restart completed
+sql select ts from m1 where ts='2020-1-1 1:5:1'
+if $rows != 1 then
+  return -1
+endi
\ No newline at end of file

tests/script/general/table/delete_writing.sim

@@ -21,8 +21,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
 
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
-
-sleep 3000
 sql connect
 
 sql create database db 

tests/script/general/wal/sync.sim

@@ -31,7 +31,6 @@ system sh/cfg.sh -n dnode3 -c maxSQLLength -v 940032
 print ============== deploy
 
 system sh/exec.sh -n dnode1 -s start 
-sleep 5001
 sql connect
 
 sql create dnode $hostname2
@@ -43,8 +42,8 @@ print  =============== step1
 $x = 0
 show1: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 5 then
+	sleep 1000
+	if $x == 10 then
 	  return -1
 	endi
 sql show mnodes -x show1
@@ -82,7 +81,7 @@ restful d1 table_rest 1591772800 30000
 restful d1 table_rest 1591872800 30000
 restful d1 table_rest 1591972800 30000
 
-sleep 1000
+sleep 100
 sql select * from table_rest;
 print rows: $rows
 if $rows != 300000 then
@@ -91,29 +90,51 @@ endi
 
 print ===============  step3
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
 sql select * from table_rest;
 print rows: $rows
 if $rows != 300000 then
   return -1
 endi
 system sh/exec.sh -n dnode1 -s start -x SIGINT
-sleep 5000
+
+$x = 0
+a1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a1
+endi
 
 print ===============  step4
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 sql select * from table_rest;
 print rows: $rows
 if $rows != 300000 then
   return -1
 endi
 system sh/exec.sh -n dnode2 -s start -x SIGINT
-sleep 5000
+$x = 0
+a2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a2
+endi
 
 print ===============  step5
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
-sleep 5000
 sql select * from table_rest;
 print rows: $rows
 if $rows != 300000 then

tests/script/jenkins/basic.txt

@@ -288,6 +288,7 @@ cd ../../../debug; make
 ./test.sh -f unique/dnode/data1.sim
 ./test.sh -f unique/dnode/offline1.sim
 ./test.sh -f unique/dnode/offline2.sim
+./test.sh -f unique/dnode/offline3.sim
 ./test.sh -f unique/dnode/reason.sim
 ./test.sh -f unique/dnode/remove1.sim
 ./test.sh -f unique/dnode/remove2.sim

tests/script/jenkins/basic_3.txt

 
-./test.sh -f unique/stable/balance_replica1.sim
-./test.sh -f unique/stable/dnode2_stop.sim
-./test.sh -f unique/stable/dnode2.sim
-./test.sh -f unique/stable/dnode3.sim
-./test.sh -f unique/stable/replica2_dnode4.sim
-./test.sh -f unique/stable/replica2_vnode3.sim
-./test.sh -f unique/stable/replica3_dnode6.sim
-./test.sh -f unique/stable/replica3_vnode3.sim
-
-./test.sh -f unique/mnode/mgmt20.sim
-./test.sh -f unique/mnode/mgmt21.sim
-./test.sh -f unique/mnode/mgmt22.sim
-./test.sh -f unique/mnode/mgmt23.sim
-./test.sh -f unique/mnode/mgmt24.sim
-#./test.sh -f unique/mnode/mgmt25.sim
-#./test.sh -f unique/mnode/mgmt26.sim
-./test.sh -f unique/mnode/mgmt33.sim
-./test.sh -f unique/mnode/mgmt34.sim
-./test.sh -f unique/mnode/mgmtr2.sim
-
-
-
-./test.sh -f general/parser/stream_on_sys.sim
-./test.sh -f general/stream/metrics_del.sim
-./test.sh -f general/stream/metrics_n.sim
-./test.sh -f general/stream/metrics_replica1_vnoden.sim
-./test.sh -f general/stream/restart_stream.sim
-./test.sh -f general/stream/stream_3.sim
-./test.sh -f general/stream/stream_restart.sim
-./test.sh -f general/stream/table_1.sim
-./test.sh -f general/stream/table_del.sim
-./test.sh -f general/stream/table_n.sim
-./test.sh -f general/stream/table_replica1_vnoden.sim
-
 ./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim
 #./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim
@@ -79,3 +45,35 @@
 ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim
 ./test.sh -f unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim
 ./test.sh -f unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim
+
+./test.sh -f unique/stable/balance_replica1.sim
+./test.sh -f unique/stable/dnode2_stop.sim
+./test.sh -f unique/stable/dnode2.sim
+./test.sh -f unique/stable/dnode3.sim
+./test.sh -f unique/stable/replica2_dnode4.sim
+./test.sh -f unique/stable/replica2_vnode3.sim
+./test.sh -f unique/stable/replica3_dnode6.sim
+./test.sh -f unique/stable/replica3_vnode3.sim
+
+./test.sh -f unique/mnode/mgmt20.sim
+./test.sh -f unique/mnode/mgmt21.sim
+./test.sh -f unique/mnode/mgmt22.sim
+./test.sh -f unique/mnode/mgmt23.sim
+./test.sh -f unique/mnode/mgmt24.sim
+#./test.sh -f unique/mnode/mgmt25.sim
+#./test.sh -f unique/mnode/mgmt26.sim
+./test.sh -f unique/mnode/mgmt33.sim
+./test.sh -f unique/mnode/mgmt34.sim
+./test.sh -f unique/mnode/mgmtr2.sim
+
+./test.sh -f general/parser/stream_on_sys.sim
+./test.sh -f general/stream/metrics_del.sim
+./test.sh -f general/stream/metrics_n.sim
+./test.sh -f general/stream/metrics_replica1_vnoden.sim
+./test.sh -f general/stream/restart_stream.sim
+./test.sh -f general/stream/stream_3.sim
+./test.sh -f general/stream/stream_restart.sim
+./test.sh -f general/stream/table_1.sim
+./test.sh -f general/stream/table_del.sim
+./test.sh -f general/stream/table_n.sim
+./test.sh -f general/stream/table_replica1_vnoden.sim

tests/script/jenkins/unique.txt

@@ -31,6 +31,8 @@ cd ../../../debug; make
 ./test.sh -f unique/dnode/balancex.sim
 ./test.sh -f unique/dnode/offline1.sim
 ./test.sh -f unique/dnode/offline2.sim
+./test.sh -f unique/dnode/offline3.sim
+./test.sh -f unique/dnode/reason.sim
 ./test.sh -f unique/dnode/remove1.sim
 ./test.sh -f unique/dnode/remove2.sim
 ./test.sh -f unique/dnode/vnode_clean.sim

tests/script/sh/deploy.sh

@@ -132,10 +132,12 @@ echo "cqdebugFlag            143"                >> $TAOS_CFG
 echo "monitor                0"                  >> $TAOS_CFG
 echo "monitorInterval        1"                  >> $TAOS_CFG
 echo "http                   0"                  >> $TAOS_CFG
+echo "slaveQuery             0"                  >> $TAOS_CFG
 echo "numOfThreadsPerCore    2.0"                >> $TAOS_CFG
 echo "defaultPass            taosdata"           >> $TAOS_CFG
 echo "numOfLogLines          20000000"           >> $TAOS_CFG
 echo "mnodeEqualVnodeNum     0"                  >> $TAOS_CFG
+echo "balanceInterval        1"                  >> $TAOS_CFG
 echo "clog                   2"                  >> $TAOS_CFG
 #echo "cache                 1"                  >> $TAOS_CFG
 echo "days                   10"                  >> $TAOS_CFG

tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim

@@ -360,12 +360,14 @@ print ============== step7: stop dnode3/dnode2, and cluster unable to provide se
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
 sleep 3000
-sql_error select count(*) from $stb
+sql select count(*) from $stb -x s71
+s71:
 
 print ============== step8: restart dnode2, and cluster Still unable to provide services
 system sh/exec.sh -n dnode2 -s start
 sleep 3000
-sql_error select count(*) from $stb
+sql select count(*) from $stb -x s81
+s81:
 
 print ============== step9: restart dnode3, and cluster Resume service delivery
 system sh/exec.sh -n dnode3 -s start

tests/script/unique/arbitrator/dn3_mn1_replica_change.sim

@@ -45,15 +45,30 @@ system sh/exec_tarbitrator.sh -s start
 
 print ============== step1: start dnode1, only deploy mnode
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 
 print ============== step2: start dnode2 and add into cluster , then create database with replica 1, and create table, insert data
 system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname2
-sleep 3000
 
-$sleepTimer = 10000
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
 
 $db = db
 sql create database $db replica 1
@@ -84,7 +99,6 @@ while $i < $tblNum
 endw 
 
 sql select count(*) from $stb
-sleep 1000
 print data00 $data00
 if $data00 != $totalRows then
 	return -1
@@ -92,118 +106,65 @@ endi
 
 print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep $sleepTimer
-
-
-$loopCnt = 0
-wait_dnode2_offline_0:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
-  return -1
-endi
-sql show dnodes
-if $rows != 2 then
-  sleep 2000
-  goto wait_dnode2_offline_0
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-#$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode2Status != offline then
-  sleep 2000
-  goto wait_dnode2_offline_0
-endi
-
 system sh/exec.sh -n dnode2 -s start
-sleep $sleepTimer
-
 
-$loopCnt = 0
-wait_dnode2_reready:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
+$x = 0
+a0: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
 	  return -1
-endi
-sql show dnodes
-if $rows != 2 then
-  sleep 2000
-  goto wait_dnode2_reready
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-#$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode2Status != ready then
-  sleep 2000
-  goto wait_dnode2_reready
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 1 then
+	goto a0
 endi
 
 sql select count(*) from $stb
-sleep 1000
 print data00 $data00
 if $data00 != $totalRows then
 	return -1
 endi
 
-
 print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname3
-sleep 3000
 
-sql alter database $db replica 2
-sleep $sleepTimer
-$loopCnt = 0
-wait_dnode3_ready:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
 		return -1
-endi
+	endi
 
 sql show dnodes
-if $rows != 3 then
-  sleep 2000
-  goto wait_dnode3_ready
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode2Status != ready then
-  sleep 2000
-  goto wait_dnode3_ready
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_3 != ready then
+  goto step2
 endi
-if $dnode3Status != ready then
-  sleep 2000
-  goto wait_dnode3_ready
+
+sql alter database $db replica 2
+
+$x = 0
+a1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
 endi
 
-sleep $sleepTimer
 # check using select
 sql select count(*) from $stb
 print data00 $data00
@@ -211,44 +172,22 @@ if $data00 != $totalRows then
 	return -1
 endi
 
-
 print ============== step4: stop dnode2 for checking if sync ok
 system sh/exec.sh -n dnode2 -s stop
-sleep $sleepTimer
-$loopCnt = 0
-wait_dnode2_offline:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
+$x = 0
+a2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
 	  return -1
-endi
+	endi
 
-sql show dnodes
-if $rows != 3 then
-  sleep 2000
-  goto wait_dnode2_offline
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode2Status != offline then
-  sleep 2000
-  goto wait_dnode2_offline
-endi
-if $dnode3Status != ready then
-  sleep 2000
-  goto wait_dnode2_offline
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 1 then
+	goto a2
 endi
 
-sleep $sleepTimer   # waitting for move master vnode of dnode2 to dnode3
 # check using select
 sql select count(*) from $stb
 print data00 $data00
@@ -258,38 +197,20 @@ endi
 
 print ============== step5: restart dnode2
 system sh/exec.sh -n dnode2 -s start
-sleep 3000
-
-$loopCnt = 0
-wait_dnode2_ready:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
+$x = 0
+a3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
 	  return -1
-endi
+	endi
 
-sql show dnodes
-if $rows != 3 then
-  sleep 2000
-  goto wait_dnode2_ready
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode2Status != ready then
-  sleep 2000
-  goto wait_dnode2_ready
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a3
 endi
 
-sleep $sleepTimer
 # check using select
 sql select count(*) from $stb
 print data00 $data00
@@ -300,40 +221,39 @@ endi
 print ============== step6: start dnode4 and add into cluster , then alter replica from 2 to 3, and waiting sync
 system sh/exec.sh -n dnode4 -s start
 sql create dnode $hostname4
-sleep 3000
-
-sql alter database $db replica 3
-sleep $sleepTimer
-$loopCnt = 0
-wait_dnode4_ready:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
 		return -1
-endi
+	endi
 
 sql show dnodes
-if $rows != 4 then
-  sleep 2000
-  goto wait_dnode4_ready
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_4 != ready then
+  goto step6
 endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-#$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode4Status != ready then
-  sleep 2000
-  goto wait_dnode4_ready
+
+sql alter database $db replica 3
+$x = 0
+a4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a4
 endi
 
-sleep $sleepTimer
 # check using select
 sql select count(*) from $stb
 print data00 $data00
@@ -343,44 +263,20 @@ endi
 
 print ============== step7: alter replica from 3 to 2, and waiting sync
 sql alter database $db replica 2
-sleep $sleepTimer
-$loopCnt = 0
-wait_vgroups_replic_to_2:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
+$x = 0
+a5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
 	  return -1
-endi
+	endi
 
 sql show vgroups 
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1  $data10_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2  $data10_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3  $data10_3
-print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4  $data5_4  $data6_4  $data7_4  $data8_4  $data9_4  $data10_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-$thirdDnode_2 = $data8_1 
-$thirdDnode_3 = $data8_2 
-$thirdDnode_4 = $data8_3 
-$thirdDnode_5 = $data8_4 
-
-if $thirdDnode_2 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_2
-endi
-if $thirdDnode_3 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_2
-endi
-if $thirdDnode_4 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_2
-endi
-if $thirdDnode_5 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_2
+print online vnodes $data03
+if $data03 != 2 then
+	goto a5
 endi
 
-sleep $sleepTimer  #waiting del one replica data
 # check using select
 sql select count(*) from $stb
 print data00 $data00
@@ -390,85 +286,20 @@ endi
 
 print ============== step8: alter replica from 2 to 1, and waiting sync
 sql alter database $db replica 1
-sleep $sleepTimer
-$loopCnt = 0
-wait_vgroups_replic_to_1:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
+$x = 0
+a6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
 	  return -1
-endi
+	endi
 
 sql show vgroups 
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3
-print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4  $data5_4  $data6_4  $data7_4  $data8_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-$sencodDnode_2 = $data6_1 
-$sencodDnode_3 = $data6_2 
-$sencodDnode_4 = $data6_3 
-$sencodDnode_5 = $data6_4 
-
-if $sencodDnode_2 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_1
-endi
-if $sencodDnode_3 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_1
-endi
-if $sencodDnode_4 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_1
-endi
-if $sencodDnode_5 != null then
-  sleep 2000
-  goto wait_vgroups_replic_to_1
-endi
-
-$loopCnt = 0
-all_dnodes_ready:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
-  return -1
-endi
-
-sql show dnodes
-if $rows != 4 then
-  sleep 2000
-  goto all_dnodes_ready
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode1Status != ready then
-  sleep 2000
-  goto all_dnodes_ready
-endi
-if $dnode2Status != ready then
-  sleep 2000
-  goto all_dnodes_ready
-endi
-if $dnode3Status != ready then
-  sleep 2000
-  goto all_dnodes_ready
-endi
-if $dnode4Status != ready then
-  sleep 2000
-  goto all_dnodes_ready
+print online vnodes $data03
+if $data03 != 1 then
+	goto a6
 endi
 
-sleep $sleepTimer  #waiting del one replica data
 # check using select
 sql select count(*) from $stb
 print data00 $data00
@@ -481,39 +312,31 @@ sql drop dnode $hostname2
 sql drop dnode $hostname3
 sleep $sleepTimer
 
-$loopCnt = 0
-wait_dnode23_dropped:
-$loopCnt = $loopCnt + 1
-if $loopCnt == 10 then 
+$x = 0
+step9: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
 	  return -1
-endi
+	endi
 
 sql show dnodes
 if $rows != 2 then
-  sleep 2000
-  goto wait_dnode23_dropped
+  goto step9
 endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-$dnode4Status = $data4_4 
-
-if $dnode2Status != null then
-  sleep 2000
-  goto wait_dnode23_dropped
-endi
-if $dnode3Status != null then
-  sleep 2000
-  goto wait_dnode23_dropped
-endi
-if $dnode4Status != ready then
+
+$x = 0
+a7: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
 	  return -1
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 1 then
+	goto a7
 endi
 
 sleep $sleepTimer  #waiting move vnode from dnode3/dnode3 to dnode4
@@ -523,3 +346,13 @@ print data00 $data00
 if $data00 != $totalRows then
 	return -1
 endi
+
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+system sh/exec.sh -n dnode5 -s stop  -x SIGINT
+system sh/exec.sh -n dnode6 -s stop  -x SIGINT
+system sh/exec.sh -n dnode7 -s stop  -x SIGINT
+system sh/exec.sh -n dnode8 -s stop  -x SIGINT

tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim

@@ -140,3 +140,11 @@ if $data00 != $totalRows then
 	return -1
 endi
 
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT
+system sh/exec.sh -n dnode5 -s stop  -x SIGINT
+system sh/exec.sh -n dnode6 -s stop  -x SIGINT
+system sh/exec.sh -n dnode7 -s stop  -x SIGINT
+system sh/exec.sh -n dnode8 -s stop  -x SIGINT

tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim

@@ -158,13 +158,15 @@ if $dnode4Vtatus != offline then
   sleep 2000
   goto wait_dnode4_vgroup_offline
 endi
-if $dnode3Vtatus != unsynced then
+if $dnode3Vtatus != master then
   sleep 2000
   goto wait_dnode4_vgroup_offline
 endi
 
-sql_error select count(*) from $stb
-sql_error insert into $tb values (now, 9988)
+sql select count(*) from $stb -x s31
+s31:
+#sql_error insert into $tb values (now, 9988) -x s32
+#s32:
 
 print ============== step4: restart dnode2, then create database with replica 2, and create table, insert data
 system sh/exec.sh -n dnode2 -s start

tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim

@@ -140,11 +140,9 @@ if $rows != 1 then
   goto wait_dnode4_vgroup_offline
 endi
 print show vgroups:
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1  $data9_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2  $data9_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3  $data5_3  $data6_3  $data7_3  $data8_3  $data9_3
-$dnode4Vtatus = $data5_2
-$dnode3Vtatus = $data7_2
+print $data00  $data01  $data02  $data03  $data04  $data05  $data06  $data07  $data08  $data09
+$dnode4Vtatus = $data05
+$dnode3Vtatus = $data07
 
 if $dnode4Vtatus != offline then
   sleep 2000
@@ -198,7 +196,3 @@ if $data00 != $totalRows then
 	sleep 2000
 	goto wait_table_altered
 endi
-
-
-
-

tests/script/unique/arbitrator/sync_replica_alterTable_background_add.sim

@@ -14,6 +14,9 @@ sql alter table $stb add column f1 float
 $tblNum = $totalTableNum
 $alterTblNum = 10
 
+sql reset query cache 
+sleep 100
+
 $i = 1
 while $i < $alterTblNum
   $tb = tb . $i

tests/script/unique/big/balance.sim

@@ -21,7 +21,6 @@ system sh/cfg.sh -n dnode5 -c maxTablesPerVnode -v 1000
 
 print =============== prepare data
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 
 $i = 0
@@ -92,8 +91,8 @@ system sh/exec.sh -n dnode2 -s start
 $x = 0
 show1: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 30 then 
 		return -1
 	endi
 	
@@ -108,7 +107,7 @@ if $data2_2 != 2 then
 endi
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select count(*) from t10
 print select count(*) from t10 $data00 expect $rowNum 
@@ -143,7 +142,6 @@ endi
 print ========== step2
 sql create dnode $hostname3
 system sh/exec.sh -n dnode3 -s start
-sleep 10000
 
 print ========== step3
 sql drop dnode $hostname2
@@ -151,8 +149,8 @@ sql drop dnode $hostname2
 $x = 0
 show3: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 30 then 
 		return -1
 	endi
 	
@@ -171,10 +169,9 @@ if $data2_3 != 2 then
 endi
 
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select count(*) from t10
 print select count(*) from t10 $data00 expect $rowNum 
@@ -212,8 +209,8 @@ sql drop dnode $hostname3
 $x = 0
 show4: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 30 then 
 		return -1
 	endi
 	
@@ -228,10 +225,9 @@ if $data2_3 != null then
 endi
 
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
-sleep 5000
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select count(*) from t10
 print select count(*) from t10 $data00 expect $rowNum 
@@ -267,14 +263,31 @@ print ========== step5
 system sh/exec.sh -n dnode4 -s start
 sql create dnode $hostname4
 
-sleep 3000
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_4 != ready then
+  goto step5
+endi
+
 sql alter database db replica 2
 
 $x = 0
 show5: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 30 then 
 		return -1
 	endi
 	
@@ -289,7 +302,7 @@ if $data2_4 != 4 then
 endi
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select count(*) from t10
 print select count(*) from t10 $data00 expect $rowNum 
@@ -321,16 +334,14 @@ if $data00 != $totalNum then
   goto show5
 endi
 
-
 print ========== step6
-sleep 3000
 sql alter database db replica 1
 
 $x = 0
 show6: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 30 then 
 		return -1
 	endi
 	
@@ -345,7 +356,7 @@ if $data2_4 != 2 then
 endi
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select count(*) from t10
 print select count(*) from t10 $data00 expect $rowNum 

tests/script/unique/big/tcp.sim

@@ -15,7 +15,6 @@ system sh/cfg.sh -n dnode1 -c httpDebugFlag   -v 135
 system sh/cfg.sh -n dnode1 -c debugFlag       -v 131
 
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 print ======================== dnode1 start
 
@@ -52,7 +51,7 @@ while $i < $tbNum
 endw 
 
 $i = 0
-while $i < 10
+while $i < 5
 	print =============== step3 $i
 	sql select count(*) from $mt
 	print ===> $data00 $data01
@@ -83,7 +82,7 @@ while $i < $tbNum
 endw 
 
 $i = 0
-while $i < 10
+while $i < 5
 	print =============== step5 $i
 	sql select count(*) from $mt where tgcol < 20200
 	print ===> $data00 $data01

tests/script/unique/cluster/balance1.sim

@@ -84,16 +84,14 @@ endi
 
 print ============================== step2
 print ========= start dnode2
-sleep 2000
 sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
 
 $x = 0
 show2: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show2
@@ -129,13 +127,12 @@ sql insert into c_b1_t3 values(1520000024031, 31)
 print ============================== step4
 print ========= drop dnode2
 sql drop dnode $hostname2
-sleep 9000
 
 $x = 0
 show4: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show4
@@ -164,16 +161,14 @@ print dnode4 ==> $dnode4Role
 print ============================== step5
 print ========= add dnode2
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname3
-sleep 9000
 
 $x = 0
 show5: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then 
+	sleep 1000
+	if $x == 40 then 
 	  return -1
 	endi
 sql show dnodes -x show5
@@ -212,13 +207,12 @@ sql_error create dnode $hostname1
 
 print ============================== step8
 sql drop dnode $hostname3
-sleep 15000
 
 $x = 0
 show8: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 30 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show8
@@ -253,13 +247,12 @@ endi
 print ============================== step9
 sql create dnode $hostname4
 system sh/exec.sh -n dnode4 -s start
-sleep 9000
 
 $x = 0
 show9: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 30 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show9
@@ -302,8 +295,8 @@ sql insert into c_b1_t4 values(1520000024041, 41)
 $x = 0
 show10: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 30 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show10
@@ -335,7 +328,23 @@ sql_error create table c_b1_t5 (t timestamp, i int) -x error3
 print ============================== step13
 sql create dnode $hostname5
 system sh/exec.sh -n dnode5 -s start
-sleep 9000
+$x = 0
+step13: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+print dnode4 $data4_5
+
+if $data4_5 != ready then
+  goto step13
+endi  
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -383,7 +392,23 @@ endi
 print ============================== step14
 sql create dnode $hostname6
 system sh/exec.sh -n dnode6 -s start
-sleep 15000
+$x = 0
+step14: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+print dnode4 $data4_5
+
+if $data4_6 != ready then
+  goto step14
+endi  
 
 sql create database c_b1_d7 
 sql use c_b1_d7
@@ -406,7 +431,7 @@ sql insert into c_b1_t8 values(1520000024081, 81)
 $x = 0
 show14: 
 	$x = $x + 1
-	sleep 2000
+	sleep 1000
 	if $x == 30 then
 	  return -1
 	endi
@@ -440,7 +465,7 @@ print ============================== step17
 print ========= check data
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql use c_b1_d1
 sql select * from c_b1_d1.c_b1_t1

tests/script/unique/cluster/balance2.sim

@@ -50,12 +50,48 @@ print ========= start dnode1
 system sh/exec.sh -n dnode1 -s start
 
 sql connect
-sleep 4001
 sql create dnode $hostname2
 sql create dnode $hostname3
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
-sleep 4001
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
+if $data2_2 != slave then
+  goto step1
+endi
+if $data2_3 != slave then
+  goto step1
+endi
+
 
 sql create database c_b2_d1 replica 2 
 sql use c_b2_d1
@@ -112,13 +148,12 @@ endi
 print ============================== step2
 print ========= drop dnode2
 sql drop dnode $hostname2
-sleep 9000
 
 $x = 0
 show2: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 20 then 
+	sleep 1000
+	if $x == 30 then 
 	  return -1
 	endi
 sql show dnodes -x show2
@@ -155,13 +190,12 @@ print ============================== step3
 print ========= start dnode4
 sql create dnode $hostname4
 system sh/exec.sh -n dnode4 -s start
-sleep 10000
 
 $x = 0
 show3: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 20 then 
+	sleep 1000
+	if $x == 30 then 
 	  return -1
 	endi
 sql show dnodes -x show3
@@ -209,13 +243,12 @@ endi
 print ============================== step4
 print ========= drop dnode3
 sql drop dnode $hostname3
-sleep 9000
 
 $x = 0
 show4: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 20 then 
+	sleep 1000
+	if $x == 40 then 
 	  return -1
 	endi
 sql show dnodes -x show4
@@ -266,12 +299,11 @@ print ============================== step5
 print ========= start dnode3
 sql create dnode $hostname5
 system sh/exec.sh -n dnode5 -s start
-sleep 9000
 
 $x = 0
 show5: 
 	$x = $x + 1
-	sleep 2000
+	sleep 1000
 	if $x == 30 then
 	  return -1
 	endi
@@ -304,7 +336,7 @@ print dnode5 ==> $dnode5Role
 print ============================== step6
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 print stop dnode1 and sleep 10000
-sleep 10000
+sleep 5000
 
 sql drop dnode $hostname1
 print drop dnode1 and sleep 9000
@@ -364,7 +396,7 @@ print ============================== step12
 print ========= check data
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql select * from c_b2_d1.c_b2_t1 order by t desc
 print $data01 $data11 $data21 $data31 $data41

tests/script/unique/cluster/balance3.sim

@@ -7,7 +7,6 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/deploy.sh -n dnode5 -i 5
 system sh/deploy.sh -n dnode6 -i 6
 system sh/deploy.sh -n dnode7 -i 7
-system sh/deploy.sh -n dnode8 -i 8
 
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
@@ -16,7 +15,6 @@ system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3
 system sh/cfg.sh -n dnode5 -c numOfMnodes -v 3
 system sh/cfg.sh -n dnode6 -c numOfMnodes -v 3
 system sh/cfg.sh -n dnode7 -c numOfMnodes -v 3
-system sh/cfg.sh -n dnode8 -c numOfMnodes -v 3
 
 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 0
 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 0
@@ -25,7 +23,6 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 0
 system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 0
 system sh/cfg.sh -n dnode6 -c mnodeEqualVnodeNum -v 0
 system sh/cfg.sh -n dnode7 -c mnodeEqualVnodeNum -v 0
-system sh/cfg.sh -n dnode8 -c mnodeEqualVnodeNum -v 0
 
 system sh/cfg.sh -n dnode1 -c wallevel -v 1
 system sh/cfg.sh -n dnode2 -c wallevel -v 1
@@ -34,19 +31,52 @@ system sh/cfg.sh -n dnode4 -c wallevel -v 1
 system sh/cfg.sh -n dnode5 -c wallevel -v 1
 system sh/cfg.sh -n dnode6 -c wallevel -v 1
 system sh/cfg.sh -n dnode7 -c wallevel -v 1
-system sh/cfg.sh -n dnode8 -c wallevel -v 1
 
 print ============== step1
 print ========= start dnode1
 system sh/exec.sh -n dnode1 -s start
 sql connect
-sleep 2001
-
 sql create dnode $hostname2
 sql create dnode $hostname3
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
-sleep 3001
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
+if $data2_2 != slave then
+  goto step1
+endi
+if $data2_3 != slave then
+  goto step1
+endi
 
 sql create database c_b3_d1 replica 3
 sql use c_b3_d1
@@ -89,8 +119,6 @@ $dnode2Vnodes = $data2_2
 print dnode2 $dnode2Vnodes
 $dnode3Vnodes = $data2_3
 print dnode3 $dnode3Vnodes
-$dnode4Vnodes = $data2_4
-print dnode4 $dnode4Vnodes
 
 if $dnode1Vnodes != 3 then
   goto show1
@@ -101,30 +129,22 @@ endi
 if $dnode3Vnodes != 3 then
   goto show1
 endi
-if $dnode4Vnodes != null then
-  goto show1
-endi
 
 sql show mnodes
 print dnode1 ==> $data2_1
 print dnode2 ==> $data2_2
 print dnode3 ==> $data2_3
-print dnode4 ==> $data2_4
-print dnode5 ==> $data2_5
-print dnode6 ==> $data2_6
-print dnode7 ==> $data2_7
 
 print ============================== step2
 print ========= start dnode4
 sql create dnode $hostname4
 system sh/exec.sh -n dnode4 -s start
-sleep 9000
 
 $x = 0
 show2: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show2
@@ -145,21 +165,16 @@ sql show mnodes
 print dnode1 ==> $data2_1
 print dnode2 ==> $data2_2
 print dnode3 ==> $data2_3
-print dnode4 ==> $data2_4
-print dnode5 ==> $data2_5
-print dnode6 ==> $data2_6
-print dnode7 ==> $data2_7
 
 print ============================== step3
 print ========= drop dnode2
 sql drop dnode $hostname2
-sleep 9000
 
 $x = 0
 show3: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show3
@@ -189,25 +204,21 @@ sql show mnodes
 print dnode1 ==> $data2_1
 print dnode2 ==> $data2_2
 print dnode3 ==> $data2_3
-print dnode4 ==> $data2_4
-print dnode5 ==> $data2_5
-print dnode6 ==> $data2_6
-print dnode7 ==> $data2_7
 
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 
 print ============================== step4
 sql create dnode $hostname5
 system sh/exec.sh -n dnode5 -s start
-sleep 10000
 
 $x = 0
 show4: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
+
 sql show dnodes -x show4
 $dnode1Vnodes = $data2_1
 print dnode1 $dnode1Vnodes
@@ -228,8 +239,10 @@ print dnode2 ==> $data2_2
 print dnode3 ==> $data2_3
 print dnode4 ==> $data2_4
 print dnode5 ==> $data2_5
-print dnode6 ==> $data2_6
-print dnode7 ==> $data2_7
+
+if $data2_4 != slave then
+  goto show4
+endi
 
 print ============================== step5
 print ========= drop dnode3
@@ -239,8 +252,8 @@ sleep 9000
 $x = 0
 show5: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show5
@@ -277,16 +290,19 @@ print dnode5 ==> $data2_5
 print dnode6 ==> $data2_6
 print dnode7 ==> $data2_7
 
+if $data2_5 != slave then
+  goto show5
+endi
+
 print ============================== step6
 sql create dnode $hostname6
 system sh/exec.sh -n dnode6 -s start
-sleep 9000
 
 $x = 0
 show6: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show6
@@ -299,6 +315,15 @@ print dnode5 $dnode5Vnodes
 $dnode6Vnodes = $data2_6
 print dnode6 $dnode6Vnodes
 
+if $dnode1Vnodes != 2 then
+  goto show6
+endi
+if $dnode4Vnodes != 2 then
+  goto show6
+endi
+if $dnode5Vnodes != 3 then
+  goto show6
+endi
 if $dnode6Vnodes != 2 then
   goto show6
 endi
@@ -315,13 +340,12 @@ print dnode7 ==> $data2_7
 print ============================== step7
 print ========= drop dnode4
 sql drop dnode $hostname4
-sleep 9000
 
 $x = 0
 show7: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show7
@@ -357,16 +381,19 @@ print dnode5 ==> $data2_5
 print dnode6 ==> $data2_6
 print dnode7 ==> $data2_7
 
+if $data2_6 != slave then
+  goto show7
+endi
+
 print ============================== step8
 sql create dnode $hostname7
 system sh/exec.sh -n dnode7 -s start
-sleep 9000
 
 $x = 0
 show8: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 sql show dnodes -x show8
@@ -379,6 +406,15 @@ print dnode6 $dnode6Vnodes
 $dnode7Vnodes = $data2_7
 print dnode7 $dnode7Vnodes
 
+if $dnode1Vnodes != 2 then
+  goto show8
+endi
+if $dnode5Vnodes != 2 then
+  goto show8
+endi
+if $dnode6Vnodes != 3 then
+  goto show8
+endi
 if $dnode7Vnodes != 2 then
   goto show8
 endi
@@ -393,19 +429,17 @@ print dnode6 ==> $data2_6
 print dnode7 ==> $data2_7
 
 print ============================== step9
-print ========= drop dnode1
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
-print stop dnode1 and sleep 10000
-sleep 10000
 
-sql drop dnode $hostname1
-print drop dnode1 and sleep 9000
-sleep 9000
+$x = 0
+show9: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+sql show mnodes -x show9
 
-sql show mnodes
-$dnode1Role = $data2_1
-$dnode4Role = $data2_4
-$dnode5Role = $data2_5
 print dnode1 ==> $data2_1
 print dnode2 ==> $data2_2
 print dnode3 ==> $data2_3
@@ -414,26 +448,27 @@ print dnode5 ==> $data2_5
 print dnode6 ==> $data2_6
 print dnode7 ==> $data2_7
 
-if $dnode1Role != offline then
-  return -1
+if $data2_1 != offline then
+  goto show9
+endi
+if $data2_5 != master then
+  goto show9
+endi
+if $data2_6 != slave then
+  goto show9
 endi
 
-print ============================== step9.1
-sleep 2000
-system sh/exec.sh -n dnode1 -s start
-
+print ============================== step10
+sql drop dnode $hostname1
 $x = 0
-show9: 
+show10: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
+sql show mnodes -x show10
 
-sql show mnodes
-$dnode1Role = $data2_1
-$dnode4Role = $data2_4
-$dnode5Role = $data2_5
 print dnode1 ==> $data2_1
 print dnode2 ==> $data2_2
 print dnode3 ==> $data2_3
@@ -442,7 +477,31 @@ print dnode5 ==> $data2_5
 print dnode6 ==> $data2_6
 print dnode7 ==> $data2_7
 
-sql show dnodes -x show9
+if $data2_1 != null then
+  goto show10
+endi
+if $data2_5 != master then
+  goto show10
+endi
+if $data2_6 != slave then
+  goto show10
+endi
+if $data2_7 != slave then
+  goto show10
+endi
+
+print ============================== step11
+system sh/exec.sh -n dnode1 -s start
+
+$x = 0
+show11: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show dnodes -x show11
 $dnode5Vnodes = $data2_5
 print dnode5 $dnode5Vnodes
 $dnode6Vnodes = $data2_6
@@ -451,17 +510,16 @@ $dnode7Vnodes = $data2_7
 print dnode7 $dnode7Vnodes
 
 if $dnode5Vnodes != 3 then
-  goto show9
+  goto show11
 endi
 if $dnode6Vnodes != 3 then
-  goto show9
+  goto show11
 endi
 if $dnode7Vnodes != 3 then
-  goto show9
+  goto show11
 endi
 
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
 
 sql show mnodes
 print dnode1 ==> $data2_1
@@ -472,15 +530,13 @@ print dnode5 ==> $data2_5
 print dnode6 ==> $data2_6
 print dnode7 ==> $data2_7
 
-print ============================== step11
-print ========= add db4
-
+print ============================== step12
 sql create database c_b3_d4 replica 3
 sql use c_b3_d4
 $x = 0
 create4: 
 	$x = $x + 1
-	sleep 2000
+	sleep 1000
 	if $x == 20 then
 	  return -1
 	endi
@@ -491,16 +547,14 @@ sql insert into c_b3_t4 values(1520000022043, 43)
 sql insert into c_b3_t4 values(1520000023042, 42)
 sql insert into c_b3_t4 values(1520000024041, 41)
 
-sleep 3000
-
 $x = 0
-show11: 
+show12: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
-sql show dnodes -x show11	
+sql show dnodes -x show12	
 $dnode5Vnodes = $data2_5
 print dnode5 $dnode5Vnodes
 $dnode6Vnodes = $data2_6
@@ -509,21 +563,18 @@ $dnode7Vnodes = $data2_7
 print dnode7 $dnode7Vnodes
 
 if $dnode5Vnodes != 4 then
-  goto show11
+  goto show12
 endi
 if $dnode6Vnodes != 4 then
-  goto show11
+  goto show12
 endi
 if $dnode7Vnodes != 4 then
-  goto show11
+  goto show12
 endi
 
-system sh/exec.sh -n dnode1 -s stop  -x SIGINT
-sleep 5000
-
 print ============================== step13
 sql reset query cache
-sleep 1000
+sleep 200
 
 print ========= check data
 
@@ -590,4 +641,3 @@ system sh/exec.sh -n dnode4 -s stop  -x SIGINT
 system sh/exec.sh -n dnode5 -s stop  -x SIGINT
 system sh/exec.sh -n dnode6 -s stop  -x SIGINT
 system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT

tests/script/unique/cluster/vgroup100.sim

@@ -17,7 +17,6 @@ system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 0
 
 print ============================== step1
 system sh/exec.sh -n dnode1 -s start
-sleep 2000
 sql connect
 
 print ============================== step2
@@ -27,8 +26,6 @@ system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname3
 system sh/exec.sh -n dnode3 -s start
 
-sleep 3000
-
 $maxNum = 102
 $maxNum = 12
 
@@ -92,13 +89,11 @@ print ============================== step5
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
-sleep 5000
 
 print ============================== step6
 system sh/exec.sh -n dnode1 -s start
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
-sleep 10000
 
 print ============================== step7
 

tests/script/unique/db/delete.sim

@@ -18,7 +18,6 @@ system sh/cfg.sh -n dnode3 -c maxTablesPerVnode -v 1000
 
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 sql create dnode $hostname2
 sql create dnode $hostname3
@@ -38,8 +37,6 @@ while $i < 2000
   $i = $i + 1
 endw 
 
-sleep 2500
-
 sql show db.vgroups
 if $rows != 2 then 
   return -1
@@ -73,7 +70,6 @@ system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
 
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
@@ -92,7 +88,6 @@ if $data2_1 != master then
   goto step3
 endi
 
-sleep 1000
 
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT

tests/script/unique/db/delete_part.sim

@@ -31,11 +31,29 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4
 
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
 
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+
 $loop = 0
 begin:
 
@@ -53,7 +71,6 @@ begin:
 	sql insert into t13 values(now, 1 )
 	sql create table t14 (ts timestamp, i int)
 	sql insert into t14 values(now, 1 )
-	sleep 1200
 
 	sql create table t21 (ts timestamp, i int)
 	sql insert into t21 values(now, 1 )
@@ -63,7 +80,6 @@ begin:
 	sql insert into t23 values(now, 1 )
 	sql create table t24 (ts timestamp, i int)
 	sql insert into t24 values(now, 1 )
-	sleep 1200
 
 	sql create table t31 (ts timestamp, i int)
 	sql insert into t31 values(now, 1 )
@@ -73,7 +89,6 @@ begin:
 	sql insert into t33 values(now, 1 )
 	sql create table t34 (ts timestamp, i int)
 	sql insert into t34 values(now, 1 )
-	sleep 1200
 
 	sql create table t41 (ts timestamp, i int)
 	sql insert into t41 values(now, 1 )
@@ -83,7 +98,6 @@ begin:
 	sql insert into t43 values(now, 1 )
 	sql create table t44 (ts timestamp, i int)
 	sql insert into t44 values(now, 1 )
-	sleep 1200
 
 	sql create table t51 (ts timestamp, i int)
 	sql insert into t51 values(now, 1 )
@@ -93,7 +107,6 @@ begin:
 	sql insert into t53 values(now, 1 )
 	sql create table t54 (ts timestamp, i int)
 	sql insert into t54 values(now, 1 )
-	sleep 1200
 
 	sql create table t61 (ts timestamp, i int)
 	sql insert into t61 values(now, 1 )
@@ -103,7 +116,6 @@ begin:
 	sql insert into t63 values(now, 1 )
 	sql create table t64 (ts timestamp, i int)
 	sql insert into t64 values(now, 1 )
-	sleep 1200
 
 	sql create table t71 (ts timestamp, i int)
 	sql insert into t71 values(now, 1 )
@@ -113,7 +125,6 @@ begin:
 	sql insert into t73 values(now, 1 )
 	sql create table t74 (ts timestamp, i int)
 	sql insert into t74 values(now, 1 )
-	sleep 1200
 
 	sql create table t81 (ts timestamp, i int)
 	sql insert into t81 values(now, 1 )
@@ -123,7 +134,6 @@ begin:
 	sql insert into t83 values(now, 1 )
 	sql create table t84 (ts timestamp, i int)
 	sql insert into t84 values(now, 1 )
-	sleep 1200
 
 	sql show dnodes
 	print dnode1 openVnodes $data2_1
@@ -138,29 +148,36 @@ begin:
 	print ======== step2 $loop
 	
 	system sh/exec.sh -n dnode2 -s stop
-	sleep 1000
 	print ==> drop database $db 
 	sql drop database $db 
 	
 	print ======== step3 $loop
 	sleep 2000
 	system sh/exec.sh -n dnode2 -s start
-	sleep 15000
 
-	sql show dnodes
+
+	$x = 0
+	step3: 
+		$x = $x + 1
+		sleep 1000
+		if $x == 10 then
+			return -1
+		endi
+
+	sql show dnodes -x step3
 	print dnode1 openVnodes $data2_1 $data4_1
 	print dnode2 openVnodes $data2_2 $data4_2
 	if $data2_1 != 0 then
-		return -1
+		goto step3
 	endi
 	if $data2_2 != 0 then
-		return -1
+		goto step3
 	endi
 	if $data4_1 != ready then
-		return -1
+		goto step3
 	endi
 	if $data4_2 != ready then
-		return -1
+		goto step3
 	endi
 	
 	print ===> test times : $loop
@@ -171,7 +188,7 @@ begin:
 	$loop = $loop + 1
 
 	sql reset query cache
-	sleep 1000
+	sleep 100
 	
 goto begin
   

tests/script/unique/db/replica_add12.sim

@@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
 
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 1
+
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
 sql connect
@@ -27,7 +32,43 @@ sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname3
 system sh/exec.sh -n dnode3 -s start
-sleep 3000
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
+if $data2_2 != null then
+  goto step1
+endi
+if $data2_3 != null then
+  goto step1
+endi
 
 print ======== step1 
 sql create database d1 replica 1
@@ -65,8 +106,6 @@ if $rows != 1 then
 	return -1
 endi
 
-sleep 2000
-
 sql show dnodes
 print dnode1 ==> openVnodes: $data2_1
 print dnode2 ==> openVnodes: $data2_2
@@ -89,10 +128,40 @@ sql alter database d1 replica 2
 sql alter database d2 replica 2
 sql alter database d3 replica 2
 sql alter database d4 replica 2
-sleep 10000
 
-print ======== step3
+$x = 0
+a1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
 
+print ======== step3
 sql show dnodes
 print dnode1 ==> openVnodes: $data2_1
 print dnode2 ==> openVnodes: $data2_2
@@ -115,7 +184,6 @@ sql insert into d1.t1 values(now, 2)
 sql insert into d2.t2 values(now, 2)
 sql insert into d3.t3 values(now, 2)
 sql insert into d4.t4 values(now, 2)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 2 then
@@ -138,33 +206,54 @@ if $rows != 2 then
 endi
 
 sql reset query cache
-sleep 2000
+sleep 200
 
 print ========= step5 
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
+sql select * from d1.t1 -x s51
+s51:
+#sql insert into d1.t1 values(now, 3) -x s52
+s52:
 
-sql_error select * from d1.t1
-sql_error select * from d2.t2
-sql_error select * from d3.t3
-sql_error select * from d4.t4
+print ========= step6
+system sh/exec.sh -n dnode2 -s start
 
-print ===== insert data
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
 
-sql_error insert into d1.t1 values(now, 3)
-sql_error insert into d2.t2 values(now, 3)
-sql_error insert into d3.t3 values(now, 3)
-sql_error insert into d4.t4 values(now, 3)
-sleep 1000
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
 
-print ========= step6
-system sh/exec.sh -n dnode2 -s start
-sleep 5000
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
 
-sql insert into d1.t1 values(now, 3)
-sql insert into d2.t2 values(now, 3)
-sql insert into d3.t3 values(now, 3)
-sql insert into d4.t4 values(now, 3)
+sql insert into d1.t1 values(now, 3) -x step6
+sql insert into d2.t2 values(now, 3) -x step6
+sql insert into d3.t3 values(now, 3) -x step6
+sql insert into d4.t4 values(now, 3) -x step6
 
 sql select * from d1.t1
 if $rows != 3 then
@@ -187,30 +276,52 @@ if $rows != 3 then
 endi
 
 print ========= step61
-
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
-sleep 5000
-
-sql_error insert into d1.t1 values(now, 3)
-sql_error insert into d2.t2 values(now, 3)
-sql_error insert into d3.t3 values(now, 3)
-sql_error insert into d4.t4 values(now, 3)
-sleep 1000
+#sql insert into d1.t1 values(now, 3) -x s61
+s61:
 
-sql_error select * from d1.t1
-sql_error select * from d2.t2
-sql_error select * from d3.t3
-sql_error select * from d4.t4
+sql select * from d2.t2 -x s62
+s62:
 
 print ========= step7
 system sh/exec.sh -n dnode3 -s start
-sleep 5000
 
-sql insert into d1.t1 values(now, 5)
-sql insert into d2.t2 values(now, 5)
-sql insert into d3.t3 values(now, 5)
-sql insert into d4.t4 values(now, 5)
-sleep 1000
+$x = 0
+step7: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step7
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step7
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step7
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step7
+endi
+
+sql insert into d1.t1 values(now, 5) -x step7
+sql insert into d2.t2 values(now, 5) -x step7
+sql insert into d3.t3 values(now, 5) -x step7
+sql insert into d4.t4 values(now, 5) -x step7
 
 sql select * from d1.t1
 if $rows != 4 then
@@ -236,7 +347,3 @@ system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
 system sh/exec.sh -n dnode4 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/unique/db/replica_add13.sim

@@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
 
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 1
+
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
 sql connect
@@ -29,7 +34,41 @@ sql create dnode $hostname3
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname4
 system sh/exec.sh -n dnode4 -s start
-sleep 3000
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+if $data4_4 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
 
 print ======== step1 
 sql create database d1 replica 1
@@ -46,7 +85,6 @@ sql insert into d1.t1 values(1589529000011, 1)
 sql insert into d2.t2 values(1589529000021, 1)
 sql insert into d3.t3 values(1589529000031, 1)
 sql insert into d4.t4 values(1589529000041, 1)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 1 then
@@ -73,17 +111,40 @@ sql alter database d1 replica 3
 sql alter database d2 replica 3
 sql alter database d3 replica 3
 sql alter database d4 replica 3
-sleep 10000
 
-print ======== step3
 $x = 0
-show3: 
+a1: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a1
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a1
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a1
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a1
+endi
+
+print ======== step3
 sql show dnodes
 print dnode1 ==> openVnodes: $data2_1
 print dnode2 ==> openVnodes: $data2_2
@@ -111,7 +172,6 @@ sql insert into d1.t1 values(1589529000012, 2)
 sql insert into d2.t2 values(1589529000022, 2)
 sql insert into d3.t3 values(1589529000032, 2)
 sql insert into d4.t4 values(1589529000042, 2)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 2 then
@@ -135,15 +195,13 @@ endi
 
 print ========= step5 
 sql reset query cache
-sleep 1000
+sleep 100
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 
 sql insert into d1.t1 values(1589529000013, 3)
 sql insert into d2.t2 values(1589529000023, 3)
 sql insert into d3.t3 values(1589529000033, 3)
 sql insert into d4.t4 values(1589529000043, 3)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 3 then
@@ -167,15 +225,45 @@ endi
 
 print ========= step6
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
+
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step6
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step6
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step6
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step6
+endi
+
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
-sleep 5000
 
 sql insert into d1.t1 values(1589529000014, 4)
 sql insert into d2.t2 values(1589529000024, 4)
 sql insert into d3.t3 values(1589529000034, 4)
 sql insert into d4.t4 values(1589529000044, 4)
-sleep 1000
 
 sql select * from d1.t1
 print select * from d1.t1 $rows
@@ -203,15 +291,45 @@ endi
 
 print ========= step7
 system sh/exec.sh -n dnode3 -s start
-sleep 5000
+
+$x = 0
+step7: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step7
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step7
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step7
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step7
+endi
+
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
-sleep 5000
 
 sql insert into d1.t1 values(1589529000015, 5)
 sql insert into d2.t2 values(1589529000025, 5)
 sql insert into d3.t3 values(1589529000035, 5)
 sql insert into d4.t4 values(1589529000045, 5)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 5 then
@@ -235,15 +353,45 @@ endi
 
 print ========= step8
 system sh/exec.sh -n dnode4 -s start
-sleep 5000
+
+$x = 0
+step8: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step8
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step8
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step8
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step8
+endi
+
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 
 sql insert into d1.t1 values(1589529000016, 6)
 sql insert into d2.t2 values(1589529000026, 6)
 sql insert into d3.t3 values(1589529000036, 6)
 sql insert into d4.t4 values(1589529000046, 6)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 6 then
@@ -269,7 +417,3 @@ system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
 system sh/exec.sh -n dnode4 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/unique/db/replica_add23.sim

@@ -20,6 +20,11 @@ system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
 
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 1
+
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
 sql connect
@@ -29,7 +34,42 @@ sql create dnode $hostname3
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname4
 system sh/exec.sh -n dnode4 -s start
-sleep 3000
+
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+if $data4_4 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
 
 print ======== step1 
 sql create database d1 replica 2
@@ -46,7 +86,6 @@ sql insert into d1.t1 values(1588262400001, 1)
 sql insert into d2.t2 values(1588262400001, 1)
 sql insert into d3.t3 values(1588262400001, 1)
 sql insert into d4.t4 values(1588262400001, 1)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 1 then
@@ -73,17 +112,40 @@ sql alter database d1 replica 3
 sql alter database d2 replica 3
 sql alter database d3 replica 3
 sql alter database d4 replica 3
-sleep 10000
 
-print ======== step3
 $x = 0
-show3: 
+a1: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 40 then
 	  return -1
 	endi
 	
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a1
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a1
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a1
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a1
+endi
+
+print ======== step3
 sql show dnodes
 print dnode1 ==> openVnodes: $data2_1
 print dnode2 ==> openVnodes: $data2_2
@@ -111,7 +173,6 @@ sql insert into d1.t1 values(1588262400002, 2)
 sql insert into d2.t2 values(1588262400002, 2)
 sql insert into d3.t3 values(1588262400002, 2)
 sql insert into d4.t4 values(1588262400002, 2)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 2 then
@@ -134,17 +195,14 @@ if $rows != 2 then
 endi
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 print ========= step5 
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
-
 sql insert into d1.t1 values(1588262400003, 3)
 sql insert into d2.t2 values(1588262400003, 3)
 sql insert into d3.t3 values(1588262400003, 3)
 sql insert into d4.t4 values(1588262400003, 3)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 3 then
@@ -168,15 +226,45 @@ endi
 
 print ========= step6
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
+
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
+	
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step6
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step6
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step6
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step6
+endi
+
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
-sleep 5000
 
 sql insert into d1.t1 values(1588262400004, 4)
 sql insert into d2.t2 values(1588262400004, 4)
 sql insert into d3.t3 values(1588262400004, 4)
 sql insert into d4.t4 values(1588262400004, 4)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 4 then
@@ -200,15 +288,45 @@ endi
 
 print ========= step7
 system sh/exec.sh -n dnode3 -s start
-sleep 5000
+
+$x = 0
+step7: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
+	
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step7
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step7
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step7
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step7
+endi
+
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
-sleep 5000
 
 sql insert into d1.t1 values(1588262400005, 5)
 sql insert into d2.t2 values(1588262400005, 5)
 sql insert into d3.t3 values(1588262400005, 5)
 sql insert into d4.t4 values(1588262400005, 5)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 5 then
@@ -232,15 +350,45 @@ endi
 
 print ========= step8
 system sh/exec.sh -n dnode4 -s start
-sleep 5000
+
+$x = 0
+step8: 
+	$x = $x + 1
+	sleep 2000
+	if $x == 10 then
+	  return -1
+	endi
+	
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step8
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step8
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step8
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step8
+endi
+
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 
 sql insert into d1.t1 values(1588262400006, 6)
 sql insert into d2.t2 values(1588262400006, 6)
 sql insert into d3.t3 values(1588262400006, 6)
 sql insert into d4.t4 values(1588262400006, 6)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 6 then
@@ -266,7 +414,3 @@ system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
 system sh/exec.sh -n dnode4 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/unique/db/replica_part.sim

@@ -14,6 +14,10 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
 
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 1
+
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
 sql connect
@@ -21,7 +25,38 @@ sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname3
 system sh/exec.sh -n dnode3 -s start
-sleep 3000
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
 
 print ======== step1 
 sql create database d1 replica 3
@@ -38,7 +73,6 @@ sql insert into d2.t2 values(now, 1)
 sql insert into d1.t1 values(now, 1)
 sql insert into d3.t3 values(now, 1)
 sql insert into d4.t4 values(now, 1)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 1 then
@@ -61,24 +95,85 @@ if $rows != 1 then
 endi
 
 print ========= step2 alter db
-system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 sql alter database d1 replica 2
 sql alter database d2 replica 2
 sql alter database d3 replica 2
 sql alter database d4 replica 2
-sleep 5000
+
+$x = 0
+a1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
 
 print ========= step3
 system sh/exec.sh -n dnode2 -s start
-sleep 10000
+
+$x = 0
+step3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step3
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step3
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step3
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step3
+endi
 
 print ========= step4
 sql insert into d1.t1 values(now, 2)
 sql insert into d2.t2 values(now, 2)
 sql insert into d3.t3 values(now, 2)
 sql insert into d4.t4 values(now, 2)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 2 then
@@ -102,44 +197,104 @@ endi
 
 print ========= step5
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql insert into d1.t1 values(now, 3) -x s1
 s1:
-sql insert into d2.t2 values(now, 3) -x s2
-s2:
-sql insert into d3.t3 values(now, 3) -x s3
-s3:
-sql insert into d4.t4 values(now, 3) -x s4
-s4:
+#sql insert into d2.t2 values(now, 3) -x s2
+#s2:
+#sql insert into d3.t3 values(now, 3) -x s3
+#s3:
+#sql insert into d4.t4 values(now, 3) -x s4
+#s4:
 
 print ========= step6
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
+
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
+
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
-sleep 5000
 
-sql insert into d1.t1 values(now, 4) -x s5
-s5:
+#sql insert into d1.t1 values(now, 4) -x s5
+#s5:
 sql insert into d2.t2 values(now, 4) -x s6
 s6:
-sql insert into d3.t3 values(now, 4) -x s7
-s7:
-sql insert into d4.t4 values(now, 4) -x s8
-s8:
+#sql insert into d3.t3 values(now, 4) -x s7
+#s7:
+#sql insert into d4.t4 values(now, 4) -x s8
+#s8:
 
 print ========= step7
 system sh/exec.sh -n dnode3 -s start
-sleep 5000
+
+$x = 0
+step7: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step7
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step7
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step7
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step7
+endi
 
 sql insert into d1.t1 values(now, 5)
 sql insert into d2.t2 values(now, 5)
 sql insert into d3.t3 values(now, 5)
 sql insert into d4.t4 values(now, 5)
-sleep 1000
 
 sql select * from d1.t1
 sql select * from d2.t2
@@ -149,8 +304,3 @@ sql select * from d4.t4
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode4 -s stop  -x SIGINT
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/unique/db/replica_reduce21.sim

@@ -14,12 +14,42 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
 
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 1
+
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
 sql connect
 sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
-sleep 3000
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+if $data2_1 != master then
+  goto step1
+endi
 
 print ======== step1 
 sql create database d1 replica 2
@@ -63,12 +93,25 @@ sql create database d5 replica 1
 
 print ========= step3 alter d1
 sql alter database d1 replica 1
-sleep 12000
+
+$x = 0
+a1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 1 then
+	goto a1
+endi
+
 
 print ========= step4 query d1
 sql insert into d1.t1 values(now, 2)
 sql select * from d1.t1
-sleep 1000
 if $rows != 2 then
 	return -1
 endi
@@ -77,22 +120,20 @@ print ========= step5 query d5
 sql create table d5.t5 (ts timestamp, i int)
 sql insert into d5.t5 values(now, 1);
 sql select * from d5.t5
-sleep 1000
 if $rows != 1 then
 	return -1
 endi
 return
 print ========= step7 drop d1
 sql drop database d1
-sleep 5000
-
 sql reset query cache
+sleep 100
+
 print ========= step8
 sql insert into d5.t5 values(now, 2)
 sql insert into d2.t2 values(now, 2)
 sql insert into d3.t3 values(now, 2)
 sql insert into d4.t4 values(now, 2)
-sleep 1000
 
 sql select * from d5.t5
 if $rows != 2 then
@@ -116,13 +157,11 @@ endi
 
 print ======== step9 stop dnode2
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 
 sql insert into d5.t5 values(now, 3)
 sql insert into d2.t2 values(now, 3)
 sql insert into d3.t3 values(now, 3)
 sql insert into d4.t4 values(now, 3)
-sleep 1000
 
 sql select * from d5.t5
 if $rows != 3 then
@@ -146,9 +185,3 @@ endi
 
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode3 -s stop  -x SIGINT
-system sh/exec.sh -n dnode4 -s stop  -x SIGINT
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/unique/db/replica_reduce31.sim

@@ -14,6 +14,11 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
 
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 1
+
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
 sql connect
@@ -21,7 +26,38 @@ sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname3
 system sh/exec.sh -n dnode3 -s start
-sleep 3000
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
 
 print ======== step1 
 sql create database d1 replica 3
@@ -38,7 +74,6 @@ sql insert into d1.t1 values(now, 1)
 sql insert into d2.t2 values(now, 1)
 sql insert into d3.t3 values(now, 1)
 sql insert into d4.t4 values(now, 1)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 1 then
@@ -68,23 +103,70 @@ sql alter database d1 replica 2
 sql alter database d2 replica 2
 sql alter database d3 replica 2
 
-sleep 8000
+$x = 0
+a2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a2
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a2
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a2
+endi
 
 sql alter database d1 replica 1
 sql alter database d2 replica 1
 sql alter database d3 replica 1
 
-sleep 8000
+$x = 0
+a1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 1 then
+	goto a1
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 1 then
+	goto a1
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 1 then
+	goto a1
+endi
 
 print ========= step3
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql insert into d1.t1 values(now, 2)
 sql insert into d2.t2 values(now, 2)
 sql insert into d3.t3 values(now, 2)
 sql insert into d4.t4 values(now, 2)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 2 then
@@ -110,13 +192,37 @@ print ========= step4 alter db
 sql alter database d1 replica 2
 sql alter database d2 replica 2
 sql alter database d3 replica 2
-sleep 8000
+
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step4
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step4
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step4
+endi
 
 sql insert into d1.t1 values(now, 3)
 sql insert into d2.t2 values(now, 3)
 sql insert into d3.t3 values(now, 3)
 sql insert into d4.t4 values(now, 3)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 3 then
@@ -140,10 +246,8 @@ endi
 
 print ========= step4
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
-
 sql reset query cache
-sleep 1000
+sleep 100
 
 sql insert into d1.t1 values(now, 4) -x step1
 step1:
@@ -156,9 +260,7 @@ step4:
 
 print ========= step5
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
-sleep 5000
 
 sql insert into d1.t1 values(now, 5) -x step5
 step5:
@@ -171,13 +273,11 @@ step8:
 
 print ========= step6
 system sh/exec.sh -n dnode3 -s start
-sleep 5000
 
 sql insert into d1.t1 values(now, 6)
 sql insert into d2.t2 values(now, 6)
 sql insert into d3.t3 values(now, 6)
 sql insert into d4.t4 values(now, 6)
-sleep 1000
 
 sql select * from d1.t1
 sql select * from d2.t2
@@ -187,8 +287,3 @@ sql select * from d4.t4
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode4 -s stop  -x SIGINT
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/unique/db/replica_reduce32.sim

@@ -14,6 +14,11 @@ system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
 
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 1
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 1
+
 print ========= start dnodes
 system sh/exec.sh -n dnode1 -s start
 sql connect
@@ -21,7 +26,43 @@ sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname3
 system sh/exec.sh -n dnode3 -s start
-sleep 3000
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
+if $data2_2 != null then
+  goto step1
+endi
+if $data2_3 != null then
+  goto step1
+endi
 
 print ======== step1 
 sql create database d1 replica 3
@@ -38,7 +79,6 @@ sql insert into d2.t2 values(now, 1)
 sql insert into d1.t1 values(now, 1)
 sql insert into d3.t3 values(now, 1)
 sql insert into d4.t4 values(now, 1)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 1 then
@@ -65,14 +105,44 @@ sql alter database d1 replica 2
 sql alter database d2 replica 2
 sql alter database d3 replica 2
 sql alter database d4 replica 2
-sleep 12000
+
+$x = 0
+a1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
 
 print ========= step3
 sql insert into d1.t1 values(now, 2)
 sql insert into d2.t2 values(now, 2)
 sql insert into d3.t3 values(now, 2)
 sql insert into d4.t4 values(now, 2)
-sleep 1000
 
 sql select * from d1.t1
 if $rows != 2 then
@@ -95,40 +165,111 @@ if $rows != 2 then
 endi
 
 sql reset query cache
-sleep 1000
+sleep 200
 
 print ========= step4
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep 5000
 
-sql_error insert into d1.t1 values(now, 3)
-sql_error insert into d2.t2 values(now, 3)
-sql_error insert into d3.t3 values(now, 3)
-sql_error insert into d4.t4 values(now, 3)
+sql insert into d1.t1 values(now, 3) -x s41
+s41:
+sql insert into d2.t2 values(now, 3) -x s42
+s42:
+sql insert into d3.t3 values(now, 3) -x s43
+s43:
+sql insert into d4.t4 values(now, 3) -x s44
+s44:
 
-sql_error select * from d1.t1
-sql_error select * from d2.t2
-sql_error select * from d3.t3
-sql_error select * from d4.t4
+sql select * from d1.t1 -x s45
+s45:
+sql select * from d2.t2 -x s46
+s46:
+sql select * from d3.t3 -x s47
+s47:
+sql select * from d4.t4 -x s48
+s48:
 
 print ========= step5
 system sh/exec.sh -n dnode2 -s start
 
-sleep 5000
-system sh/exec.sh -n dnode3 -s stop -x SIGINT
-sleep 5000
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step5
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step5
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step5
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step5
+endi
 
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
 sql reset query cache
-sleep 1000
+sleep 100
 
-sql_error insert into d1.t1 values(now, 3)
-sql_error insert into d2.t2 values(now, 3)
-sql_error insert into d3.t3 values(now, 3)
-sql_error insert into d4.t4 values(now, 3)
+sql_error insert into d1.t1 values(now, 3) -x s51
+s51:
+sql_error insert into d2.t2 values(now, 3) -x s52
+s52:
+sql_error insert into d3.t3 values(now, 3) -x s53
+s53:
+sql_error insert into d4.t4 values(now, 3) -x s54
+s54:
 
 print ========= step6
 system sh/exec.sh -n dnode3 -s start
-sleep 5000
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show d1.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
+
+sql show d2.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
+
+sql show d3.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
+
+sql show d4.vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto step6
+endi
 
 sql insert into d1.t1 values(now, 5)
 sql insert into d2.t2 values(now, 5)
@@ -151,8 +292,3 @@ print d4.t4 $rows
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode4 -s stop  -x SIGINT
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/unique/dnode/balance1.sim

@@ -33,7 +33,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4
 print ========== step1
 system sh/exec.sh -n dnode1 -s start
 sql connect
-sleep 3000
 
 sql create database d1 
 sql create table d1.t1 (t timestamp, i int) 
@@ -50,15 +49,14 @@ if $data2_1 != 1 then
 endi
 
 print ========== step2
-sleep 2000
 sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
 
 $x = 0
 show2: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 		return -1
 	endi
 	
@@ -124,8 +122,8 @@ system sh/exec.sh -n dnode3 -s start
 $x = 0
 show5: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 20 then 
+	sleep 1000
+	if $x == 30 then 
 		return -1
 	endi
 	
@@ -174,8 +172,8 @@ system sh/exec.sh -n dnode4 -s start
 $x = 0
 show7: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 	  return -1
 	endi
 	
@@ -258,7 +256,7 @@ endi
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
 
 sql reset query cache
-sleep 1000
+sleep 100
 
 print ========== step10
 sql select * from d1.t1 order by t desc

tests/script/unique/dnode/balance2.sim

@@ -32,7 +32,29 @@ sql create dnode $hostname2
 sql create dnode $hostname3
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
-sleep 3000
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
 
 sql create database d1 replica 2  
 sql create table d1.t1 (t timestamp, i int)
@@ -70,8 +92,8 @@ sql drop dnode $hostname2
 $x = 0
 show2: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 		return -1
 	endi
 	
@@ -90,7 +112,6 @@ if $data2_3 != 2 then
 endi
 
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 
 print ========== step3
 sql create dnode $hostname4
@@ -156,8 +177,8 @@ system sh/exec.sh -n dnode5 -s start
 $x = 0
 show5: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 		return -1
 	endi
 	
@@ -189,8 +210,8 @@ sql drop dnode $hostname3
 $x = 0
 show6: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 	  return -1
 	endi
 	
@@ -217,10 +238,8 @@ if $data2_5 != 3 then
 endi
 
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
-
 sql reset query cache
-sleep 1000
+sleep 100
 
 print ========== step7
 sql select * from d1.t1 order by t desc

tests/script/unique/dnode/balance3.sim

@@ -38,7 +38,32 @@ sql create dnode $hostname4
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
 system sh/exec.sh -n dnode4 -s start
-sleep 3000
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+if $data4_4 != ready then
+  goto step1
+endi
 
 sql create database d1 replica 3  
 sql create table d1.t1 (t timestamp, i int) 
@@ -81,7 +106,7 @@ sql drop dnode $hostname2
 $x = 0
 show2: 
 	$x = $x + 1
-	sleep 2000
+	sleep 1000
 	if $x == 20 then 
 	  return -1
 	endi
@@ -106,7 +131,6 @@ if $data2_4 != 2 then
 endi
 
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 print ========== step
 sql create dnode $hostname5
 system sh/exec.sh -n dnode5 -s start
@@ -114,8 +138,8 @@ system sh/exec.sh -n dnode5 -s start
 $x = 0
 show3: 
 	$x = $x + 1
-	sleep 4000
-	if $x == 15 then 
+	sleep 1000
+	if $x == 60 then 
 	  return -1
 	endi
 	
@@ -154,8 +178,8 @@ sql insert into d3.t3 values(now+5s, 31)
 $x = 0
 show4: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then 
+	sleep 1000
+	if $x == 30 then 
 	  return -1
 	endi
 
@@ -189,8 +213,8 @@ system sh/exec.sh -n dnode6 -s start
 $x = 0
 show5: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 20 then 
 	  return -1
 	endi
 
@@ -216,8 +240,8 @@ sql drop dnode $hostname3
 $x = 0
 show6: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 20 then 
+	sleep 1000
+	if $x == 30 then 
 	  return -1
 	endi
 
@@ -245,10 +269,8 @@ if $data2_5 != 3 then
 endi
 
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
-sleep 5000
-
 sql reset query cache
-sleep 1000
+sleep 100
 
 print ========== step7
 sql select * from d1.t1 order by t desc

tests/script/unique/dnode/offline3.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c offlineThreshold -v 3
+system sh/cfg.sh -n dnode2 -c offlineThreshold -v 3
+system sh/cfg.sh -n dnode3 -c offlineThreshold -v 3
+system sh/cfg.sh -n dnode4 -c offlineThreshold -v 3
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 300
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 300
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 300
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 300
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+
+print ========== step1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+sql create dnode $hostname2
+system sh/exec.sh -n dnode2 -s start
+sql create dnode $hostname3
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname4
+system sh/exec.sh -n dnode4 -s start
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+if $data4_4 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+if $data2_1 != master then
+  goto step1
+endi
+
+print ========== step2
+sql create database db replica 3
+sql use db
+sql create table tb (ts timestamp, value int)
+sql insert into tb values (now, 1)
+sql insert into tb values (now, 2)
+
+sql show vgroups;
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+print ========== step2
+system sh/exec.sh -n dnode4 -s stop -x SIGINT
+
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+		return -1
+	endi
+
+sql show vgroups;
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07 $data08 $data09
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step2
+endi
+if $data4_2 != ready then
+  goto step2
+endi
+if $data4_3 != ready then
+  goto step2
+endi
+if $data4_4 != null then
+  goto step2
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
\ No newline at end of file

tests/script/unique/dnode/reason.sim

@@ -5,7 +5,6 @@ system sh/deploy.sh -n dnode2 -i 2
 
 print ========== step1
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 sql create dnode $hostname2
 
@@ -18,38 +17,75 @@ endi
 
 print ========== step2
 system sh/exec.sh -n dnode2 -s start
-sleep 3000
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
 sql show dnodes
-print dnode1 off: $data7_1
-print dnode2 off: $data7_2
+print dnode1 $data4_1
+print dnode2 $data4_2
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
 
 print ========== step3
 system sh/exec.sh -n dnode2 -s stop
-sleep 3000
+
+$x = 0
+step3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 sql show dnodes
 print dnode1 off: $data7_1
 print dnode2 off: $data7_2
 if $data7_2 != @status msg timeout@ then
-	return -1
+	goto step3
 endi
 
 print ========== step4
 sql drop dnode $hostname2
-sleep 5000
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
 sql show dnodes
 if $rows != 1 then
-	return -1
+	goto step4
 endi
 
 print ========== step5
 system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname2
-sleep 3000
+
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
 sql show dnodes
 print dnode1 off: $data7_1
 print dnode2 off: $data7_3
 if $data7_3 != @dnodeId not match@ then
-	return -1
+	goto step5
 endi
 
 print ========== step6
@@ -58,12 +94,19 @@ system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 3
 system sh/exec.sh -n dnode4 -s start
 sql create dnode $hostname4
 
-sleep 3000
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
 sql show dnodes
 print dnode1 off: $data7_1
 print dnode4 off: $data7_4
 if $data7_4 != @mnEqualVn not match@ then
-	return -1
+	goto step6
 endi
 
 print ========== step7
@@ -72,12 +115,19 @@ system sh/cfg.sh -n dnode5 -c statusInterval -v 3
 system sh/exec.sh -n dnode5 -s start
 sql create dnode $hostname5
 
-sleep 3000
+$x = 0
+step7: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
 sql show dnodes
 print dnode1 off: $data7_1
 print dnode5 off: $data7_5
 if $data7_5 != @interval not match@ then
-	return -1
+	goto step7
 endi
 
 print ========== step8
@@ -86,12 +136,19 @@ system sh/cfg.sh -n dnode6 -c balance -v 0
 system sh/exec.sh -n dnode6 -s start
 sql create dnode $hostname6
 
-sleep 3000
+$x = 0
+step8: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
 sql show dnodes
 print dnode1 off: $data7_1
 print dnode6 off: $data7_6
 if $data7_6 != @balance not match@ then
-	return -1
+	goto step8
 endi
 
 print ========== step9
@@ -100,12 +157,19 @@ system sh/cfg.sh -n dnode7 -c maxTablesPerVnode -v 3000
 system sh/exec.sh -n dnode7 -s start
 sql create dnode $hostname7
 
-sleep 3000
+$x = 0
+step9: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
 sql show dnodes
 print dnode1 off: $data7_1
 print dnode7 off: $data7_7
 if $data7_7 != @maxTabPerVn not match@ then
-	return -1
+	goto step9
 endi
 
 print ========== step10
@@ -114,12 +178,19 @@ system sh/cfg.sh -n dnode8 -c maxVgroupsPerDb -v 3
 system sh/exec.sh -n dnode8 -s start
 sql create dnode $hostname8
 
-sleep 3000
+$x = 0
+step10: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
 sql show dnodes
 print dnode1 off: $data7_1
 print dnode8 off: $data7_8
 if $data7_8 != @maxVgPerDb not match@ then
-	return -1
+	goto step10
 endi
 
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT

tests/script/unique/dnode/remove1.sim

@@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4
 
 print ========== step1
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 
 sql create database d1 
@@ -50,7 +49,26 @@ endi
 print ========== step2
 sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
-sleep 9000
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step2
+endi
+if $data4_2 != ready then
+  goto step2
+endi
 
 sql create database d3 replica 2 
 sql create table d3.t3 (t timestamp, i int)
@@ -81,12 +99,11 @@ endi
 print ========== step3
 sql drop dnode $hostname2
 
-sleep 7001
 $x = 0
 show3: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 	  return -1
 	endi
 	
@@ -101,8 +118,8 @@ system sh/exec.sh -n dnode3 -s start
 $x = 0
 show4: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 	  return -1
 	endi
 	
@@ -115,7 +132,6 @@ if $data2_2 != null then
 endi
 
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 
 print ========== step5
 sql create dnode $hostname4
@@ -124,8 +140,8 @@ system sh/exec.sh -n dnode4 -s start
 $x = 0
 show5: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 20 then 
 	  return -1
 	endi
 sql show dnodes

tests/script/unique/dnode/remove2.sim

@@ -22,7 +22,6 @@ system sh/cfg.sh -n dnode4 -c maxTablesPerVnode -v 4
 
 print ========== step1
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 
 sql create database d1 
@@ -50,7 +49,26 @@ endi
 print ========== step2
 sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
-sleep 9000
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step2
+endi
+if $data4_2 != ready then
+  goto step2
+endi
 
 sql create database d3 replica 2 
 sql create table d3.t3 (t timestamp, i int)
@@ -81,7 +99,6 @@ endi
 print ========== step3
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sql drop dnode $hostname2
-sleep 5000
 
 sql show dnodes
 print dnode1 openVnodes $data2_1
@@ -91,14 +108,20 @@ print ========== step4
 sql create dnode $hostname3
 system sh/exec.sh -n dnode3 -s start
 
-sleep 5000
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+	  return -1
+	endi
 
 sql show dnodes
 print dnode1 openVnodes $data2_1
 print dnode2 openVnodes $data2_2
 print dnode3 openVnodes $data2_3
 if $data2_3 != 0 then
-	return -1
+	goto step4
 endi
 
 print ============ step 4.1
@@ -107,8 +130,8 @@ system sh/exec.sh -n dnode2 -s start
 $x = 0
 show4: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 	  return -1
 	endi
 

tests/script/unique/dnode/vnode_clean.sim

@@ -45,8 +45,8 @@ system sh/exec.sh -n dnode2 -s start
 $x = 0
 show2: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 	  return -1
 	endi
 sql show dnodes
@@ -86,8 +86,8 @@ sql drop dnode $hostname2
 $x = 0
 show4: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 	  return -1
 	endi
 sql show dnodes
@@ -106,15 +106,14 @@ endi
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 
 print ========== step5
-sleep 5000
 sql create dnode $hostname3
 system sh/exec.sh -n dnode3 -s start
 
 $x = 0
 show5: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 20 then 
+	sleep 1000
+	if $x == 40 then 
 	  return -1
 	endi
 sql show dnodes
@@ -153,8 +152,8 @@ system sh/exec.sh -n dnode4 -s start
 $x = 0
 show7: 
 	$x = $x + 1
-	sleep 3000
-	if $x == 20 then 
+	sleep 1000
+	if $x == 40 then 
 		return -1
 	endi
 	
@@ -184,8 +183,8 @@ sql insert into d4.t4 values(now+5s, 41)
 $x = 0
 show8: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 	  return -1
 	endi
 sql show dnodes
@@ -208,8 +207,8 @@ sql drop dnode $hostname3
 $x = 0
 show9: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 	  return -1
 	endi
 
@@ -228,7 +227,6 @@ if $data2_4 != 4 then
 endi
 
 system sh/exec.sh -n dnode3 -s stop 
-sleep 5000
 
 print ========== step10
 sql select * from d1.t1  order by t desc

tests/script/unique/import/replica2.sim

@@ -27,11 +27,30 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2
 
 print ========= start dnode1
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 
 sql create dnode $hostname2
 system sh/exec.sh -n dnode2 -s start
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
 
 sql create database ir2db replica 2 days 7
 sql use ir2db
@@ -96,9 +115,22 @@ endi
 
 print ================== dnode restart
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+
+$x = 0
+a1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a1
+endi
+
 sql select * from tb;
 if $rows != 14 then 
   return -1
@@ -163,9 +195,22 @@ endi
 
 print ================= step10
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+
+$x = 0
+a2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a2
+endi
+
 sql select * from tb;
 print $rows
 if $rows != 35 then 
@@ -193,9 +238,20 @@ endi
 
 print ================= step13
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode2 -s start
-sleep 5000
+$x = 0
+a3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a3
+endi
 
 print ================= step14
 #1520000000000
@@ -214,12 +270,24 @@ endi
 
 print ================= step15
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+a4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 2 then
+	goto a4
+endi
 
+sql select * from tb;
 if $rows != 52 then 
-  return -1
+  goto a4
 endi
 
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT

tests/script/unique/import/replica3.sim

@@ -27,7 +27,6 @@ system sh/cfg.sh -n dnode4 -c walLevel -v 2
 
 print ========= start dnode1
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 
 sql create dnode $hostname2
@@ -35,6 +34,29 @@ sql create dnode $hostname3
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
 
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
 
 sql create database ir3db replica 3 days 7
 sql use ir3db
@@ -99,9 +121,20 @@ endi
 
 print ================== dnode restart
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+$x = 0
+a4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto a4
+endi
+
 sql select * from tb;
 if $rows != 14 then 
   return -1
@@ -166,9 +199,21 @@ endi
 
 print ================= step10
 system sh/exec.sh -n dnode1 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode1 -s start
-sleep 5000
+
+$x = 0
+step10: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step10
+endi
+
 sql select * from tb;
 print $rows
 if $rows != 35 then 
@@ -217,10 +262,21 @@ endi
 
 print ================= step15
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
-sleep 5000
 system sh/exec.sh -n dnode3 -s start
-sleep 5000
+$x = 0
+step15: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+sql show vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step15
+endi
 
+sql select * from tb;
 if $rows != 52 then 
   return -1
 endi

tests/script/unique/mnode/mgmt20.sim

@@ -11,7 +11,6 @@ system sh/cfg.sh -n dnode2 -c monitor -v 1
 print ============== step1
 system sh/exec.sh -n dnode1 -s start
 system sh/exec.sh -n dnode2 -s start
-sleep 3000
 sql connect
 
 print ============== step2
@@ -20,8 +19,8 @@ sql create dnode $hostname2
 $x = 0
 show2: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 20 then 
 		return -1
 	endi
 	

tests/script/unique/mnode/mgmt22.sim

@@ -9,7 +9,6 @@ system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2
 
 print ============== step1
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 
 sql show mnodes
@@ -26,8 +25,8 @@ sql create dnode $hostname2
 $x = 0
 show2: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 20 then 
 		return -1
 	endi
 	
@@ -64,8 +63,8 @@ sql connect
 $x = 0
 show6: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 20 then 
 		return -1
 	endi
 	
@@ -82,13 +81,12 @@ endi
 print ============== step7
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname3
-sleep 5000
 
 $x = 0
 show7: 
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then 
+	sleep 1000
+	if $x == 20 then 
 		return -1
 	endi
 	

tests/script/unique/mnode/mgmt23.sim

@@ -25,8 +25,8 @@ sql create dnode $hostname2
 $x = 0
 show2:
 	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
+	sleep 1000
+	if $x == 20 then
 		return -1
 	endi
 
@@ -65,7 +65,14 @@ endi
 
 print ============== step4
 sql drop dnode $hostname2
-sleep 10000
+
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -76,13 +83,13 @@ print dnode2 ==> $dnode2Role
 print dnode3 ==> $dnode3Role
 
 if $dnode1Role != master then
-  return -1
+  goto step4
 endi
 if $dnode2Role != null then
-  return -1
+  goto step4
 endi
 if $dnode3Role != slave then
-  return -1
+  goto step4
 endi
 
 system sh/exec.sh -n dnode2 -s stop
@@ -93,7 +100,14 @@ sql create dnode $hostname2
 system sh/deploy.sh -n dnode2 -i 2
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2
 system sh/exec.sh -n dnode2 -s start
-sleep 8000
+
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -104,19 +118,17 @@ print dnode2 ==> $dnode2Role
 print dnode3 ==> $dnode3Role
 
 if $dnode1Role != master then
-  return -1
+  goto step5
 endi
 if $dnode2Role != null then
-  return -1
+  goto step5
 endi
 if $dnode3Role != slave then
-  return -1
+  goto step5
 endi
 
 print ============== step6
 system sh/exec.sh -n dnode1 -s stop
-sleep 10000
-
 sql_error show mnodes
 
 print ============== step7
@@ -127,6 +139,3 @@ system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
 system sh/exec.sh -n dnode4 -s stop  -x SIGINT
 system sh/exec.sh -n dnode5 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/unique/mnode/mgmt33.sim

@@ -28,7 +28,14 @@ endi
 print ============== step2
 system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname2
-sleep 8000
+
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -39,19 +46,26 @@ print dnode2 ==> $dnode2Role
 print dnode3 ==> $dnode3Role
 
 if $dnode1Role != master then
-  return -1
+  goto step2
 endi
 if $dnode2Role != slave then
-  return -1
+  goto step2
 endi
 if $dnode3Role != null then
-  return -1
+  goto step2
 endi
 
 print ============== step3
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname3
-sleep 8000
+
+$x = 0
+step3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -62,18 +76,25 @@ print dnode2 ==> $dnode2Role
 print dnode3 ==> $dnode3Role
 
 if $dnode1Role != master then
-  return -1
+  goto step3
 endi
 if $dnode2Role != slave then
-  return -1
+  goto step3
 endi
 if $dnode3Role != slave then
-  return -1
+  goto step3
 endi
 
 print ============== step4
 sql drop dnode $hostname2
-sleep 8000
+
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -84,25 +105,30 @@ print dnode2 ==> $dnode2Role
 print dnode3 ==> $dnode3Role
 
 if $dnode1Role != master then
-  return -1
+  goto step4
 endi
 if $dnode2Role != null then
-  return -1
+  goto step4
 endi
 if $dnode3Role != slave then
-  return -1
+  goto step4
 endi
 
 system sh/exec.sh -n dnode2 -s stop
-sleep 3000
-
 system sh/deploy.sh -n dnode2 -i 2
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
 system sh/exec.sh -n dnode2 -s start
 
 print ============== step5
 sql create dnode $hostname2
-sleep 8000
+
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -113,20 +139,26 @@ print dnode2 ==> $dnode2Role
 print dnode3 ==> $dnode3Role
 
 if $dnode1Role != master then
-  return -1
+  goto step5
 endi
 if $dnode2Role != slave then
-  return -1
+  goto step5
 endi
 if $dnode3Role != slave then
-  return -1
+  goto step5
 endi
 
 print ============== step6
 system sh/exec.sh -n dnode1 -s stop
-sleep 10000
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
-sql show mnodes
+sql show mnodes -x step6
 $dnode1Role = $data2_1
 $dnode2Role = $data2_4
 $dnode3Role = $data2_3
@@ -135,7 +167,7 @@ print dnode2 ==> $dnode2Role
 print dnode3 ==> $dnode3Role
 
 if $dnode1Role != offline then
-  return -1
+  goto step6
 endi
 #if $dnode2Role != master then
 #  return -1
@@ -146,9 +178,15 @@ endi
 
 print ============== step7
 sql drop dnode $hostname1
-sleep 8000
+$x = 0
+step7: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
-sql show mnodes
+sql show mnodes -x step7
 $dnode1Role = $data2_1
 $dnode2Role = $data2_2
 $dnode3Role = $data2_3
@@ -157,7 +195,7 @@ print dnode2 ==> $dnode2Role
 print dnode3 ==> $dnode3Role
 
 if $dnode1Role != null then
-  return -1
+  goto step7
 endi
 #if $dnode2Role != master then
 #  return -1

tests/script/unique/mnode/mgmt34.sim

@@ -12,7 +12,6 @@ system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3
 
 print ============== step1
 system sh/exec.sh -n dnode1 -s start
-sleep 3000
 sql connect
 
 sql show mnodes
@@ -32,7 +31,13 @@ endi
 print ============== step2
 system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname2
-sleep 8000
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -45,22 +50,29 @@ print dnode3 ==> $dnode3Role
 print dnode4 ==> $dnode4Role
 
 if $dnode1Role != master then
-  return -1
+  goto step2
 endi
 if $dnode2Role != slave then
-  return -1
+  goto step2
 endi
 if $dnode3Role != null then
-  return -1
+  goto step2
 endi
 if $dnode4Role != null then
-  return -1
+  goto step2
 endi
 
 print ============== step3
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname3
-sleep 8000
+
+$x = 0
+step3: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -73,23 +85,29 @@ print dnode3 ==> $dnode3Role
 print dnode4 ==> $dnode4Role
 
 if $dnode1Role != master then
-  return -1
+  goto step3
 endi
 if $dnode2Role != slave then
-  return -1
+  goto step3
 endi
 if $dnode3Role != slave then
-  return -1
+  goto step3
 endi
 if $dnode4Role != null then
-  return -1
+  goto step3
 endi
 
 
 print ============== step4
 system sh/exec.sh -n dnode4 -s start
 sql create dnode $hostname4
-sleep 8000
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -102,21 +120,27 @@ print dnode3 ==> $dnode3Role
 print dnode4 ==> $dnode4Role
 
 if $dnode1Role != master then
-  return -1
+  goto step4
 endi
 if $dnode2Role != slave then
-  return -1
+  goto step4
 endi
 if $dnode3Role != slave then
-  return -1
+  goto step4
 endi
 if $dnode4Role != null then
-  return -1
+  goto step4
 endi
 
 print ============== step5
 sql drop dnode $hostname2
-sleep 8000
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -129,28 +153,32 @@ print dnode3 ==> $dnode3Role
 print dnode4 ==> $dnode4Role
 
 if $dnode1Role != master then
-  return -1
+  goto step5
 endi
 if $dnode2Role != null then
-  return -1
+  goto step5
 endi
 if $dnode3Role != slave then
-  return -1
+  goto step5
 endi
 if $dnode4Role != slave then
-  return -1
+  goto step5
 endi
 
 system sh/exec.sh -n dnode2 -s stop
-sleep 3000
-
 system sh/deploy.sh -n dnode2 -i 2
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
 system sh/exec.sh -n dnode2 -s start
 
 print ============== step6
 sql create dnode $hostname2
-sleep 8000
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
 sql show mnodes
 $dnode1Role = $data2_1
@@ -163,23 +191,29 @@ print dnode3 ==> $dnode3Role
 print dnode4 ==> $dnode4Role
 
 if $dnode1Role != master then
-  return -1
+  goto step6
 endi
 if $dnode2Role != null then
-  return -1
+  goto step6
 endi
 if $dnode3Role != slave then
-  return -1
+  goto step6
 endi
 if $dnode4Role != slave then
-  return -1
+  goto step6
 endi
 
 print ============== step7
 system sh/exec.sh -n dnode1 -s stop
-sleep 4000
+$x = 0
+step7: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
-sql show mnodes
+sql show mnodes -x step7
 $dnode1Role = $data2_1
 $dnode2Role = $data2_2
 $dnode3Role = $data2_3
@@ -190,14 +224,19 @@ print dnode3 ==> $dnode3Role
 print dnode4 ==> $dnode4Role
 
 if $dnode1Role != offline then
-  return -1
+  goto step7
 endi
 
 print ============== step8
 sql drop dnode $hostname1
-sleep 8000
+step8: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
 
-sql show mnodes
+sql show mnodes -x step8
 $dnode1Role = $data2_1
 $dnode2Role = $data2_5
 $dnode3Role = $data2_3
@@ -208,10 +247,10 @@ print dnode3 ==> $dnode3Role
 print dnode4 ==> $dnode4Role
 
 if $dnode1Role != null then
-  return -1
+  goto step8
 endi
 if $dnode2Role != slave then
-  return -1
+  goto step8
 endi
 #if $dnode3Role != master then
 #  return -1

tests/script/unique/mnode/mgmtr2.sim

@@ -55,6 +55,14 @@ system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
 
 print ============== step4
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+		return -1
+	endi
+
 sql show mnodes
 
 $dnode1Role = $data2_1
@@ -65,20 +73,15 @@ print dnode2 ==> $dnode2Role
 print dnode3 ==> $dnode3Role
 
 if $dnode1Role != master then
-  return -1
+  goto step4
 endi
 if $dnode2Role != slave then
-  return -1
+  goto step4
 endi
 if $dnode3Role != null then
-  return -1
+  goto step4
 endi
 
 system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
\ No newline at end of file
-system sh/exec.sh -n dnode4 -s stop  -x SIGINT
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/unique/vnode/many.sim

@@ -23,7 +23,37 @@ sql create dnode $hostname2
 sql create dnode $hostname3
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
-sleep 3000
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
 
 print ========= step1
 sql create database db1 replica 2 

tests/script/unique/vnode/replica2_repeat.sim

@@ -23,7 +23,38 @@ sql create dnode $hostname2
 sql create dnode $hostname3
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
-sleep 3000
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
 
 print ========= step1
 sql create database db replica 2 
@@ -64,7 +95,7 @@ print ======== step7
 $lastRows = $data00
 print ======== loop Times $x
 
-if $x < 5 then
+if $x < 2 then
   $x = $x + 1
   goto loop
 endi

tests/script/unique/vnode/replica3_basic.sim

@@ -16,14 +16,48 @@ sql create dnode $hostname2
 sql create dnode $hostname3
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
-sleep 3000
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
+if $data2_2 != slave then
+  goto step1
+endi
+if $data2_3 != slave then
+  goto step1
+endi
 
 $N = 10
 $table = table_r3
 $db = db1
 
-sleep 3000
-
 print =================== step 1 
 
 sql create database $db replica 3 
@@ -66,7 +100,21 @@ endi
 
 print =================== step 4
 system sh/exec.sh -n dnode2 -s start
-sleep 2000
+
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show db1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step4
+endi
+
 $y = $x + $N
 $expect = $N * 3
 while $x < $y
@@ -83,7 +131,6 @@ endi
 
 print =================== step 5
 system sh/exec.sh -n dnode3 -s stop
-sleep 2000
 $y = $x + $N
 $expect = $N * 4
 while $x < $y
@@ -100,7 +147,21 @@ endi
 
 print =================== step 6
 system sh/exec.sh -n dnode3 -s start
-sleep 2000
+
+$x = 0
+step6: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show db1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step6
+endi
+
 $y = $x + $N
 $expect = $N * 5
 while $x < $y
@@ -117,7 +178,6 @@ endi
 
 print =================== step 7
 system sh/exec.sh -n dnode1 -s stop
-sleep 2000
 $y = $x + $N
 $expect = $N * 6
 while $x < $y
@@ -134,7 +194,21 @@ endi
 
 print =================== step 8
 system sh/exec.sh -n dnode1 -s start
-sleep 2000
+
+$x = 0
+step8: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 40 then
+	  return -1
+	endi
+
+sql show db1.vgroups 
+print online vnodes $data03
+if $data03 != 3 then
+	goto step8
+endi
+
 $y = $x + $N
 $expect = $N * 7
 while $x < $y

tests/script/unique/vnode/replica3_repeat.sim

@@ -26,7 +26,42 @@ sql create dnode $hostname4
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
 system sh/exec.sh -n dnode4 -s start
-sleep 3000
+
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+print dnode4 $data4_4
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+if $data4_4 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
 
 print ========= step1
 sql create database db replica 3
@@ -75,7 +110,7 @@ print ======== step8
 $lastRows = $data00
 print ======== loop Times $x
 
-if $x < 5 then
+if $x < 2 then
   $x = $x + 1
   goto loop
 endi

tests/test-all.sh

@@ -31,10 +31,16 @@ function runSimCaseOneByOnefq {
 			case=`echo $line | grep sim$ |awk '{print $NF}'`
 
       start_time=`date +%s`
+      IN_TDINTERNAL="community"
+      if [[ "$tests_dir" == *"$IN_TDINTERNAL"* ]]; then
         ./test.sh -f $case > /dev/null 2>&1 && \
         echo -e "${GREEN}$case success${NC}" | tee -a out.log || \
         ( grep 'script.*success.*m$' ../../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log
-        
+      else
+        ./test.sh -f $case > /dev/null 2>&1 && \
+        echo -e "${GREEN}$case success${NC}" | tee -a out.log || \
+        ( grep 'script.*success.*m$' ../../sim/tsim/log/taoslog0.0 && echo -e "${GREEN}$case success${NC}" | tee -a out.log ) || echo -e "${RED}$case failed${NC}" | tee -a out.log
+      fi
       
       out_log=`tail -1 out.log  `
       if [[ $out_log =~ 'failed' ]];then