Merge pull request #4879 from taosdata/feature/sync

[TD-2680]<feature>: display the arbitrator as a special dnode in the show dnode instruction

src/common/inc/tglobal.h

@@ -36,6 +36,7 @@ extern int8_t   tsEnableVnodeBak;
 extern int8_t   tsEnableTelemetryReporting;
 extern char     tsEmail[];
 extern char     tsArbitrator[];
+extern int8_t   tsArbOnline;
 
 // common
 extern int      tsRpcTimer;

src/common/src/tglobal.c

@@ -41,6 +41,7 @@ int32_t  tsStatusInterval = 1;  // second
 int32_t  tsNumOfMnodes = 3;
 int8_t   tsEnableVnodeBak = 1;
 int8_t   tsEnableTelemetryReporting = 1;
+int8_t   tsArbOnline = 0;
 char     tsEmail[TSDB_FQDN_LEN] = {0};
 
 // common

src/mnode/src/mnodeDnode.c

@@ -810,6 +810,10 @@ static int32_t mnodeGetDnodeMeta(STableMetaMsg *pMeta, SShowObj *pShow, void *pC
   }
 
   pShow->numOfRows = mnodeGetDnodesNum();
+  if (tsArbitrator[0] != 0) {
+    pShow->numOfRows++;
+  }
+
   pShow->rowSize = pShow->offset[cols - 1] + pShow->bytes[cols - 1];
   pShow->pIter = NULL;
 
@@ -868,6 +872,45 @@ static int32_t mnodeRetrieveDnodes(SShowObj *pShow, char *data, int32_t rows, vo
     mnodeDecDnodeRef(pDnode);
   }
 
+  if (tsArbitrator[0] != 0) {
+    cols = 0;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = 0;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_WITH_MAXSIZE_TO_VARSTR(pWrite, tsArbitrator, pShow->bytes[cols]);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = 0;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int16_t *)pWrite = 0;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    char *status = dnodeStatus[tsArbOnline > 0 ? TAOS_DN_STATUS_READY : TAOS_DN_STATUS_OFFLINE];
+    STR_TO_VARSTR(pWrite, status);
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, "arb");
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    *(int64_t *)pWrite = 0;
+    cols++;
+
+    pWrite = data + pShow->offset[cols] * rows + pShow->bytes[cols] * numOfRows;
+    STR_TO_VARSTR(pWrite, "-");
+    cols++;
+
+    numOfRows++;
+  }
+
   mnodeVacuumResult(data, pShow->numOfColumns, numOfRows, rows, pShow);
   pShow->numOfReads += numOfRows;
   return numOfRows;

src/sync/inc/syncInt.h

@@ -87,6 +87,7 @@ typedef struct SsyncPeer {
   int32_t  numOfRetrieves;  // number of retrieves tried
   int32_t  fileChanged;     // a flag to indicate file is changed during retrieving process
   int32_t  refCount;
+  int8_t   isArb;
   int64_t  rid;
   void *   timer;
   void *   pConn;

src/sync/src/syncMain.c

@@ -475,7 +475,13 @@ static void syncAddArbitrator(SSyncNode *pNode) {
     }
   }
 
-  pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = syncAddPeer(pNode, &nodeInfo);
+  pPeer = syncAddPeer(pNode, &nodeInfo);
+  if (pPeer != NULL) {
+    pPeer->isArb = 1;
+    sInfo("%s, is added as arbitrator", pPeer->id);
+  }
+
+  pNode->peerInfo[TAOS_SYNC_MAX_REPLICA] = pPeer;
 }
 
 static void syncFreeNode(void *param) {
@@ -651,9 +657,14 @@ static void syncChooseMaster(SSyncNode *pNode) {
 
   // add arbitrator connection
   SSyncPeer *pArb = pNode->peerInfo[TAOS_SYNC_MAX_REPLICA];
-  if (pArb && pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
+  if (pArb) {
+    if (pArb->role != TAOS_SYNC_ROLE_OFFLINE) {
       onlineNum++;
       replica = pNode->replica + 1;
+      sDebug("vgId:%d, arb:%s is used while choose master", pNode->vgId, pArb->id);
+    } else {
+      sError("vgId:%d, arb:%s is not used while choose master for its offline", pNode->vgId, pArb->id);
+    }
   }
 
   if (index < 0 && onlineNum > replica / 2.0) {
@@ -1118,6 +1129,7 @@ static void syncSetupPeerConnection(SSyncPeer *pPeer) {
     pPeer->peerFd = connFd;
     pPeer->role = TAOS_SYNC_ROLE_UNSYNCED;
     pPeer->pConn = syncAllocateTcpConn(tsTcpPool, pPeer->rid, connFd);
+    if (pPeer->isArb) tsArbOnline = 1;
   } else {
     sDebug("%s, failed to setup peer connection to server since %s, try later", pPeer->id, strerror(errno));
     taosClose(connFd);
@@ -1196,7 +1208,7 @@ static void syncProcessIncommingConnection(int32_t connFd, uint32_t sourceIp) {
     return;
   }
 
-  sDebug("vgId:%d, sync connection is incomming, tranId:%u", vgId, msg.tranId);
+  sDebug("vgId:%d, sync connection is incoming, tranId:%u", vgId, msg.tranId);
 
   SSyncNode *pNode = *ppNode;
   pthread_mutex_lock(&pNode->mutex);
@@ -1243,6 +1255,9 @@ static void syncProcessBrokenLink(int64_t rid) {
 
   sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd);
   pPeer->peerFd = -1;
+  if (pPeer->isArb) {
+    tsArbOnline = 0;
+  }
 
   syncRestartConnection(pPeer);
   pthread_mutex_unlock(&pNode->mutex);

src/vnode/src/vnodeStatus.c

@@ -44,8 +44,6 @@ bool vnodeSetReadyStatus(SVnodeObj* pVnode) {
       pVnode->status == TAOS_VN_STATUS_UPDATING || pVnode->status == TAOS_VN_STATUS_RESET) {
     pVnode->status = TAOS_VN_STATUS_READY;
     set = true;
-  } else {
-    vDebug("vgId:%d, cannot set status:ready, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]);
   }
 
   qQueryMgmtReOpen(pVnode->qMgmt);
@@ -61,8 +59,6 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) {
   if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_INIT) {
     pVnode->status = TAOS_VN_STATUS_CLOSING;
     set = true;
-  } else {
-    vTrace("vgId:%d, cannot set status:closing, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]);
   }
 
   pthread_mutex_unlock(&pVnode->statusMutex);
@@ -70,11 +66,8 @@ static bool vnodeSetClosingStatusImp(SVnodeObj* pVnode) {
 }
 
 bool vnodeSetClosingStatus(SVnodeObj* pVnode) {
-  int32_t i = 0;
   while (!vnodeSetClosingStatusImp(pVnode)) {
-    if (++i % 1000 == 0) {
-      sched_yield();
-    }
+    taosMsleep(1);
   }
 
   // release local resources only after cutting off outside connections
@@ -92,8 +85,6 @@ bool vnodeSetUpdatingStatus(SVnodeObj* pVnode) {
   if (pVnode->status == TAOS_VN_STATUS_READY) {
     pVnode->status = TAOS_VN_STATUS_UPDATING;
     set = true;
-  } else {
-    vDebug("vgId:%d, cannot set status:updating, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]);
   }
 
   pthread_mutex_unlock(&pVnode->statusMutex);
@@ -107,8 +98,6 @@ static bool vnodeSetResetStatusImp(SVnodeObj* pVnode) {
   if (pVnode->status == TAOS_VN_STATUS_READY || pVnode->status == TAOS_VN_STATUS_INIT) {
     pVnode->status = TAOS_VN_STATUS_RESET;
     set = true;
-  } else {
-    vDebug("vgId:%d, cannot set status:reset, old:%s", pVnode->vgId, vnodeStatus[pVnode->status]);
   }
 
   pthread_mutex_unlock(&pVnode->statusMutex);
@@ -116,11 +105,8 @@ static bool vnodeSetResetStatusImp(SVnodeObj* pVnode) {
 }
 
 bool vnodeSetResetStatus(SVnodeObj* pVnode) {
-  int32_t i = 0;
   while (!vnodeSetResetStatusImp(pVnode)) {
-    if (++i % 1000 == 0) {
-      sched_yield();
-    }
+    taosMsleep(1);
   }
 
   // release local resources only after cutting off outside connections
@@ -167,18 +153,6 @@ bool vnodeInReadyOrUpdatingStatus(SVnodeObj* pVnode) {
   return in;
 }
 
-bool vnodeInClosingStatus(SVnodeObj* pVnode) {
-  bool in = false;
-  pthread_mutex_lock(&pVnode->statusMutex);
-
-  if (pVnode->status == TAOS_VN_STATUS_CLOSING) {
-    in = true;
-  }
-
-  pthread_mutex_unlock(&pVnode->statusMutex);
-  return in;
-}
-
 bool vnodeInResetStatus(SVnodeObj* pVnode) {
   bool in = false;
   pthread_mutex_lock(&pVnode->statusMutex);

tests/script/issue/TD-2680.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c role -v 1
+system sh/cfg.sh -n dnode2 -c role -v 2
+system sh/cfg.sh -n dnode3 -c role -v 2
+system sh/cfg.sh -n dnode4 -c role -v 2
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+sql create dnode $hostname2
+sql create dnode $hostname3
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
+
+print ============== step2
+sql show dnodes
+if $rows != 4 then
+  return -1
+endi
+
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27
+print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37
+
+if $data30 != 0 then
+  return -1
+endi
+
+if $data32 != 0 then
+  return -1
+endi
+
+if $data33 != 0 then
+  return -1
+endi
+
+if $data34 != ready then
+  return -1
+endi
+
+if $data35 != arb then
+  return -1
+endi
+
+if $data37 != - then
+  return -1
+endi
+
+print ============== step4
+system sh/exec_tarbitrator.sh -s stop
+
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+		return -1
+	endi
+
+sql show dnodes
+if $rows != 4 then
+  return -1
+endi
+
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27
+print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37
+
+if $data30 != 0 then
+  return -1
+endi
+
+if $data32 != 0 then
+  return -1
+endi
+
+if $data33 != 0 then
+  return -1
+endi
+
+if $data34 != offline then
+  goto step4
+endi
+
+if $data35 != arb then
+  return -1
+endi
+
+if $data37 != - then
+  return -1
+endi
+
+print ============== step5
+system sh/exec_tarbitrator.sh -s start
+
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+		return -1
+	endi
+
+sql show dnodes
+if $rows != 4 then
+  return -1
+endi
+
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27
+print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37
+
+if $data30 != 0 then
+  return -1
+endi
+
+if $data32 != 0 then
+  return -1
+endi
+
+if $data33 != 0 then
+  return -1
+endi
+
+if $data34 != ready then
+  goto step5
+endi
+
+if $data35 != arb then
+  return -1
+endi
+
+if $data37 != - then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT

tests/script/issue/TD-2713.sim

+system sh/stop_dnodes.sh
+
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c wallevel -v 2
+system sh/cfg.sh -n dnode2 -c wallevel -v 2
+system sh/cfg.sh -n dnode3 -c wallevel -v 2
+system sh/cfg.sh -n dnode4 -c wallevel -v 2
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 3
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c slaveQuery -v 1
+system sh/cfg.sh -n dnode2 -c slaveQuery -v 1
+system sh/cfg.sh -n dnode3 -c slaveQuery -v 1
+system sh/cfg.sh -n dnode4 -c slaveQuery -v 1
+
+print ========= step1 
+system sh/exec.sh -n dnode1 -s start
+sql connect
+sql create dnode $hostname2
+sql create dnode $hostname3
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
+if $data2_2 != slave then
+  goto step1
+endi
+if $data2_3 != slave then
+  goto step1
+endi
+
+print ========= step2
+sql create database d1 replica 3
+sql create table d1.t1 (ts timestamp, i int)
+sql insert into d1.t1 values(now, 1)
+
+$x = 0
+step2: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show d1.vgroups
+print online vgroups: $data03
+if $data03 != 3 then
+  goto step2
+endi
+sleep 1000
+
+print ========= step3
+$i = 0
+while $i < 100
+	$i = $i + 1
+	sql select * from d1.t1
+	print d1.t1 rows: $rows
+	if $rows != 1 then
+		return -1
+	endi	
+endw 
+
+print ========= step4
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+
+system rm -rf ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/*
+system rm -rf ../../../sim/dnode3/data/vnode/vnode2/version.json
+
+system sh/exec.sh -n dnode1 -s start  -x SIGINT
+system sh/exec.sh -n dnode2 -s start  -x SIGINT
+system sh/exec.sh -n dnode3 -s start  -x SIGINT
+
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 30 then
+		return -1
+	endi
+
+sql show d1.vgroups
+print online vgroups: $data03
+if $data03 != 3 then
+  goto step4
+endi
+sleep 1000
+
+print ========= step5
+$i = 0
+while $i < 100
+	$i = $i + 1
+	sql select * from d1.t1
+	if $rows != 1 then
+		return -1
+	endi
+	print d1.t1 rows: $rows
+endw 
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT

tests/script/jenkins/basic.txt

 cd ../../../debug; cmake ..
 cd ../../../debug; make
 
+./test.sh -f issue/TD-2680.sim
+./test.sh -f issue/TD-2713.sim
+
 ./test.sh -f general/alter/cached_schema_after_alter.sim
 ./test.sh -f general/alter/count.sim
 ./test.sh -f general/alter/dnode.sim

tests/script/unique/arbitrator/check_cluster_cfg_para.sim

@@ -115,7 +115,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 7 then
+if $rows != 8 then
   sleep 2000
   goto wait_dnode_created
 endi
@@ -172,7 +172,7 @@ print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
 print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
 print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
 print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode_offline_overtime_dropped
 endi

tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim

@@ -103,7 +103,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode3_offline_0
 endi
@@ -170,7 +170,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode3_reready
 endi
@@ -237,7 +237,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode2_offline
 endi
@@ -282,7 +282,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode2_reready
 endi

tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim

@@ -105,7 +105,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -172,7 +172,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_reready
 endi
@@ -239,7 +239,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode23_offline
 endi
@@ -283,7 +283,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode23_reready
 endi

tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim

@@ -107,7 +107,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_dropped
 endi

tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim

@@ -106,7 +106,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -167,7 +167,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_reready
 endi
@@ -227,7 +227,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode3_offline
 endi
@@ -267,7 +267,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode3_reready
 endi
@@ -343,7 +343,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode2_offline
 endi
@@ -383,7 +383,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode2_reready
 endi

tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim

@@ -107,7 +107,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode3_offline_0
 endi
@@ -198,7 +198,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode3_reready
 endi
@@ -276,7 +276,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode2_offline_0
 endi
@@ -357,7 +357,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode23_reready_2
 endi

tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim

@@ -110,7 +110,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -219,7 +219,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode3_offline_0
 endi

tests/script/unique/arbitrator/insert_duplicationTs.sim

@@ -132,7 +132,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode2_offline
 endi
@@ -190,7 +190,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode2_ready
 endi

tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim

@@ -108,7 +108,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -174,7 +174,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -204,7 +204,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica2_createTable_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -190,7 +190,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -176,7 +176,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -178,7 +178,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -204,7 +204,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica3_createTable_online.sim

@@ -108,7 +108,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -191,7 +191,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -176,7 +176,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/replica_changeWithArbitrator.sim

@@ -115,7 +115,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode2_ready
 endi
@@ -161,7 +161,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode_ready
 endi

tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica2_dropDb.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica2_dropTable.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica3_dropDb.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica3_dropTable.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim

@@ -112,7 +112,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_offline
 endi
@@ -159,7 +159,7 @@ if $loopCnt == 20 then
 endi
 
 sql show dnodes -x wait_dnode1_ready
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_ready
 endi
@@ -238,7 +238,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode2_offline
 endi

tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim

@@ -112,7 +112,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_offline
 endi
@@ -161,7 +161,7 @@ if $loopCnt == 20 then
 endi
 
 sql show dnodes -x wait_dnode1_ready
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_ready
 endi
@@ -234,7 +234,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode2_offline
 endi

tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim

@@ -126,7 +126,7 @@ if $loopCnt == 20 then
 endi
 
 sql show dnodes -x wait_dnode1_ready
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_ready
 endi

tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim

@@ -112,7 +112,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_offline
 endi