Merge pull request #2310 from taosdata/hotfix/test

Hotfix/test

tests/script/jenkins/basic.txt

@@ -323,19 +323,20 @@ cd ../../../debug; make
 ./test.sh -f unique/vnode/replica3_repeat.sim
 ./test.sh -f unique/vnode/replica3_vgroup.sim
 
-./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim
-#./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync_second.sim
+./test.sh -f unique/arbitrator/check_cluster_cfg_para.sim
+./test.sh -f unique/arbitrator/dn2_mn1_cache_file_sync.sim                   # TODO: check file number using sim ?
 ./test.sh -f unique/arbitrator/dn3_mn1_full_createTableFail.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim
-./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim
+#./test.sh -f unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim      #  fail
 ./test.sh -f unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_replica_change.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_change.sim
-./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
+#./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim         # fail
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
+#./test.sh -f unique/arbitrator/dn3_mn1_vnode_delDir.sim                  # unsupport
 ./test.sh -f unique/arbitrator/dn3_mn1_vnode_nomaster.sim
 ./test.sh -f unique/arbitrator/dn3_mn2_killDnode.sim
 ./test.sh -f unique/arbitrator/insert_duplicationTs.sim
@@ -349,7 +350,7 @@ cd ../../../debug; make
 ./test.sh -f unique/arbitrator/offline_replica3_createTable_online.sim
 ./test.sh -f unique/arbitrator/offline_replica3_dropDb_online.sim
 ./test.sh -f unique/arbitrator/offline_replica3_dropTable_online.sim
-./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim
+#./test.sh -f unique/arbitrator/replica_changeWithArbitrator.sim                # fail
 ./test.sh -f unique/arbitrator/sync_replica2_alterTable_add.sim
 ./test.sh -f unique/arbitrator/sync_replica2_alterTable_drop.sim
 ./test.sh -f unique/arbitrator/sync_replica2_dropDb.sim

tests/script/unique/arbitrator/check_cluster_cfg_para.sim

@@ -7,84 +7,83 @@ system sh/deploy.sh -n dnode5 -i 5
 system sh/deploy.sh -n dnode6 -i 6
 system sh/deploy.sh -n dnode7 -i 7
 
-
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2
 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode1 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode1 -c offlineThreshold -v 8
 system sh/cfg.sh -n dnode1 -c statusInterval -v 3
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 #system sh/cfg.sh -n dnode1 -c timezone -v ""
 #system sh/cfg.sh -n dnode1 -c locale -v ""
 #system sh/cfg.sh -n dnode1 -c charset -v ""
-system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 3
 
 ######## dnode 2 the same with dnode1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2
 system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode2 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode2 -c offlineThreshold -v 8
 system sh/cfg.sh -n dnode2 -c statusInterval -v 3
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 #system sh/cfg.sh -n dnode2 -c timezone -v ""
 #system sh/cfg.sh -n dnode2 -c locale -v ""
 #system sh/cfg.sh -n dnode2 -c charset -v ""
-system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 3
 
 ######## dnode 3 one para no same with dnode1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 3
 system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode3 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode3 -c offlineThreshold -v 8
 system sh/cfg.sh -n dnode3 -c statusInterval -v 3
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
 #system sh/cfg.sh -n dnode3 -c timezone -v ""
 #system sh/cfg.sh -n dnode3 -c locale -v ""
 #system sh/cfg.sh -n dnode3 -c charset -v ""
-system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 3
 
 ######## dnode 4 one para no same with dnode1
 system sh/cfg.sh -n dnode4 -c numOfMnodes -v 2
 system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 5
-system sh/cfg.sh -n dnode4 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode4 -c offlineThreshold -v 8
 system sh/cfg.sh -n dnode4 -c statusInterval -v 3
 system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 #system sh/cfg.sh -n dnode4 -c timezone -v ""
 #system sh/cfg.sh -n dnode4 -c locale -v ""
 #system sh/cfg.sh -n dnode4 -c charset -v ""
-system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 3
 
 ######## dnode 5 one para no same with dnode1
 system sh/cfg.sh -n dnode5 -c numOfMnodes -v 2
 system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode5 -c offlineThreshold -v 16
+system sh/cfg.sh -n dnode5 -c offlineThreshold -v 7
 system sh/cfg.sh -n dnode5 -c statusInterval -v 3
 system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
 #system sh/cfg.sh -n dnode5 -c timezone -v ""
 #system sh/cfg.sh -n dnode5 -c locale -v ""
 #system sh/cfg.sh -n dnode5 -c charset -v ""
-system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode5 -c balanceInterval -v 3
 
 
 ######## dnode 6 one para no same with dnode1
 system sh/cfg.sh -n dnode6 -c numOfMnodes -v 2
 system sh/cfg.sh -n dnode6 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode6 -c offlineThreshold -v 15 
+system sh/cfg.sh -n dnode6 -c offlineThreshold -v 8 
 system sh/cfg.sh -n dnode6 -c statusInterval -v 2
 system sh/cfg.sh -n dnode6 -c arbitrator -v $arbitrator
 #system sh/cfg.sh -n dnode6 -c timezone -v ""
 #system sh/cfg.sh -n dnode6 -c locale -v ""
 #system sh/cfg.sh -n dnode6 -c charset -v ""
-system sh/cfg.sh -n dnode6 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode6 -c balanceInterval -v 3
 
 
 ######## dnode 7 one para no same with dnode1
 system sh/cfg.sh -n dnode7 -c numOfMnodes -v 2
 system sh/cfg.sh -n dnode7 -c mnodeEqualVnodeNum -v 4
-system sh/cfg.sh -n dnode7 -c offlineThreshold -v 15
+system sh/cfg.sh -n dnode7 -c offlineThreshold -v 8
 system sh/cfg.sh -n dnode7 -c statusInterval -v 3
 system sh/cfg.sh -n dnode7 -c arbitrator -v "plum-VirtualBox:8001"
 #system sh/cfg.sh -n dnode7 -c timezone -v ""
 #system sh/cfg.sh -n dnode7 -c locale -v ""
 #system sh/cfg.sh -n dnode7 -c charset -v ""
-system sh/cfg.sh -n dnode7 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode7 -c balanceInterval -v 3
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -109,7 +108,12 @@ sql create dnode $hostname6
 sql create dnode $hostname7
 sleep 10000
 
+$loopCnt = 0
 wait_dnode_created:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
 sql show dnodes
 if $rows != 7 then
   sleep 2000
@@ -154,7 +158,12 @@ endi
 
 sleep 10000
 
+$loopCnt = 0
 wait_dnode_offline_overtime_dropped:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
 sql show dnodes
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2

tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim

@@ -117,7 +117,13 @@ print ============== step4: stop dnode3, so date rows falling disc, generate two
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
 sleep $sleepTimer
 
+$loopCnt = 0
 wait_dnode3_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -179,7 +185,13 @@ print ============== step7: restart dnode3, waiting sync end
 system sh/exec.sh -n dnode3 -s start
 sleep 3000
 
+$loopCnt = 0
 wait_dnode3_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000

tests/script/unique/arbitrator/dn3_mn1_full_createTableFail.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start

tests/script/unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -37,6 +38,7 @@ system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start

tests/script/unique/arbitrator/dn3_mn1_multiCreateDropTable.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start

tests/script/unique/arbitrator/dn3_mn1_nw_disable_timeout_autoDropDnode.sim

@@ -3,37 +3,31 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
 system sh/deploy.sh -n dnode4 -i 4
-system sh/deploy.sh -n dnode5 -i 5
 
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
-system sh/cfg.sh -n dnode5 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 1 
 system sh/cfg.sh -n dnode2 -c walLevel -v 1
 system sh/cfg.sh -n dnode3 -c walLevel -v 1
 system sh/cfg.sh -n dnode4 -c walLevel -v 1
-system sh/cfg.sh -n dnode5 -c walLevel -v 1
 
 system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
-system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
 
 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4
 
 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
 system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
@@ -44,19 +38,16 @@ system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
-system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
 
 system sh/cfg.sh -n dnode1 -c offlineThreshold -v 10
 system sh/cfg.sh -n dnode2 -c offlineThreshold -v 10
 system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
 system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10
-system sh/cfg.sh -n dnode5 -c offlineThreshold -v 10
 
 system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1
 system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1
 system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1
 system sh/cfg.sh -n dnode4 -c enableCoreFile -v 1
-system sh/cfg.sh -n dnode5 -c enableCoreFile -v 1
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -126,7 +117,13 @@ system sh/exec.sh -n dnode4 -s stop
 #system sh/port.sh -p 7400 -s down
 sleep 12000
 
+$loopCnt = 0
 wait_dnode4_dropped:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000

tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim

@@ -36,14 +36,16 @@ system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 200
-#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 200
-#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200
-#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 200
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 200
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200
+system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 200
+system sh/cfg.sh -n dnode5 -c mnodeEqualVnodeNum -v 200
 
 system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
 system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode5 -c alternativeRole -v 2
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
@@ -71,7 +73,7 @@ system sh/exec.sh -n dnode1 -s start
 sleep 3000
 sql connect
 
-print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data
+print ============== step2: start dnode2/dnode3 and add into cluster, then create database replica 2, create table , and insert data
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname2
@@ -84,7 +86,7 @@ $totalRows = 0
 $tsStart = 1420041600000
 
 $db = db
-sql create database $db replica 2 maxTables 4
+sql create database $db replica 2
 sql use $db
 
 # create table , insert data
@@ -164,7 +166,13 @@ endi
 
 sleep 15000
 
+$loopCnt = 0
 wait_drop:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 3000
@@ -193,14 +201,17 @@ if $dnode4Status != ready then
 endi
 
 print ============== step4-1: restart dnode3, adn add into cluster
-system rm -rf ../../sim/dnode3
-sleep 3000
+system rm -rf ../../../sim/dnode3
+sleep 1000
 
 system sh/deploy.sh -n dnode3 -i 3
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c walLevel -v 1
 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
 system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1
@@ -209,7 +220,13 @@ system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname3
 sleep 3000
 
+$loopCnt = 0
 wait_dnode3_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 print rows: $rows
 if $rows != 4 then
@@ -228,19 +245,19 @@ $dnode4Status = $data4_4
 $dnode5Status = $data4_5
 
 if $dnode1Status != ready then
-  return -1
+  goto wait_dnode3_ready
 endi
 if $dnode2Status != ready then
-  return -1
+  goto wait_dnode3_ready
 endi
 if $dnode3Status != null then
-  return -1
+  goto wait_dnode3_ready
 endi
 if $dnode4Status != ready then
-  return -1
+  goto wait_dnode3_ready
 endi
 if $dnode5Status != ready then
-  return -1
+  goto wait_dnode3_ready
 endi
 
 sql select count(*) from $stb
@@ -255,7 +272,13 @@ sleep 1000
 system sh/exec.sh -n dnode5 -s start
 sql create dnode $hostname5
 sleep 3000
+$loopCnt = 0
 wait_dnode5:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 5 then
   sleep 3000
@@ -274,22 +297,22 @@ $dnode5Status = $data4_5
 $dnode6Status = $data4_6
 
 if $dnode1Status != ready then
-  return -1
+  goto wait_dnode5
 endi
 if $dnode2Status != ready then
-  return -1
+  goto wait_dnode5
 endi
 if $dnode3Status != null then
-  return -1
+  goto wait_dnode5
 endi
 if $dnode4Status != ready then
-  return -1
+  goto wait_dnode5
 endi
 if $dnode5Status != ready then
-  return -1
+  goto wait_dnode5
 endi
 if $dnode6Status != ready then
-  return -1
+  goto wait_dnode5
 endi
 
 print ============== step6: create database and table until not free vnodes 

tests/script/unique/arbitrator/dn3_mn1_replica_change.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -92,7 +93,14 @@ endi
 print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
+
+
+$loopCnt = 0
 wait_dnode2_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
 sql show dnodes
 if $rows != 2 then
   sleep 2000
@@ -117,7 +125,14 @@ endi
 
 system sh/exec.sh -n dnode2 -s start
 sleep $sleepTimer
+
+
+$loopCnt = 0
 wait_dnode2_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
 sql show dnodes
 if $rows != 2 then
   sleep 2000
@@ -155,8 +170,13 @@ sleep 3000
 
 sql alter database $db replica 2
 sleep $sleepTimer
-
+$loopCnt = 0
 wait_dnode3_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -195,8 +215,13 @@ endi
 print ============== step4: stop dnode2 for checking if sync ok
 system sh/exec.sh -n dnode2 -s stop
 sleep $sleepTimer
-
+$loopCnt = 0
 wait_dnode2_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -235,7 +260,13 @@ print ============== step5: restart dnode2
 system sh/exec.sh -n dnode2 -s start
 sleep 3000
 
+$loopCnt = 0
 wait_dnode2_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -273,8 +304,13 @@ sleep 3000
 
 sql alter database $db replica 3
 sleep $sleepTimer
-
+$loopCnt = 0
 wait_dnode4_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -308,8 +344,13 @@ endi
 print ============== step7: alter replica from 3 to 2, and waiting sync
 sql alter database $db replica 2
 sleep $sleepTimer
-
+$loopCnt = 0
 wait_vgroups_replic_to_2:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show vgroups
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2
@@ -350,8 +391,13 @@ endi
 print ============== step8: alter replica from 2 to 1, and waiting sync
 sql alter database $db replica 1
 sleep $sleepTimer
-
+$loopCnt = 0
 wait_vgroups_replic_to_1:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show vgroups
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1  $data5_1  $data6_1  $data7_1  $data8_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2  $data5_2  $data6_2  $data7_2  $data8_2
@@ -381,7 +427,13 @@ if $sencodDnode_5 != null then
   goto wait_vgroups_replic_to_1
 endi
 
+$loopCnt = 0
 all_dnodes_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -429,7 +481,13 @@ sql drop dnode $hostname2
 sql drop dnode $hostname3
 sleep $sleepTimer
 
+$loopCnt = 0
 wait_dnode23_dropped:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 2 then
   sleep 2000

tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -97,8 +98,13 @@ print ============== step3: drop dnode4, then check rows
 #system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sql drop dnode $hostname4
 sleep $sleepTimer
-
+$loopCnt = 0
 wait_dnode4_dropped:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000

tests/script/unique/arbitrator/dn3_mn1_stopDnode_timeout.sim

@@ -3,37 +3,31 @@ system sh/deploy.sh -n dnode1 -i 1
 system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
 system sh/deploy.sh -n dnode4 -i 4
-system sh/deploy.sh -n dnode5 -i 5
 
-system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode5 -c numOfMPeers -v 1
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 1 
 system sh/cfg.sh -n dnode2 -c walLevel -v 1
 system sh/cfg.sh -n dnode3 -c walLevel -v 1
 system sh/cfg.sh -n dnode4 -c walLevel -v 1
-system sh/cfg.sh -n dnode5 -c walLevel -v 1
 
 system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
-system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
 
 system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4
 
 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
 system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
@@ -44,19 +38,16 @@ system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
-system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
 
 system sh/cfg.sh -n dnode1 -c offlineThreshold -v 10
 system sh/cfg.sh -n dnode2 -c offlineThreshold -v 10
 system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
 system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10
-system sh/cfg.sh -n dnode5 -c offlineThreshold -v 10
 
 system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1
 system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1
 system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1
 system sh/cfg.sh -n dnode4 -c enableCoreFile -v 1
-system sh/cfg.sh -n dnode5 -c enableCoreFile -v 1
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -124,7 +115,13 @@ print ============== step3: stop dnode4, after timerout dnode4 will be auto-drop
 system sh/exec.sh -n dnode4 -s stop
 sleep 12000
 
+$loopCnt = 0
 wait_dnode4_dropped:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -178,7 +175,7 @@ system sh/exec.sh -n dnode4 -s stop
 system rm -rf ../../../sim/dnode4
 
 system sh/deploy.sh -n dnode4 -i 4
-system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode4 -c walLevel -v 1
 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
@@ -192,7 +189,13 @@ system sh/exec.sh -n dnode4 -s start
 sql create dnode $hostname4
 sleep 6000
 
+$loopCnt = 0
 wait_dnode4_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then 
+  return -1
+endi
+
 sql show dnodes
 if $rows != 4 then
   sleep 2000

tests/script/unique/arbitrator/dn3_mn1_vnode_change.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -46,7 +47,7 @@ system sh/exec.sh -n dnode1 -s start
 sleep 3000
 sql connect
 
-print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 3, and create table, insert data
+print ============== step2: start dnode2/dnode3/dnode4 and add into cluster , then create database with replica 2, and create table, insert data
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
 system sh/exec.sh -n dnode4 -s start
@@ -59,7 +60,7 @@ $totalTableNum = 10
 $sleepTimer = 3000
 
 $db = db
-sql create database $db replica 3 maxTables $totalTableNum
+sql create database $db replica 2 maxTables $totalTableNum
 sql use $db
 
 # create table , insert data
@@ -87,13 +88,18 @@ while $i < $tblNum
 endw 
 
 sql select count(*) from $stb
-sleep 1000
 print data00 $data00
 if $data00 != $totalRows then
 	return -1
 endi
 
-print ============== step3: stop dnode4, and remove its vnodeX subdirector 
+sql show dnodes
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+
+print ============== step3: stop dnode4, then destroy the contents of its data file
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
 $loopCnt = 0
@@ -110,14 +116,11 @@ endi
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+$dnode1Status = $data4_1  
 $dnode2Status = $data4_2
 $dnode3Status = $data4_3  
 $dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
 
 if $dnode4Status != offline then
   sleep 2000
@@ -146,10 +149,20 @@ if $dnode3Vtatus != master then
   goto wait_dnode4_vgroup_offline
 endi
 
-
-system rm -rf ../../../sim/dnode4/data/vnode/*
+system echo "haha, nothing......" > ../../../sim/dnode4/data/vnode/vnode2/tsdb/data/f1643.data
+#system rm -rf ../../../sim/dnode4/data/vnode/*
 sleep 1000
 
+print ============== step3-1: insert new data 
+sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x )
+$totalRows = $totalRows + 3
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
 print ============== step4: restart dnode4, waiting sync end 
 system sh/exec.sh -n dnode4 -s start
 sleep $sleepTimer
@@ -213,12 +226,17 @@ if $dnode3Vtatus != master then
   goto wait_dnode4_vgroup_slave
 endi
 
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
 print ============== step5: stop dnode3/dnode2, and check rows
-system sh/exec.sh -n dnode2 -s stop
-system sh/exec.sh -n dnode3 -s stop
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+system sh/exec.sh -n dnode3 -s stop -x SIGINT
 sleep $sleepTimer
 
-
 $loopCnt = 0
 wait_dnode23_offline:
 $loopCnt = $loopCnt + 1
@@ -234,13 +252,10 @@ print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
 print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
+$dnode1Status = $data4_1  
 $dnode2Status = $data4_2
 $dnode3Status = $data4_3  
 $dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
 
 if $dnode2Status != offline then
   sleep 2000

tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -96,7 +97,14 @@ endi
 print ============== step3: stop dnode3 for falling disc, then corrupt vnode data file in dnode3
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
 sleep $sleepTimer
+
+$loopCnt = 0
 wait_dnode3_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -119,7 +127,13 @@ if $dnode3Status != offline then
   goto wait_dnode3_offline_0
 endi
 
+$loopCnt = 0
 wait_dnode3_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -154,14 +168,25 @@ print ---->dnode3 data files: [ $system_content ], expect is 3
 #  return -1
 #endi
 
-#system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data
-#sleep 1000
+system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data
+sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x )
+$totalRows = $totalRows + 3
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
 
 print ============== step4: restart dnode3, and run query 
 system sh/exec.sh -n dnode3 -s start
 sleep $sleepTimer
-
+$loopCnt = 0
 wait_dnode3_reready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -183,8 +208,13 @@ if $dnode3Status != ready then
   sleep 2000
   goto wait_dnode3_reready
 endi
-
+$loopCnt = 0
 wait_dnode3_vgroup_slave:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -225,12 +255,16 @@ print ----> dnode3 data files: [ $system_content ], expect is 0
 #  return -1
 #endi
 
-return -1
-
 print ============== step5: stop dnode2, and check if dnode3 sync ok
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
+
+$loopCnt = 0
 wait_dnode2_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -253,7 +287,13 @@ if $dnode2Status != offline then
   goto wait_dnode2_offline_0
 endi
 
+$loopCnt = 0
 wait_dnode3_vgroup_master:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+
 sql show vgroups
 if $rows != 1 then
   sleep 2000

tests/script/unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -94,11 +95,13 @@ if $data00 != $totalRows then
 endi
 
 print ============== step3: corrupt vnode data file in dnode3, not stop dnode3
-# del the second row
-system sed '2d' ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/v1849.data
+system echo "haha, nothing......" > ../../../sim/dnode3/data/vnode/vnode2/tsdb/data/f1643.data
 sleep 1000
 
-print ============== step4: run query 
+print ============== step4: insert new data, and run query 
+sql insert into $tb values ( now + 0a , $x ) ( now + 1a , $x ) ( now + 2a , $x ) 
+$totalRows = $totalRows + 3
+
 sql select count(*) from $stb
 print data00 $data00
 if $data00 != $totalRows then
@@ -108,7 +111,14 @@ endi
 print ============== step5: stop dnode2, and check if dnode3 sync ok
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
+
+$loopCnt = 0
 wait_dnode2_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -131,7 +141,13 @@ if $dnode2Status != offline then
   goto wait_dnode2_offline_0
 endi
 
+$loopCnt = 0
 wait_dnode3_vgroup_master:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+
 sql show vgroups
 if $rows != 1 then
   sleep 2000

tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -96,7 +97,13 @@ print ============== step3: stop dnode4/dnode2
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
+
+$loopCnt = 0
 wait_dnode4_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -123,7 +130,13 @@ if $dnode2Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$loopCnt = 0
 wait_dnode4_vgroup_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -192,7 +205,12 @@ endi
 print ============== step5: stop dnode3
 system sh/exec.sh -n dnode3 -s stop -x SIGINT
 sleep $sleepTimer
+$loopCnt = 0
 wait_dnode3_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -221,7 +239,12 @@ if $dnode2Status != ready then
   goto wait_dnode3_offline_0
 endi
 
+$loopCnt = 0
 wait_dnode2_vgroup_master:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -243,8 +266,8 @@ if $dnode2Vtatus != master then
   goto wait_dnode2_vgroup_master
 endi
 
-sql insert into tb98 values (now, 9000) (now + 1s, 9001) (now + 2s, 9002) tb99 values (now, 9000) (now + 1s, 9001) (now + 2s, 9002)
-$totalRows = $totalRows + 6
+sql insert into $tb values (now, 9000) (now + 1s, 9001) (now + 2s, 9002)
+$totalRows = $totalRows + 3
 sql select count(*) from $stb
 print data00 $data00
 if $data00 != $totalRows then

tests/script/unique/arbitrator/dn3_mn2_killDnode.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -32,11 +33,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start

tests/script/unique/arbitrator/insert_duplicationTs.sim

@@ -43,11 +43,11 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -117,7 +117,13 @@ print ============== step4: stop dnode2, so date rows falling disc, generate two
 system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
 
+$loopCnt = 0
 wait_dnode2_offline:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -160,8 +166,13 @@ $totalRows = $totalRows + 2
 print ============== step5: restart dnode2, waiting sync end
 system sh/exec.sh -n dnode2 -s start
 sleep 3000
-
+$loopCnt = 0
 wait_dnode2_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
+
 sql show dnodes
 if $rows != 3 then
   sleep 2000

tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim

@@ -99,7 +99,13 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+
+$loopCnt = 0
 wait_dnode4_offline_0:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -160,7 +166,12 @@ $totalRows = $totalRows + $addRows
 print ============== step5: restart dnode4, waiting dnode4 synced
 system sh/exec.sh -n dnode4 -s start
 
+$loopCnt = 0
 wait_dnode4_ready:
+$loopCnt = $loopCnt + 1
+if $loopCnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000

tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim

@@ -100,7 +100,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -123,7 +128,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -187,7 +197,12 @@ endi
 print ============== step5: restart dnode4, waiting dnode4 synced
 system sh/exec.sh -n dnode4 -s start
 
+$cnt = 0
 wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000

tests/script/unique/arbitrator/offline_replica2_createTable_online.sim

@@ -101,7 +101,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -124,7 +129,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -174,7 +184,12 @@ endi
 print ============== step5: restart dnode4, waiting dnode4 synced
 system sh/exec.sh -n dnode4 -s start
 
+$cnt = 0
 wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000

tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000

tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -158,7 +168,12 @@ $tblNum = $tblNum - 5
 print ============== step5: restart dnode4, waiting dnode4 synced
 system sh/exec.sh -n dnode4 -s start
 
+$cnt = 0
 wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000

tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -160,7 +170,12 @@ $totalRows = $totalRows + $addRows
 print ============== step5: restart dnode4, waiting dnode4 synced
 system sh/exec.sh -n dnode4 -s start
 
+$cnt = 0
 wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000

tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim

@@ -100,7 +100,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -123,7 +128,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -187,7 +197,12 @@ endi
 print ============== step5: restart dnode4, waiting dnode4 synced
 system sh/exec.sh -n dnode4 -s start
 
+$cnt = 0
 wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000

tests/script/unique/arbitrator/offline_replica3_createTable_online.sim

@@ -100,7 +100,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -123,7 +128,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -173,7 +183,12 @@ endi
 print ============== step5: restart dnode4, waiting dnode4 synced
 system sh/exec.sh -n dnode4 -s start
 
+$cnt = 0
 wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000

tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000

tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -158,7 +168,12 @@ $tblNum = $tblNum - 5
 print ============== step5: restart dnode4, waiting dnode4 synced
 system sh/exec.sh -n dnode4 -s start
 
+$cnt = 0
 wait_dnode4_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000

tests/script/unique/arbitrator/replica_changeWithArbitrator.sim

@@ -7,6 +7,7 @@ system sh/deploy.sh -n dnode4 -i 4
 system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
 system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -37,11 +38,21 @@ system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
-system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+system sh/cfg.sh -n dnode1 -c offlineThreshold -v 8
+system sh/cfg.sh -n dnode2 -c offlineThreshold -v 8
+system sh/cfg.sh -n dnode3 -c offlineThreshold -v 8
+system sh/cfg.sh -n dnode4 -c offlineThreshold -v 8
+
+system sh/cfg.sh -n dnode1 -c statusInterval -v 3
+system sh/cfg.sh -n dnode2 -c statusInterval -v 3
+system sh/cfg.sh -n dnode3 -c statusInterval -v 3
+system sh/cfg.sh -n dnode4 -c statusInterval -v 3
 
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
@@ -96,7 +107,12 @@ sql create dnode $hostname2
 sleep 3000
 
 # expect after balanced, 2 vondes in dnode1, 1 vonde in dnode2
+$cnt = 0
 wait_dnode2_ready:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 2 then
   sleep 2000
@@ -105,14 +121,8 @@ endi
 print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
 print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
 $dnode1Status = $data4_1  
 $dnode2Status = $data4_2
-#$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
 
 if $dnode1Status != ready then
   sleep 2000
@@ -123,7 +133,7 @@ if $dnode2Status != ready then
   goto wait_dnode2_ready
 endi
 
-print ============== step3: stop dnode1/dnode2, modify cfg mpeers to 2, and restart dnode1/dnode2
+print ============== step3: stop dnode1/dnode2, modify cfg numOfMnodes to 2, and restart dnode1/dnode2
 system sh/exec.sh -n dnode1 -s stop
 system sh/exec.sh -n dnode2 -s stop
 sleep 3000
@@ -143,7 +153,12 @@ sleep 5000
 
 
 print ============= step4: wait dnode ready
+$cnt = 0
 wait_dnode_ready:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 2 then
   sleep 2000
@@ -180,7 +195,12 @@ print ============== step5: stop dnode1
 system sh/exec.sh -n dnode1 -s stop
 sleep 3000
 
+$cnt = 0
 wait_dnode2_master:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show mnodes
 if $rows != 2 then
   sleep 2000

tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000

tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -172,7 +182,7 @@ system sh/exec.sh -n dnode4 -s start
 run_back unique/arbitrator/sync_replica_alterTable_background_drop.sim
 
 print ============== step6: check result
-#in background.sim, drop one column and add one new column, then insert 200 rows
+#in background.sim, drop one column and add one new column, then insert 36 rows
 $totalRows = $totalRows + 36
 
 $cnt = 0

tests/script/unique/arbitrator/sync_replica2_dropDb.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000

tests/script/unique/arbitrator/sync_replica2_dropTable.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 3 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000

tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -162,7 +172,7 @@ while $i < $tblNum
 endw 
 
 sql select count(*) from $stb
-print data00 $data00
+print data00:$data00   totalRows:$totalRows 
 if $data00 != $totalRows then
 	return -1
 endi
@@ -172,7 +182,7 @@ system sh/exec.sh -n dnode4 -s start
 run_back unique/arbitrator/sync_replica_alterTable_background_add.sim
 
 print ============== step6: check result
-#in background.sim, add one column and insert 200 rows
+#in background.sim, add one column and insert 36 rows
 $totalRows = $totalRows + 36
 
 $cnt = 0
@@ -183,7 +193,7 @@ if $cnt == 20 then
 endi
 sql select count(*) from $stb
 if $data00 != $totalRows then
-  print data00: $data00 totalRows: $totalRows
+  print data00:$data00   totalRows:$totalRows 
 	sleep 2000
 	goto wait_table_altered
 endi

tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 20 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000

tests/script/unique/arbitrator/sync_replica3_dropDb.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000

tests/script/unique/arbitrator/sync_replica3_dropTable.sim

@@ -99,7 +99,12 @@ endi
 print ============== step3: stop dnode4 
 system sh/exec.sh -n dnode4 -s stop -x SIGINT
 sleep $sleepTimer
+$cnt = 0
 wait_dnode4_offline_0:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show dnodes
 if $rows != 4 then
   sleep 2000
@@ -122,7 +127,12 @@ if $dnode4Status != offline then
   goto wait_dnode4_offline_0
 endi
 
+$cnt = 0
 wait_dnode4_vgroup_offline:
+$cnt = $cnt + 1
+if $cnt == 10 then
+  return -1
+endi
 sql show vgroups
 if $rows != 1 then
   sleep 2000
@@ -143,7 +153,7 @@ if $dnode3Vtatus != master then
   sleep 2000
   goto wait_dnode4_vgroup_offline
 endi
-sleep 2000
+
 print ============== step4: insert more data rows
 $tsStart = $tsEnd + 1000
 $i = 0
@@ -162,17 +172,17 @@ while $i < $tblNum
 endw 
 
 sql select count(*) from $stb
-print data00 $data00
+print data00:$data00  totalRows:$totalRows
 if $data00 != $totalRows then
 	return -1
 endi
 
-print ============== step5: restart dnode4, while drop database in other thead when dnode4 is syncing
+print ============== step5: restart dnode4, while drop some tables in other thread when dnode4 is syncing
 system sh/exec.sh -n dnode4 -s start
 run_back unique/arbitrator/sync_replica_dropTable_background.sim
 
 print ============== step6: check result
-#in background.sim, drop 10 tables
+#in background.sim, drop 5 tables
 $totalRows = $totalRows - 5400
 
 $cnt = 0
@@ -183,7 +193,7 @@ if $cnt == 20 then
 endi
 sql select count(*) from $stb
 if $data00 != $totalRows then
-  print data00: $data00
+  print data00:$data00  totalRows:$totalRows
 	sleep 2000
 	goto wait_table_dropped
 endi
@@ -191,7 +201,7 @@ endi
 $tblNum = $tblNum - 5
 sql select count(tbname) from $stb
 if $data00 != $tblNum then
-  print data00: $data00
+  print data00: $data00   tblNum: $tblNum
 	sleep 2000
 	goto wait_table_dropped
 endi

tests/script/unique/arbitrator/testSuite.sim

+run unique/arbitrator/check_cluster_cfg_para.sim
 run unique/arbitrator/dn2_mn1_cache_file_sync.sim
-run unique/arbitrator/dn2_mn1_cache_file_sync_second.sim
 run unique/arbitrator/dn3_mn1_full_createTableFail.sim
 run unique/arbitrator/dn3_mn1_full_dropDnodeFail.sim
 run unique/arbitrator/dn3_mn1_multiCreateDropTable.sim
@@ -11,7 +11,7 @@ run unique/arbitrator/dn3_mn1_stopDnode_timeout.sim
 run unique/arbitrator/dn3_mn1_vnode_change.sim
 run unique/arbitrator/dn3_mn1_vnode_corruptFile_offline.sim
 run unique/arbitrator/dn3_mn1_vnode_corruptFile_online.sim
-####run unique/arbitrator/dn3_mn1_vnode_delDir.sim
+run unique/arbitrator/dn3_mn1_vnode_delDir.sim
 run unique/arbitrator/dn3_mn1_vnode_nomaster.sim
 run unique/arbitrator/dn3_mn2_killDnode.sim
 run unique/arbitrator/insert_duplicationTs.sim
@@ -33,4 +33,4 @@ run unique/arbitrator/sync_replica2_dropTable.sim
 run unique/arbitrator/sync_replica3_alterTable_add.sim
 run unique/arbitrator/sync_replica3_alterTable_drop.sim
 run unique/arbitrator/sync_replica3_dropDb.sim
-run unique/arbitrator/sync_replica3_dropTable.sim
+run unique/arbitrator/sync_replica3_dropTable.sim
\ No newline at end of file