[add sim cases]

tests/script/unique/arbitrator/dn2_mn1_cache_file_sync.sim

@@ -2,11 +2,11 @@
 # step 1: start dnode1
 # step 2: start dnode2 and dnode3, and all added into cluster (Suppose dnode2 is master-vnode)
 # step 3: create db, table, insert data, and Falling disc into file (control only one file, e.g. 1841)
-# step 4: insert old data(now-15d) and new data(now+15d), control data rows  in order to save in cache, not falling disc 
+# step 4: insert old data(now-20d) and new data(now-40d), control data rows  in order to save in cache, not falling disc 
 # step 5: stop dnode2, so date rows falling disc, generate two new files 1840, 1842 in dnode2
-# step 6: insert two data rows: now-16d, now+16d
+# step 6: insert two data rows: now-21d, now-41d
 # step 7: restart dnode2, waiting sync end
-# expect: in dnode2, the files 1840 and 1842 will be removed
+# expect: in dnode2, the files 1837 and 1839 will be removed
 
 system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
@@ -14,10 +14,10 @@ system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
 system sh/deploy.sh -n dnode4 -i 4
 
-system sh/cfg.sh -n dnode1 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode2 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode3 -c numOfMPeers -v 1
-system sh/cfg.sh -n dnode4 -c numOfMPeers -v 1
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
 
 system sh/cfg.sh -n dnode1 -c walLevel -v 2
 system sh/cfg.sh -n dnode2 -c walLevel -v 2
@@ -39,11 +39,6 @@ system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
 system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
 
-system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
-system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
-
 system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
 system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
@@ -64,16 +59,16 @@ sql connect
 
 print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data
 system sh/exec.sh -n dnode2 -s start
-#system sh/exec.sh -n dnode3 -s start
+system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname2
-#sql create dnode $hostname3
+sql create dnode $hostname3
 sleep 3000
 
 $totalTableNum = 1
 $sleepTimer = 3000
 
 $db = db
-sql create database $db replica 1 cache 1
+sql create database $db replica 2 cache 1
 sql use $db
 
 # create table , insert data
@@ -82,7 +77,7 @@ sql create table $stb (ts timestamp, c1 double) tags(t1 int)
 $rowNum = 130000
 $tblNum = $totalTableNum
 $totalRows = 0
-#$tsStart = 1420041600000
+$tsStart = 1577808000000  # 2020-01-01 00:00:00.000
 
 # insert over 2M data in order to falling disc, generate one file
 $i = 0
@@ -102,19 +97,24 @@ while $i < $tblNum
 endw 
 
 sql select count(*) from $stb
-sleep 1000
-print data00 $data00
-if $data00 != $totalRows then
-	return -1
+print rows:$rows data00:$data00
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 == 0 then
+  return -1
 endi
 
+$totalRows = $data00
+
 print ============== step3: insert old data(now-15d) and new data(now+15d), control data rows  in order to save in cache, not falling disc
-sql insert into $tb values ( now - 15d , -15 )
-sql insert into $tb values ( now + 15d ,  15 )
+sql insert into $tb values ( now - 20d , -20 )
+sql insert into $tb values ( now - 40d , -40 )
 $totalRows = $totalRows + 2
 
 print ============== step4: stop dnode2, so date rows falling disc, generate two new files in dnode2
-system sh/exec.sh -n dnode2 -s stop
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
 
 wait_dnode2_offline:
@@ -153,48 +153,14 @@ if $data00 != $totalRows then
 endi
 
 print ============== step5: insert two data rows: now-16d, now+16d, 
-sql insert into $tb values ( now - 16d , -16 )
-sql insert into $tb values ( now + 16d ,  16 )
+sql insert into $tb values ( now - 21d , -21 )
+sql insert into $tb values ( now - 41d , -41 )
 $totalRows = $totalRows + 2
 
-return 1
-
-
-
-print ============== step5: restart dnode2, waiting sync end
-system sh/exec.sh -n dnode2 -s start
-sleep 3000
-
-wait_dnode2_ready:
-sql show dnodes
-if $rows != 3 then
-  sleep 2000
-  goto wait_dnode2_ready
-endi
-print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
-print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
-print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
-#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
-#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
-#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
-#$dnode1Status = $data4_1  
-$dnode2Status = $data4_2
-$dnode3Status = $data4_3  
-#$dnode4Status = $data4_4 
-#$dnode5Status = $data4_5
-
-if $dnode2Status != ready then
-  sleep 2000
-  goto wait_dnode2_ready
-endi
-
-sleep $sleepTimer
-# check using select
 sql select count(*) from $stb
 print data00 $data00
 if $data00 != $totalRows then
 	return -1
 endi
 
-print ============== step6: in dnode2, the files 1840 and 1842 will be removed
-# how check in script ???
\ No newline at end of file
+print ============== step6: please check there should be 3 file in sim/dnode2/data/vnode/vnode2/tsdb/data/, and 1 file sim/dnode3/data/vnode/vnode2/tsdb/data/

tests/script/unique/arbitrator/dn2_mn1_cache_file_sync_second.sim

+# Test case describe: dnode1 is only mnode, dnode2/dnode3 are only vnode
+# step 1: start dnode1
+# step 2: start dnode2 and dnode3, and all added into cluster (Suppose dnode2 is master-vnode)
+# step 3: create db, table, insert data, and Falling disc into file (control only one file, e.g. 1841)
+# step 4: insert old data(now-20d) and new data(now-40d), control data rows  in order to save in cache, not falling disc 
+# step 5: stop dnode2, so date rows falling disc, generate two new files 1840, 1842 in dnode2
+# step 6: insert two data rows: now-21d, now-41d
+# step 7: restart dnode2, waiting sync end
+# expect: in dnode2, the files 1837 and 1839 will be removed
+
+sql connect
+sleep 3000
+
+print ============== step7: restart dnode2, waiting sync end
+system sh/exec.sh -n dnode2 -s start
+sleep 3000
+
+wait_dnode2_ready:
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+
+sleep 1000
+# check using select
+$db = db
+$stb = stb
+sql use $db
+sql select count(*) from $stb
+print data00 $data00, should equal to dn2_mn1_cache_file_sync.sim output
+#if $data00 != $totalRows then
+#	return -1
+#endi
+
+print ============== step8: please check there should be 1 file in sim/dnode2/data/vnode/vnode2/tsdb/data/, and 1 file sim/dnode3/data/vnode/vnode2/tsdb/data/

tests/script/unique/arbitrator/dn3_mn1_replica2_wal1_AddDelDnode.sim

@@ -23,28 +23,39 @@ system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode5 -c balanceInterval -v 10
 
-system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 200
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
 system sh/cfg.sh -n dnode5 -c numOfTotalVnodes -v 4
 
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
+
 system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 200
 #system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 200
 #system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 200
 #system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 200
 
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode5 -c arbitrator -v $arbitrator
 
-system sh/cfg.sh -n dnode1 -c offlineThreshold -v 20
-system sh/cfg.sh -n dnode2 -c offlineThreshold -v 20
-system sh/cfg.sh -n dnode3 -c offlineThreshold -v 20
-system sh/cfg.sh -n dnode4 -c offlineThreshold -v 20
-system sh/cfg.sh -n dnode5 -c offlineThreshold -v 20
+system sh/cfg.sh -n dnode1 -c offlineThreshold -v 10
+system sh/cfg.sh -n dnode2 -c offlineThreshold -v 10
+system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
+system sh/cfg.sh -n dnode4 -c offlineThreshold -v 10
+system sh/cfg.sh -n dnode5 -c offlineThreshold -v 10
 
 system sh/cfg.sh -n dnode1 -c enableCoreFile -v 1
 system sh/cfg.sh -n dnode2 -c enableCoreFile -v 1
@@ -63,9 +74,9 @@ sql connect
 print ============== step2: start dnode2/dnode3 and add into cluster, then create database, create table , and insert data
 system sh/exec.sh -n dnode2 -s start
 system sh/exec.sh -n dnode3 -s start
-sleep 1000
 sql create dnode $hostname2
 sql create dnode $hostname3
+sleep 3000
 
 $rowNum = 100
 $tblNum = 16
@@ -151,7 +162,7 @@ endi
 #  return -1
 #endi
 
-sleep 30000
+sleep 15000
 
 wait_drop:
 sql show dnodes
@@ -174,10 +185,69 @@ endi
 if $dnode2Status != ready then
   return -1
 endi
+if $dnode3Status != null then
+  return -1
+endi
 if $dnode4Status != ready then
   return -1
 endi
 
+print ============== step4-1: restart dnode3, adn add into cluster
+system rm -rf ../../sim/dnode3
+sleep 3000
+
+system sh/deploy.sh -n dnode3 -i 3
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c walLevel -v 1
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c offlineThreshold -v 10
+system sh/cfg.sh -n dnode3 -c enableCoreFile -v 1
+
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname3
+sleep 3000
+
+wait_dnode3_ready:
+sql show dnodes
+print rows: $rows
+if $rows != 4 then
+  sleep 3000
+  goto wait_dnode3_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+$dnode4Status = $data4_4  
+$dnode5Status = $data4_5
+
+if $dnode1Status != ready then
+  return -1
+endi
+if $dnode2Status != ready then
+  return -1
+endi
+if $dnode3Status != null then
+  return -1
+endi
+if $dnode4Status != ready then
+  return -1
+endi
+if $dnode5Status != ready then
+  return -1
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
 
 print ============== step5: start dnode5 and add into cluster , drop database
 sql drop database $db
@@ -187,7 +257,7 @@ sql create dnode $hostname5
 sleep 3000
 wait_dnode5:
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 3000
   goto wait_dnode5
 endi
@@ -196,10 +266,12 @@ print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
 print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
 print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
 print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
 $dnode1Status = $data4_1  
 $dnode2Status = $data4_2
 $dnode4Status = $data4_4  
-$dnode5Status = $data4_5
+$dnode5Status = $data4_5 
+$dnode6Status = $data4_6
 
 if $dnode1Status != ready then
   return -1
@@ -207,17 +279,22 @@ endi
 if $dnode2Status != ready then
   return -1
 endi
+if $dnode3Status != null then
+  return -1
+endi
 if $dnode4Status != ready then
   return -1
 endi
 if $dnode5Status != ready then
   return -1
 endi
-
+if $dnode6Status != ready then
+  return -1
+endi
 
 print ============== step6: create database and table until not free vnodes 
 $rowNum = 100
-$tblNum = 24
+$tblNum = 32
 $totalRows = 0
 $tsStart = 1420041600000
 
@@ -259,8 +336,9 @@ if $data00 != $totalRows then
 	return -1
 endi
 
-print ============== step7: drop dnode3, and system should prompt cannot drop dnodes 
-sql_error drop dnode $hostname3
-print ============== step8: add one new table, and system should prompt 'need more dnode' 
+print ============== step7: drop dnode $hostname5, system should prompt "DB error: no enough dnodes" 
+sql_error drop dnode $hostname5
+print error: $error 
+print ============== step8: create table tb_more using $stb tags( 1000 ), system should prompt 'DB error: no enough dnodes' 
 sql_error create table tb_more using $stb tags( 1000 )
-
+print error: $error 

tests/script/unique/arbitrator/dn3_mn1_replica_change.sim

@@ -89,11 +89,8 @@ if $data00 != $totalRows then
 	return -1
 endi
 
-
-
-
-print ============== step2-1: start dnode2 for falling disc, then restart dnode2, and check rows
-system sh/exec.sh -n dnode2 -s stop
+print ============== step2-1: stop dnode2 for falling disc, then restart dnode2, and check rows
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
 sleep $sleepTimer
 wait_dnode2_offline_0:
 sql show dnodes
@@ -151,10 +148,6 @@ if $data00 != $totalRows then
 endi
 
 
-
-
-
-
 print ============== step3: start dnode3 and add into cluster , then alter replica from 1 to 2, and waiting sync
 system sh/exec.sh -n dnode3 -s start
 sql create dnode $hostname3

tests/script/unique/arbitrator/insert_duplicationTs.sim

+# Test case describe: dnode1 is only mnode, dnode2/dnode3 are only vnode
+# step 1: start dnode1
+# step 2: start dnode2 and dnode3, and all added into cluster (Suppose dnode2 is master-vnode)
+# step 3: create db, table, insert data, and Falling disc into file (control only one file, e.g. 1841)
+# step 4: insert old data(now-15d) and new data(now+15d), control data rows  in order to save in cache, not falling disc 
+# step 5: stop dnode2, so date rows falling disc, generate two new files 1840, 1842 in dnode2
+# step 6: insert two data rows: now-16d, now+16d
+# step 7: restart dnode2, waiting sync end
+# expect: in dnode2, the files 1840 and 1842 will be removed
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 4
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 2
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 2
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+
+print ============== step0: start tarbitrator
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1: start dnode1, only deploy mnode
+system sh/exec.sh -n dnode1 -s start
+sleep 3000
+sql connect
+
+print ============== step2: start dnode2/dnode3 and add into cluster , then create database with replica 2, and create table, insert data
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+sql create dnode $hostname2
+sql create dnode $hostname3
+sleep 3000
+
+$totalTableNum = 1
+$sleepTimer = 3000
+
+$db = db
+sql create database $db replica 2 cache 1
+sql use $db
+
+# create table , insert data
+$stb = stb
+sql create table $stb (ts timestamp, c1 double) tags(t1 int)
+$rowNum = 130000
+$tblNum = $totalTableNum
+$totalRows = 0
+$tsStart = 1577808000000  # 2020-01-01 00:00:00.000
+
+# insert over 2M data in order to falling disc, generate one file
+$i = 0
+while $i < $tblNum
+  $tb = tb . $i
+  sql create table $tb using $stb tags( $i )
+ 
+  $x = 0
+  while $x < $rowNum
+#    $ts = $tsStart + $x
+    sql insert into $tb values ( now + 0s , $x ) ( now + 1s , $x ) ( now + 2s , $x ) ( now + 3s , $x ) ( now + 4s , $x ) ( now + 5s , $x ) ( now + 6s , $x ) ( now + 7s , $x ) ( now + 8s , $x ) ( now + 9s , $x )
+    $x = $x + 10
+  endw 
+  $totalRows = $totalRows + $x
+  print       info: inserted $x rows into $tb and totalRows: $totalRows
+  $i = $i + 1
+endw 
+
+sql select count(*) from $stb
+print rows:$rows data00:$data00
+if $rows != 1 then
+  return -1
+endi
+
+if $data00 == 0 then
+  return -1
+endi
+
+$totalRows = $data00
+
+print ============== step3: insert old data(now-15d) and new data(now+15d), control data rows  in order to save in cache, not falling disc
+sql insert into $tb values ( now - 20d , -20 )
+sql insert into $tb values ( now - 40d , -40 )
+$totalRows = $totalRows + 2
+
+print ============== step4: stop dnode2, so date rows falling disc, generate two new files in dnode2
+system sh/exec.sh -n dnode2 -s stop -x SIGINT
+sleep $sleepTimer
+
+wait_dnode2_offline:
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != offline then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+if $dnode3Status != ready then
+  sleep 2000
+  goto wait_dnode2_offline
+endi
+
+sleep $sleepTimer   # waitting for move master vnode of dnode2 to dnode3
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step5: insert two data rows: now-16d, now+16d, 
+sql insert into $tb values ( now - 21d , -21 )
+sql insert into $tb values ( now - 41d , -41 )
+$totalRows = $totalRows + 2
+
+print ============== step5: restart dnode2, waiting sync end
+system sh/exec.sh -n dnode2 -s start
+sleep 3000
+
+wait_dnode2_ready:
+sql show dnodes
+if $rows != 3 then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+#$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+
+sleep $sleepTimer
+# check using select
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+

tests/script/unique/arbitrator/replica_changeWithArbitrator.sim

@@ -2,18 +2,42 @@ system sh/stop_dnodes.sh
 system sh/deploy.sh -n dnode1 -i 1
 system sh/deploy.sh -n dnode2 -i 2
 system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
 
-system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2
-system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2
-system sh/cfg.sh -n dnode3 -c numOfMnodes -v 2
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
 
-system sh/cfg.sh -n dnode1 -c walLevel -v 1
-system sh/cfg.sh -n dnode2 -c walLevel -v 1
-system sh/cfg.sh -n dnode3 -c walLevel -v 1
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
 
 system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
 system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c numOfTotalVnodes -v 8
+system sh/cfg.sh -n dnode2 -c numOfTotalVnodes -v 8
+system sh/cfg.sh -n dnode3 -c numOfTotalVnodes -v 8
+system sh/cfg.sh -n dnode4 -c numOfTotalVnodes -v 8
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+#system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+#system sh/cfg.sh -n dnode3 -c mnodeEqualVnodeNum -v 4
+#system sh/cfg.sh -n dnode4 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode3 -c alternativeRole -v 0
+system sh/cfg.sh -n dnode4 -c alternativeRole -v 0
+
+system sh/cfg.sh -n dnode1 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode2 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode3 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode4 -c maxtablesPerVnode -v 4
+system sh/cfg.sh -n dnode5 -c maxtablesPerVnode -v 4
 
 system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
 system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
@@ -22,21 +46,23 @@ system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
 print ============== step0: start tarbitrator
 system sh/exec_tarbitrator.sh -s start
 
-
-print ============== step1: replica is 1, and start 1 dnode
+print ============== step1: replica is 1, and start 1 dnode, then create tables and insert data
 system sh/exec.sh -n dnode1 -s start
 sleep 3000
 sql connect
 
-$db = replica_db1
-sql create database $db replica 1 maxTables 4
+$totalTableNum = 12
+
+$db = db
+sql create database $db replica 1 maxTables  $totalTableNum
 sql use $db
 
 # create table , insert data
-$stb = repl_stb
+$stb = stb
 sql create table $stb (ts timestamp, c1 int) tags(t1 int)
 $rowNum = 10
-$tblNum = 12
+$tblNum = $totalTableNum
+$totalRows = $rowNum * $tblNum
 
 $ts0 = 1420041600000
 $ts = $ts0
@@ -55,46 +81,136 @@ while $i < $tblNum
     $x = $x + 1
   endw 
   $i = $i + 1
+  print       $tb inserted rows: $x
 endw 
 
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
 print ============== step2: add 1 new dnode, expect balanced
 system sh/exec.sh -n dnode2 -s start
 sql create dnode $hostname2
+sleep 3000
 
 # expect after balanced, 2 vondes in dnode1, 1 vonde in dnode2
-$x = 0
-show2: 
-	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
-		return -1
-	endi
-	
+wait_dnode2_ready:
 sql show dnodes
-print dnode1 openVnodes $data2_1
-print dnode2 openVnodes $data2_2
-if $data2_1 != 2 then
-	goto show2
+if $rows != 2 then
+  sleep 2000
+  goto wait_dnode2_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+#$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode1Status != ready then
+  sleep 2000
+  goto wait_dnode2_ready
 endi
-if $data2_2 != 1 then
-	goto show2
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode2_ready
 endi
 
-print ============== step4: stop dnode1, and wait dnode2 master
+print ============== step3: stop dnode1/dnode2, modify cfg mpeers to 2, and restart dnode1/dnode2
 system sh/exec.sh -n dnode1 -s stop
+system sh/exec.sh -n dnode2 -s stop
+sleep 3000
 
-$x = 0
-loop_wait:
-	$x = $x + 1
-	sleep 2000
-	if $x == 10 then
-	  print ERROR: after dnode1 stop, dnode2 didn't become a master! 
-		return -1
-	endi
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 2
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 2
+
+system sh/cfg.sh -n dnode1 -c mnodeEqualVnodeNum -v 4
+system sh/cfg.sh -n dnode2 -c mnodeEqualVnodeNum -v 4
+
+system sh/cfg.sh -n dnode1 -c alternativeRole -v 1
+system sh/cfg.sh -n dnode2 -c alternativeRole -v 0
+
+system sh/exec.sh -n dnode1 -s start
+system sh/exec.sh -n dnode2 -s start
+sleep 5000
+
+
+print ============= step4: wait dnode ready
+wait_dnode_ready:
+sql show dnodes
+if $rows != 2 then
+  sleep 2000
+  goto wait_dnode_ready
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+#print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+#$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $dnode1Status != ready then
+  sleep 2000
+  goto wait_dnode_ready
+endi
+if $dnode2Status != ready then
+  sleep 2000
+  goto wait_dnode_ready
+endi
+
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
+endi
+
+print ============== step5: stop dnode1
+system sh/exec.sh -n dnode1 -s stop
+sleep 3000
+
+wait_dnode2_master:
 sql show mnodes
-$dnodeRole = $data2_1
-print dnodeRole ==> $dnodeRole
+if $rows != 2 then
+  sleep 2000
+  goto wait_dnode2_master
+endi
+print $data0_1  $data1_1  $data2_1  $data3_1  $data4_1
+print $data0_2  $data1_2  $data2_2  $data3_2  $data4_2
+#print $data0_3  $data1_3  $data2_3  $data3_3  $data4_3
+#print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
+#print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
+#print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
+$dnode1Status = $data4_1  
+$dnode2Status = $data4_2
+#$dnode3Status = $data4_3  
+#$dnode4Status = $data4_4 
+#$dnode5Status = $data4_5
+
+if $data2_1 != offline then
+  sleep 2000
+  goto wait_dnode2_master
+endi
+if $data2_2 != master then
+  sleep 2000
+  goto wait_dnode2_master
+endi
 
-if $dnodeRole != master then
-  goto loop_wait
+sql select count(*) from $stb
+print data00 $data00
+if $data00 != $totalRows then
+	return -1
 endi
+
+