TD-2680

src/sync/src/syncMain.c

@@ -1255,6 +1255,9 @@ static void syncProcessBrokenLink(int64_t rid) {
 
   sDebug("%s, TCP link is broken since %s, pfd:%d sfd:%d", pPeer->id, strerror(errno), pPeer->peerFd, pPeer->syncFd);
   pPeer->peerFd = -1;
+  if (pPeer->isArb) {
+    tsArbOnline = 0;
+  }
 
   syncRestartConnection(pPeer);
   pthread_mutex_unlock(&pNode->mutex);

tests/script/issue/TD-2680.sim

+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/deploy.sh -n dnode2 -i 2
+system sh/deploy.sh -n dnode3 -i 3
+system sh/deploy.sh -n dnode4 -i 4
+
+system sh/cfg.sh -n dnode1 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode2 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode3 -c numOfMnodes -v 1
+system sh/cfg.sh -n dnode4 -c numOfMnodes -v 1
+
+system sh/cfg.sh -n dnode1 -c walLevel -v 2
+system sh/cfg.sh -n dnode2 -c walLevel -v 2
+system sh/cfg.sh -n dnode3 -c walLevel -v 2
+system sh/cfg.sh -n dnode4 -c walLevel -v 2
+
+system sh/cfg.sh -n dnode1 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode2 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode3 -c balanceInterval -v 10
+system sh/cfg.sh -n dnode4 -c balanceInterval -v 10
+
+system sh/cfg.sh -n dnode1 -c role -v 1
+system sh/cfg.sh -n dnode2 -c role -v 2
+system sh/cfg.sh -n dnode3 -c role -v 2
+system sh/cfg.sh -n dnode4 -c role -v 2
+
+system sh/cfg.sh -n dnode1 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode2 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode3 -c arbitrator -v $arbitrator
+system sh/cfg.sh -n dnode4 -c arbitrator -v $arbitrator
+
+print ============== step0
+system sh/exec_tarbitrator.sh -s start
+
+print ============== step1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+sql create dnode $hostname2
+sql create dnode $hostname3
+system sh/exec.sh -n dnode2 -s start
+system sh/exec.sh -n dnode3 -s start
+
+$x = 0
+step1: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 10 then
+		return -1
+	endi
+
+sql show dnodes
+print dnode1 $data4_1
+print dnode2 $data4_2
+print dnode3 $data4_3
+
+if $data4_1 != ready then
+  goto step1
+endi
+if $data4_2 != ready then
+  goto step1
+endi
+if $data4_3 != ready then
+  goto step1
+endi
+
+sql show mnodes
+print mnode1 $data2_1
+print mnode1 $data2_2
+print mnode1 $data2_3
+if $data2_1 != master then
+  goto step1
+endi
+
+print ============== step2
+sql show dnodes
+if $rows != 4 then
+  return -1
+endi
+
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27
+print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37
+
+if $data30 != 0 then
+  return -1
+endi
+
+if $data32 != 0 then
+  return -1
+endi
+
+if $data33 != 0 then
+  return -1
+endi
+
+if $data34 != ready then
+  return -1
+endi
+
+if $data35 != arb then
+  return -1
+endi
+
+if $data37 != - then
+  return -1
+endi
+
+print ============== step4
+system sh/exec_tarbitrator.sh -s stop
+
+$x = 0
+step4: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+		return -1
+	endi
+
+sql show dnodes
+if $rows != 4 then
+  return -1
+endi
+
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27
+print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37
+
+if $data30 != 0 then
+  return -1
+endi
+
+if $data32 != 0 then
+  return -1
+endi
+
+if $data33 != 0 then
+  return -1
+endi
+
+if $data34 != offline then
+  goto step4
+endi
+
+if $data35 != arb then
+  return -1
+endi
+
+if $data37 != - then
+  return -1
+endi
+
+print ============== step5
+system sh/exec_tarbitrator.sh -s start
+
+$x = 0
+step5: 
+	$x = $x + 1
+	sleep 1000
+	if $x == 20 then
+		return -1
+	endi
+
+sql show dnodes
+if $rows != 4 then
+  return -1
+endi
+
+print $data00 $data01 $data02 $data03 $data04 $data05 $data06 $data07
+print $data10 $data11 $data12 $data13 $data14 $data15 $data16 $data17
+print $data20 $data21 $data22 $data23 $data24 $data25 $data26 $data27
+print $data30 $data31 $data32 $data33 $data34 $data35 $data36 $data37
+
+if $data30 != 0 then
+  return -1
+endi
+
+if $data32 != 0 then
+  return -1
+endi
+
+if $data33 != 0 then
+  return -1
+endi
+
+if $data34 != ready then
+  goto step5
+endi
+
+if $data35 != arb then
+  return -1
+endi
+
+if $data37 != - then
+  return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop  -x SIGINT
+system sh/exec.sh -n dnode2 -s stop  -x SIGINT
+system sh/exec.sh -n dnode3 -s stop  -x SIGINT
+system sh/exec.sh -n dnode4 -s stop  -x SIGINT

tests/script/issue/TD-2713.sim

@@ -143,7 +143,3 @@ system sh/exec.sh -n dnode1 -s stop  -x SIGINT
 system sh/exec.sh -n dnode2 -s stop  -x SIGINT
 system sh/exec.sh -n dnode3 -s stop  -x SIGINT
 system sh/exec.sh -n dnode4 -s stop  -x SIGINT
-system sh/exec.sh -n dnode5 -s stop  -x SIGINT
-system sh/exec.sh -n dnode6 -s stop  -x SIGINT
-system sh/exec.sh -n dnode7 -s stop  -x SIGINT
-system sh/exec.sh -n dnode8 -s stop  -x SIGINT
\ No newline at end of file

tests/script/jenkins/basic.txt

 cd ../../../debug; cmake ..
 cd ../../../debug; make
 
+./test.sh -f issue/TD-2680.sim
 ./test.sh -f issue/TD-2713.sim
 
 ./test.sh -f general/alter/cached_schema_after_alter.sim

tests/script/unique/arbitrator/check_cluster_cfg_para.sim

@@ -115,7 +115,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 7 then
+if $rows != 8 then
   sleep 2000
   goto wait_dnode_created
 endi
@@ -172,7 +172,7 @@ print $data0_4  $data1_4  $data2_4  $data3_4  $data4_4
 print $data0_5  $data1_5  $data2_5  $data3_5  $data4_5
 print $data0_6  $data1_6  $data2_6  $data3_6  $data4_6
 print $data0_7  $data1_7  $data2_7  $data3_7  $data4_7
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode_offline_overtime_dropped
 endi

tests/script/unique/arbitrator/dn3_mn1_r2_vnode_delDir.sim

@@ -103,7 +103,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode3_offline_0
 endi
@@ -170,7 +170,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode3_reready
 endi
@@ -237,7 +237,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode2_offline
 endi
@@ -282,7 +282,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode2_reready
 endi

tests/script/unique/arbitrator/dn3_mn1_r3_vnode_delDir.sim

@@ -105,7 +105,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -172,7 +172,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_reready
 endi
@@ -239,7 +239,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode23_offline
 endi
@@ -283,7 +283,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode23_reready
 endi

tests/script/unique/arbitrator/dn3_mn1_replica_change_dropDnod.sim

@@ -107,7 +107,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_dropped
 endi

tests/script/unique/arbitrator/dn3_mn1_vnode_delDir.sim

@@ -106,7 +106,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -167,7 +167,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_reready
 endi
@@ -227,7 +227,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode3_offline
 endi
@@ -267,7 +267,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode3_reready
 endi
@@ -343,7 +343,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode2_offline
 endi
@@ -383,7 +383,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode2_reready
 endi

tests/script/unique/arbitrator/dn3_mn1_vnode_noCorruptFile_offline.sim

@@ -107,7 +107,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode3_offline_0
 endi
@@ -198,7 +198,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode3_reready
 endi
@@ -276,7 +276,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode2_offline_0
 endi
@@ -357,7 +357,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode23_reready_2
 endi

tests/script/unique/arbitrator/dn3_mn1_vnode_nomaster.sim

@@ -110,7 +110,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -219,7 +219,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode3_offline_0
 endi

tests/script/unique/arbitrator/insert_duplicationTs.sim

@@ -132,7 +132,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode2_offline
 endi
@@ -190,7 +190,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode2_ready
 endi

tests/script/unique/arbitrator/offline_replica2_alterTable_online.sim

@@ -108,7 +108,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -174,7 +174,7 @@ if $loopCnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica2_alterTag_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -204,7 +204,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica2_createTable_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -190,7 +190,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica2_dropDb_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/offline_replica2_dropTable_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -176,7 +176,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica3_alterTable_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -178,7 +178,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica3_alterTag_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -204,7 +204,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica3_createTable_online.sim

@@ -108,7 +108,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -191,7 +191,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/offline_replica3_dropDb_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/offline_replica3_dropTable_online.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi
@@ -176,7 +176,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_ready
 endi

tests/script/unique/arbitrator/replica_changeWithArbitrator.sim

@@ -115,7 +115,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode2_ready
 endi
@@ -161,7 +161,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode_ready
 endi

tests/script/unique/arbitrator/sync_replica2_alterTable_add.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica2_alterTable_drop.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica2_dropDb.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica2_dropTable.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 3 then
+if $rows != 4 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica3_alterTable_add.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica3_alterTable_drop.sim

@@ -107,7 +107,7 @@ if $cnt == 20 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica3_dropDb.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/arbitrator/sync_replica3_dropTable.sim

@@ -107,7 +107,7 @@ if $cnt == 10 then
   return -1
 endi
 sql show dnodes
-if $rows != 4 then
+if $rows != 5 then
   sleep 2000
   goto wait_dnode4_offline_0
 endi

tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeDir.sim

@@ -112,7 +112,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_offline
 endi
@@ -159,7 +159,7 @@ if $loopCnt == 20 then
 endi
 
 sql show dnodes -x wait_dnode1_ready
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_ready
 endi
@@ -238,7 +238,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode2_offline
 endi

tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir.sim

@@ -112,7 +112,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_offline
 endi
@@ -161,7 +161,7 @@ if $loopCnt == 20 then
 endi
 
 sql show dnodes -x wait_dnode1_ready
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_ready
 endi
@@ -234,7 +234,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode2_offline
 endi

tests/script/unique/migrate/mn2_vn2_repl2_rmMnodeVnodeDir_stopAll_starAll.sim

@@ -126,7 +126,7 @@ if $loopCnt == 20 then
 endi
 
 sql show dnodes -x wait_dnode1_ready
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_ready
 endi

tests/script/unique/migrate/mn2_vn2_repl2_rmVnodeDir.sim

@@ -112,7 +112,7 @@ if $loopCnt == 10 then
 endi
 
 sql show dnodes
-if $rows != 2 then
+if $rows != 3 then
   sleep 2000
   goto wait_dnode1_offline
 endi