Analysis: DeadIntermediateMasterAndSlaves

go/inst/analysis.go

@@ -45,6 +45,7 @@ const (
 	DeadIntermediateMasterWithSingleSlave                              = "DeadIntermediateMasterWithSingleSlave"
 	DeadIntermediateMasterWithSingleSlaveFailingToConnect              = "DeadIntermediateMasterWithSingleSlaveFailingToConnect"
 	DeadIntermediateMasterAndSomeSlaves                                = "DeadIntermediateMasterAndSomeSlaves"
+	DeadIntermediateMasterAndSlaves                                    = "DeadIntermediateMasterAndSlaves"
 	UnreachableIntermediateMaster                                      = "UnreachableIntermediateMaster"
 	AllIntermediateMasterSlavesFailingToConnectOrDead                  = "AllIntermediateMasterSlavesFailingToConnectOrDead"
 	AllIntermediateMasterSlavesNotReplicating                          = "AllIntermediateMasterSlavesNotReplicating"

go/inst/analysis_dao.go

@@ -318,6 +318,10 @@ func GetReplicationAnalysis(clusterName string, includeDowntimed bool, auditAnal
 			a.Analysis = DeadIntermediateMasterAndSomeSlaves
 			a.Description = "Intermediate master cannot be reached by orchestrator; some of its replicas are unreachable and none of its reachable replicas is replicating"
 			//
+		} else if !a.IsMaster && !a.LastCheckValid && a.CountReplicas > 0 && a.CountValidReplicas == 0 {
+			a.Analysis = DeadIntermediateMasterAndSlaves
+			a.Description = "Intermediate master cannot be reached by orchestrator and all of its replicas are unreachable"
+			//
 		} else if !a.IsMaster && !a.LastCheckValid && a.CountValidReplicas > 0 && a.CountValidReplicatingReplicas > 0 {
 			a.Analysis = UnreachableIntermediateMaster
 			a.Description = "Intermediate master cannot be reached by orchestrator but it has replicating replicas; possibly a network/host issue"

tests/integration/analysis-dead-intermediate-master-and-replicas/create.sql

+-- 22295 replicates from 22294
+UPDATE database_instance SET last_seen=last_checked - interval 1 minute where master_port=22294;
+UPDATE database_instance SET last_seen=last_checked - interval 1 minute where port=22294;

tests/integration/analysis-dead-intermediate-master-and-replicas/expect_output

+testhost:22294 (cluster testhost:22293): DeadIntermediateMasterAndSlaves

tests/integration/analysis-dead-intermediate-master-and-replicas/extra_args

+-c replication-analysis