diff --git a/test/integration/helpers.go b/test/integration/helpers.go index b997a035f69ea4e9c11376c2e2b2fb5bbe1dc83e..cc3036a7e93884e03e76f913d9a548bd67bcaa84 100644 --- a/test/integration/helpers.go +++ b/test/integration/helpers.go @@ -191,7 +191,7 @@ func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) { } // PostMortemLogs shows logs for debugging a failed cluster -func PostMortemLogs(t *testing.T, profile string, node ...string) { +func PostMortemLogs(t *testing.T, profile string, multinode ...bool) { if !t.Failed() { return } @@ -201,57 +201,73 @@ func PostMortemLogs(t *testing.T, profile string, node ...string) { return } + m := false + if len(multinode) > 0 { + m = multinode[0] + } + + nodes := []string{profile} + if m { + nodes = append(nodes, SecondNodeName, ThirdNodeName) + } + t.Logf("-----------------------post-mortem--------------------------------") - if DockerDriver() { - t.Logf("======> post-mortem[%s]: docker inspect <======", t.Name()) - rr, err := Run(t, exec.Command("docker", "inspect", profile)) - if err != nil { - t.Logf("failed to get docker inspect: %v", err) - } else { - t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) + for _, n := range nodes { + machine := profile + if n != profile { + machine = fmt.Sprintf("%s-%s", profile, n) + } + if DockerDriver() { + t.Logf("======> post-mortem[%s]: docker inspect <======", t.Name()) + rr, err := Run(t, exec.Command("docker", "inspect", machine)) + if err != nil { + t.Logf("failed to get docker inspect: %v", err) + } else { + t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) + } } - } - st := Status(context.Background(), t, Target(), profile, "Host") - if st != state.Running.String() { - t.Logf("%q host is not running, skipping log retrieval (state=%q)", profile, st) - return - } - t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name()) - t.Logf("======> post-mortem[%s]: minikube logs <======", t.Name()) + st := Status(context.Background(), t, Target(), profile, "Host", n) + if st != state.Running.String() { + t.Logf("%q host is not running, skipping log retrieval (state=%q)", profile, st) + return + } + t.Logf("<<< %s FAILED: start of post-mortem logs <<<", t.Name()) + t.Logf("======> post-mortem[%s]: minikube logs <======", t.Name()) - rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "25")) - if err != nil { - t.Logf("failed logs error: %v", err) - return - } - t.Logf("%s logs: %s", t.Name(), rr.Output()) + rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "25")) + if err != nil { + t.Logf("failed logs error: %v", err) + return + } + t.Logf("%s logs: %s", t.Name(), rr.Output()) - st = Status(context.Background(), t, Target(), profile, "APIServer") - if st != state.Running.String() { - t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st) - return - } + st = Status(context.Background(), t, Target(), profile, "APIServer", n) + if st != state.Running.String() { + t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st) + return + } - // Get non-running pods. NOTE: This does not yet contain pods which are "running", but not "ready" - rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-o=jsonpath={.items[*].metadata.name}", "-A", "--field-selector=status.phase!=Running")) - if rerr != nil { - t.Logf("%s: %v", rr.Command(), rerr) - return - } - notRunning := strings.Split(rr.Stdout.String(), " ") - t.Logf("non-running pods: %s", strings.Join(notRunning, " ")) + // Get non-running pods. NOTE: This does not yet contain pods which are "running", but not "ready" + rr, rerr := Run(t, exec.Command("kubectl", "--context", profile, "get", "po", "-o=jsonpath={.items[*].metadata.name}", "-A", "--field-selector=status.phase!=Running")) + if rerr != nil { + t.Logf("%s: %v", rr.Command(), rerr) + return + } + notRunning := strings.Split(rr.Stdout.String(), " ") + t.Logf("non-running pods: %s", strings.Join(notRunning, " ")) - t.Logf("======> post-mortem[%s]: describe non-running pods <======", t.Name()) + t.Logf("======> post-mortem[%s]: describe non-running pods <======", t.Name()) - args := append([]string{"--context", profile, "describe", "pod"}, notRunning...) - rr, rerr = Run(t, exec.Command("kubectl", args...)) - if rerr != nil { - t.Logf("%s: %v", rr.Command(), rerr) - return + args := append([]string{"--context", profile, "describe", "pod"}, notRunning...) + rr, rerr = Run(t, exec.Command("kubectl", args...)) + if rerr != nil { + t.Logf("%s: %v", rr.Command(), rerr) + return + } + t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) } - t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Output()) t.Logf("<<< %s FAILED: end of post-mortem logs <<<", t.Name()) t.Logf("---------------------/post-mortem---------------------------------") @@ -356,10 +372,10 @@ func PodWait(ctx context.Context, t *testing.T, profile string, ns string, selec } // Status returns a minikube component status as a string -func Status(ctx context.Context, t *testing.T, path string, profile string, key string, node ...string) string { +func Status(ctx context.Context, t *testing.T, path string, profile string, key string, node string) string { t.Helper() // Reminder of useful keys: "Host", "Kubelet", "APIServer" - rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile)) + rr, err := Run(t, exec.CommandContext(ctx, path, "status", fmt.Sprintf("--format={{.%s}}", key), "-p", profile, "-n", node)) if err != nil { t.Logf("status error: %v (may be ok)", err) } @@ -369,7 +385,7 @@ func Status(ctx context.Context, t *testing.T, path string, profile string, key // showPodLogs logs debug info for pods func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, names []string) { t.Helper() - st := Status(context.Background(), t, Target(), profile, "APIServer") + st := Status(context.Background(), t, Target(), profile, "APIServer", profile) if st != state.Running.String() { t.Logf("%q apiserver is not running, skipping kubectl commands (state=%q)", profile, st) return diff --git a/test/integration/main.go b/test/integration/main.go index fb706aebddb6fbcb880010ff7fc3a1e66585d4b5..7a2865fd0878e5beb000e5b4563f1503ec14019b 100644 --- a/test/integration/main.go +++ b/test/integration/main.go @@ -40,6 +40,12 @@ var timeOutMultiplier = flag.Float64("timeout-multiplier", 1, "multiply the time var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary") var testdataDir = flag.String("testdata-dir", "testdata", "the directory relative to test/integration where the testdata lives") +// Node names are consistent, let's store these for easy access later +const ( + SecondNodeName = "m02" + ThirdNodeName = "m03" +) + // TestMain is the test main func TestMain(m *testing.M) { flag.Parse() diff --git a/test/integration/multinode_test.go b/test/integration/multinode_test.go index 8ddf42b5f03dc57841fd9a02e67fce8b63dc9b85..7e09c180e68ef0362bc059a805ff701dc35b768d 100644 --- a/test/integration/multinode_test.go +++ b/test/integration/multinode_test.go @@ -50,6 +50,7 @@ func TestMultiNode(t *testing.T) { for _, tc := range tests { tc := tc t.Run(tc.name, func(t *testing.T) { + defer PostMortemLogs(t, profile) tc.validator(ctx, t, profile) }) } @@ -104,11 +105,8 @@ func validateAddNodeToMultiNode(ctx context.Context, t *testing.T, profile strin } func validateStopRunningNode(ctx context.Context, t *testing.T, profile string) { - // Names are autogenerated using the node.Name() function - name := "m03" - // Run minikube node stop on that node - rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "stop", name)) + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "stop", ThirdNodeName)) if err != nil { t.Errorf("node stop returned an error. args %q: %v", rr.Command(), err) } @@ -143,11 +141,8 @@ func validateStartNodeAfterStop(ctx context.Context, t *testing.T, profile strin // TODO (#7496): remove skip once restarts work t.Skip("Restarting nodes is broken :(") - // Grab the stopped node - name := "m03" - // Start the node back up - rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "start", name)) + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "start", ThirdNodeName)) if err != nil { t.Errorf("node start returned an error. args %q: %v", rr.Command(), err) } @@ -168,10 +163,9 @@ func validateStartNodeAfterStop(ctx context.Context, t *testing.T, profile strin } func validateDeleteNodeFromMultiNode(ctx context.Context, t *testing.T, profile string) { - name := "m03" // Start the node back up - rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "delete", name)) + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "node", "delete", ThirdNodeName)) if err != nil { t.Errorf("node stop returned an error. args %q: %v", rr.Command(), err) } @@ -195,7 +189,7 @@ func validateDeleteNodeFromMultiNode(ctx context.Context, t *testing.T, profile if err != nil { t.Errorf("failed to run %q : %v", rr.Command(), err) } - if strings.Contains(rr.Stdout.String(), fmt.Sprintf("%s-%s", profile, name)) { + if strings.Contains(rr.Stdout.String(), fmt.Sprintf("%s-%s", profile, ThirdNodeName)) { t.Errorf("docker volume was not properly deleted: %s", rr.Stdout.String()) } }