From 26c747137e4f5d20a0642afc86165f8bb6fafb13 Mon Sep 17 00:00:00 2001 From: Sharif Elgamal Date: Thu, 2 Apr 2020 14:36:46 -0700 Subject: [PATCH] failback to alternate drivers if startup fails with automatic choice --- cmd/minikube/cmd/start.go | 65 ++++++++++++++++++++++++++++---------- go.mod | 1 + pkg/minikube/exit/exit.go | 34 +------------------- pkg/minikube/node/start.go | 5 +-- pkg/minikube/out/out.go | 30 ++++++++++++++++++ 5 files changed, 84 insertions(+), 51 deletions(-) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 61965e87c..a00c26570 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" "math" + "math/rand" "net" "net/url" "os" @@ -304,11 +305,41 @@ func runStart(cmd *cobra.Command, args []string) { } validateSpecifiedDriver(existing) - ds, alts := selectDriver(existing) + ds, alts, specified := selectDriver(existing) + err = startWithDriver(cmd, ds, existing) + if err != nil { + if specified { + // User specified an invalid or broken driver + exit.WithCodeT(exit.Failure, "Startup with {{.driver}} driver failed: {{.error}}", out.V{"driver": ds.Name, "error": err}) + } + + // Walk down the rest of the options + for _, alt := range alts { + out.WarningT("Startup with {{.old_driver}} driver failed, trying with {{.new_driver}}.", out.V{"old_driver": ds.Name, "new_driver": alt.Name}) + ds = alt + // Delete the existing cluster and try again with the next driver on the list + profile, err := config.LoadProfile(ClusterFlagValue()) + if err != nil { + out.ErrT(out.Meh, `"{{.name}}" profile does not exist, trying anyways.`, out.V{"name": ClusterFlagValue()}) + } + + err = deleteProfile(profile) + if err != nil { + out.WarningT("Failed to delete cluster {{.name}}, proceeding with retry anyway.", out.V{"name": ClusterFlagValue()}) + } + err = startWithDriver(cmd, ds, existing) + if err != nil { + continue + } + } + } } func startWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *config.ClusterConfig) error { + if rand.Int()%2 == 0 { + return errors.New("OH NO RANDOM FAILURE") + } driverName := ds.Name glog.Infof("selected driver: %s", driverName) validateDriver(ds, existing) @@ -328,7 +359,7 @@ func startWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *conf k8sVersion := getKubernetesVersion(existing) cc, n, err := generateCfgFromFlags(cmd, k8sVersion, driverName) if err != nil { - exit.WithError("Failed to generate config", err) + return errors.Wrap(err, "Failed to generate config") } // This is about as far as we can go without overwriting config files @@ -340,7 +371,7 @@ func startWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *conf if driver.IsVM(driverName) { url, err := download.ISO(viper.GetStringSlice(isoURL), cmd.Flags().Changed(isoURL)) if err != nil { - exit.WithError("Failed to cache ISO", err) + return errors.Wrap(err, "Failed to cache ISO") } cc.MinikubeISO = url } @@ -361,7 +392,10 @@ func startWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *conf kubeconfig, err := node.Start(cc, n, existingAddons, true) if err != nil { - kubeconfig = maybeDeleteAndRetry(cc, n, existingAddons, err) + kubeconfig, err = maybeDeleteAndRetry(cc, n, existingAddons, err) + if err != nil { + return err + } } numNodes := viper.GetInt(nodes) @@ -383,7 +417,7 @@ func startWithDriver(cmd *cobra.Command, ds registry.DriverState, existing *conf out.Ln("") // extra newline for clarity on the command line err := node.Add(&cc, n) if err != nil { - exit.WithError("adding node", err) + return errors.Wrap(err, "adding node") } } } @@ -468,9 +502,9 @@ func showKubectlInfo(kcs *kubeconfig.Settings, k8sVersion string, machineName st return nil } -func maybeDeleteAndRetry(cc config.ClusterConfig, n config.Node, existingAddons map[string]bool, originalErr error) *kubeconfig.Settings { +func maybeDeleteAndRetry(cc config.ClusterConfig, n config.Node, existingAddons map[string]bool, originalErr error) (*kubeconfig.Settings, error) { if viper.GetBool(deleteOnFailure) { - out.T(out.Warning, "Node {{.name}} failed to start, deleting and trying again.", out.V{"name": n.Name}) + out.WarningT("Node {{.name}} failed to start, deleting and trying again.", out.V{"name": n.Name}) // Start failed, delete the cluster and try again profile, err := config.LoadProfile(cc.Name) if err != nil { @@ -490,14 +524,13 @@ func maybeDeleteAndRetry(cc config.ClusterConfig, n config.Node, existingAddons } if err != nil { // Ok we failed again, let's bail - exit.WithError("Start failed after cluster deletion", err) + return nil, err } } - return kubeconfig + return kubeconfig, nil } // Don't delete the cluster unless they ask - exit.WithError("startup failed", originalErr) - return nil + return nil, errors.Wrap(originalErr, "startup failed") } func kubectlVersion(path string) (string, error) { @@ -525,7 +558,7 @@ func kubectlVersion(path string) (string, error) { return cv.ClientVersion.GitVersion, nil } -func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []registry.DriverState) { +func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []registry.DriverState, bool) { // Technically unrelated, but important to perform before detection driver.SetLibvirtURI(viper.GetString(kvmQemuURI)) @@ -534,7 +567,7 @@ func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []regis old := hostDriver(existing) ds := driver.Status(old) out.T(out.Sparkle, `Using the {{.driver}} driver based on existing profile`, out.V{"driver": ds.String()}) - return ds, nil + return ds, nil, true } // Default to looking at the new driver parameter @@ -554,7 +587,7 @@ func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []regis exit.WithCodeT(exit.Unavailable, "The driver '{{.driver}}' is not supported on {{.os}}", out.V{"driver": d, "os": runtime.GOOS}) } out.T(out.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()}) - return ds, nil + return ds, nil, true } // Fallback to old driver parameter @@ -564,7 +597,7 @@ func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []regis exit.WithCodeT(exit.Unavailable, "The driver '{{.driver}}' is not supported on {{.os}}", out.V{"driver": d, "os": runtime.GOOS}) } out.T(out.Sparkle, `Using the {{.driver}} driver based on user configuration`, out.V{"driver": ds.String()}) - return ds, nil + return ds, nil, true } pick, alts := driver.Suggest(driver.Choices(viper.GetBool("vm"))) @@ -581,7 +614,7 @@ func selectDriver(existing *config.ClusterConfig) (registry.DriverState, []regis } else { out.T(out.Sparkle, `Automatically selected the {{.driver}} driver`, out.V{"driver": pick.String()}) } - return pick, alts + return pick, alts, false } // hostDriver returns the actual driver used by a libmachine host, which can differ from our config diff --git a/go.mod b/go.mod index 29a5ac30d..2a00c7499 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,7 @@ require ( google.golang.org/genproto v0.0.0-20200117163144-32f20d992d24 // indirect google.golang.org/grpc v1.26.0 // indirect gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect + gopkg.in/yaml.v2 v2.2.8 gotest.tools/v3 v3.0.2 // indirect k8s.io/api v0.17.3 k8s.io/apimachinery v0.17.3 diff --git a/pkg/minikube/exit/exit.go b/pkg/minikube/exit/exit.go index 221001f86..77b2cb5c4 100644 --- a/pkg/minikube/exit/exit.go +++ b/pkg/minikube/exit/exit.go @@ -18,14 +18,11 @@ limitations under the License. package exit import ( - "fmt" "os" "runtime" - "github.com/golang/glog" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/problem" - "k8s.io/minikube/pkg/minikube/translate" ) // Exit codes based on sysexits(3) @@ -40,9 +37,6 @@ const ( IO = 74 // IO represents an I/O error Config = 78 // Config represents an unconfigured or misconfigured state Permissions = 77 // Permissions represents a permissions error - - // MaxLogEntries controls the number of log entries to show for each source - MaxLogEntries = 3 ) // UsageT outputs a templated usage error and exits with error code 64 @@ -63,7 +57,7 @@ func WithError(msg string, err error) { if p != nil { WithProblem(msg, err, p) } - displayError(msg, err) + out.DisplayError(msg, err) os.Exit(Software) } @@ -79,29 +73,3 @@ func WithProblem(msg string, err error, p *problem.Problem) { } os.Exit(Config) } - -// WithLogEntries outputs an error along with any important log entries, and exits. -func WithLogEntries(msg string, err error, entries map[string][]string) { - displayError(msg, err) - - for name, lines := range entries { - out.T(out.FailureType, "Problems detected in {{.entry}}:", out.V{"entry": name}) - if len(lines) > MaxLogEntries { - lines = lines[:MaxLogEntries] - } - for _, l := range lines { - out.T(out.LogEntry, l) - } - } - os.Exit(Software) -} - -func displayError(msg string, err error) { - // use Warning because Error will display a duplicate message to stderr - glog.Warningf(fmt.Sprintf("%s: %v", msg, err)) - out.ErrT(out.Empty, "") - out.FatalT("{{.msg}}: {{.err}}", out.V{"msg": translate.T(msg), "err": err}) - out.ErrT(out.Empty, "") - out.ErrT(out.Sad, "minikube is exiting due to an error. If the above message is not useful, open an issue:") - out.ErrT(out.URL, "https://github.com/kubernetes/minikube/issues/new/choose") -} diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index 37202613f..5b66e1286 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -86,7 +86,7 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo // Abstraction leakage alert: startHost requires the config to be saved, to satistfy pkg/provision/buildroot. // Hence, saveConfig must be called before startHost, and again afterwards when we know the IP. if err := config.SaveProfile(viper.GetString(config.ProfileName), &cc); err != nil { - exit.WithError("Failed to save config", err) + return nil, errors.Wrap(err, "Failed to save config") } handleDownloadOnly(&cacheGroup, &kicGroup, n.KubernetesVersion) @@ -120,7 +120,8 @@ func Start(cc config.ClusterConfig, n config.Node, existingAddons map[string]boo bs = setupKubeAdm(machineAPI, cc, n) err = bs.StartCluster(cc) if err != nil { - exit.WithLogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, cc, mRunner)) + out.LogEntries("Error starting cluster", err, logs.FindProblems(cr, bs, cc, mRunner)) + return nil, err } // write the kubeconfig to the file system after everything required (like certs) are created by the bootstrapper diff --git a/pkg/minikube/out/out.go b/pkg/minikube/out/out.go index 38c817339..253eb09b4 100644 --- a/pkg/minikube/out/out.go +++ b/pkg/minikube/out/out.go @@ -26,6 +26,7 @@ import ( "github.com/golang/glog" isatty "github.com/mattn/go-isatty" + "k8s.io/minikube/pkg/minikube/translate" ) // By design, this package uses global references to language and output objects, in preference @@ -51,6 +52,9 @@ var ( OverrideEnv = "MINIKUBE_IN_STYLE" ) +// MaxLogEntries controls the number of log entries to show for each source +const MaxLogEntries = 3 + // fdWriter is the subset of file.File that implements io.Writer and Fd() type fdWriter interface { io.Writer @@ -175,3 +179,29 @@ func wantsColor(fd uintptr) bool { glog.Infof("isatty.IsTerminal(%d) = %v\n", fd, isT) return isT } + +// LogEntries outputs an error along with any important log entries. +func LogEntries(msg string, err error, entries map[string][]string) { + DisplayError(msg, err) + + for name, lines := range entries { + T(FailureType, "Problems detected in {{.entry}}:", V{"entry": name}) + if len(lines) > MaxLogEntries { + lines = lines[:MaxLogEntries] + } + for _, l := range lines { + T(LogEntry, l) + } + } +} + +// DisplayError +func DisplayError(msg string, err error) { + // use Warning because Error will display a duplicate message to stderr + glog.Warningf(fmt.Sprintf("%s: %v", msg, err)) + ErrT(Empty, "") + FatalT("{{.msg}}: {{.err}}", V{"msg": translate.T(msg), "err": err}) + ErrT(Empty, "") + ErrT(Sad, "minikube is exiting due to an error. If the above message is not useful, open an issue:") + ErrT(URL, "https://github.com/kubernetes/minikube/issues/new/choose") +} -- GitLab