diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kubelet.go b/pkg/minikube/bootstrapper/bsutil/kverify/kubelet.go new file mode 100644 index 0000000000000000000000000000000000000000..e3c5b2f8ae0e6deb4e7c86ce79ac5505895ffcc5 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/kverify/kubelet.go @@ -0,0 +1,60 @@ +/* +Copyright 2020 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package kverify verifies a running Kubernetes cluster is healthy +package kverify + +import ( + "fmt" + "time" + + "github.com/docker/machine/libmachine/state" + "k8s.io/klog/v2" + "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/sysinit" + "k8s.io/minikube/pkg/util/retry" +) + +// KubeletStatus checks the kubelet status +func KubeletStatus(cr command.Runner) state.State { + klog.Infof("Checking kubelet status ...") + active := sysinit.New(cr).Active("kubelet") + if active { + return state.Running + } + return state.Stopped +} + +// WaitForKubelet will wait for Kubelet service to be running ... +func WaitForKubelet(cr command.Runner, timeout time.Duration) error { + pStart := time.Now() + klog.Infof("waiting for kubelet to be running ....") + kr := func() error { + if st := KubeletStatus(cr); st != state.Running { + return fmt.Errorf("status %s", st) + } + return nil + } + + if err := retry.Local(kr, timeout); err != nil { + return fmt.Errorf("not running: %s", err) + } + + klog.Infof("duration metric: took %s WaitForKubelet to finish.", time.Since(pStart)) + + return nil + +} diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go index 1cac68a2cf2d933f57b2abda5b7ee02db96ad377..76a976e36ad3a20c58bd35646d5162cbdffa2c72 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/kverify.go @@ -35,6 +35,8 @@ const ( AppsRunningKey = "apps_running" // NodeReadyKey is the name used in the flags for waiting for the node status to be ready NodeReadyKey = "node_ready" + // NodeReadyKey is the name used in the flags for waiting for the node status to be ready + KubeletKey = "kubelet" ) // vars related to the --wait flag @@ -42,13 +44,13 @@ var ( // DefaultComponents is map of the the default components to wait for DefaultComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true} // NoWaitComponents is map of componets to wait for if specified 'none' or 'false' - NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false} + NoComponents = map[string]bool{APIServerWaitKey: false, SystemPodsWaitKey: false, DefaultSAWaitKey: false, AppsRunningKey: false, NodeReadyKey: false, KubeletKey: false} // AllComponents is map for waiting for all components. - AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true} + AllComponents = map[string]bool{APIServerWaitKey: true, SystemPodsWaitKey: true, DefaultSAWaitKey: true, AppsRunningKey: true, KubeletKey: true} // DefaultWaitList is list of all default components to wait for. only names to be used for start flags. DefaultWaitList = []string{APIServerWaitKey, SystemPodsWaitKey} // AllComponentsList list of all valid components keys to wait for. only names to be used used for start flags. - AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunningKey, NodeReadyKey} + AllComponentsList = []string{APIServerWaitKey, SystemPodsWaitKey, DefaultSAWaitKey, AppsRunningKey, NodeReadyKey, KubeletKey} // AppsRunningList running list are valid k8s-app components to wait for them to be running AppsRunningList = []string{ "kube-dns", // coredns diff --git a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go index a331f35c18a2720a61262cfae4a5015c8da3db6b..094fd39536f18fe4d7b51fb568f83623127e2e8e 100644 --- a/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go +++ b/pkg/minikube/bootstrapper/bsutil/kverify/system_pods.go @@ -22,7 +22,14 @@ import ( "strings" "time" +<<<<<<< HEAD "github.com/docker/machine/libmachine/state" +||||||| parent of a1b34164f (ensure kubelet is running and wait for it) + "github.com/docker/machine/libmachine/state" + "github.com/golang/glog" +======= + "github.com/golang/glog" +>>>>>>> a1b34164f (ensure kubelet is running and wait for it) "github.com/pkg/errors" core "k8s.io/api/core/v1" meta "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -34,7 +41,6 @@ import ( "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/cruntime" "k8s.io/minikube/pkg/minikube/logs" - "k8s.io/minikube/pkg/minikube/sysinit" "k8s.io/minikube/pkg/util/retry" ) @@ -156,12 +162,4 @@ func announceProblems(r cruntime.Manager, bs bootstrapper.Bootstrapper, cfg conf } } -// KubeletStatus checks the kubelet status -func KubeletStatus(cr command.Runner) state.State { - klog.Infof("Checking kubelet status ...") - active := sysinit.New(cr).Active("kubelet") - if active { - return state.Running - } - return state.Stopped -} + diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index 72756f1d5419ff90eeaeccb67914a0ce9467a5c7..594666d00d5a56000b26393c942a2430671daec8 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -392,10 +392,13 @@ func (k *Bootstrapper) client(ip string, port int) (*kubernetes.Clientset, error // WaitForNode blocks until the node appears to be healthy func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, timeout time.Duration) error { start := time.Now() - register.Reg.SetStep(register.VerifyingKubernetes) out.T(style.HealthCheck, "Verifying Kubernetes components...") - + // regardless if waiting or not, we make sure it is not stopped + // to solve corner cases when a container is hibernated and once coming back kubelet not running. + if err := k.ensureKubeletStarted(); err != nil { + klog.Warningf("Couldn't ensure kubelet is started this might cause issues: %v", err) + } // TODO: #7706: for better performance we could use k.client inside minikube to avoid asking for external IP:PORT cp, err := config.PrimaryControlPlane(&cfg) if err != nil { @@ -455,6 +458,12 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time } } } + if cfg.VerifyComponents[kverify.KubeletKey] { + if err := kverify.WaitForKubelet(k.c, timeout); err != nil { + return errors.Wrap(err, "waiting for kubelet") + } + + } if cfg.VerifyComponents[kverify.NodeReadyKey] { if err := kverify.WaitForNodeReady(client, timeout); err != nil { @@ -471,6 +480,16 @@ func (k *Bootstrapper) WaitForNode(cfg config.ClusterConfig, n config.Node, time return nil } +// ensureKubeletStarted will start kubelet if whatever reason it is stopped. +func (k *Bootstrapper) ensureKubeletStarted() error { + fmt.Println("inside ensureKubeletStarted") + if kverify.KubeletStatus(k.c) != state.Running { + fmt.Println("kubelet was not running for some reason , starting it...") + return sysinit.New(k.c).Start("kubelet") + } + return nil +} + // needsReconfigure returns whether or not the cluster needs to be reconfigured func (k *Bootstrapper) needsReconfigure(conf string, hostname string, port int, client *kubernetes.Clientset, version string) bool { if rr, err := k.c.RunCmd(exec.Command("sudo", "diff", "-u", conf, conf+".new")); err != nil { diff --git a/pkg/minikube/node/start.go b/pkg/minikube/node/start.go index b2753f9cf9aaa115977e36511a4506eff10dad20..6df0b0588fcc72cc7f721d6b084962c5c0fb9844 100644 --- a/pkg/minikube/node/start.go +++ b/pkg/minikube/node/start.go @@ -196,7 +196,7 @@ func Start(starter Starter, apiServer bool) (*kubeconfig.Settings, error) { } } - klog.Infof("Will wait %s for node ...", waitTimeout) + klog.Infof("Will wait %s for node up to ", viper.GetDuration(waitTimeout)) if err := bs.WaitForNode(*starter.Cfg, *starter.Node, viper.GetDuration(waitTimeout)); err != nil { return nil, errors.Wrapf(err, "wait %s for node", viper.GetDuration(waitTimeout)) } diff --git a/site/content/en/docs/commands/start.md b/site/content/en/docs/commands/start.md index ec69092d85fd78d4123109c9c6059f3a140e261c..43c6c5108135cdef40e8f818b2f189fb588f0c2d 100644 --- a/site/content/en/docs/commands/start.md +++ b/site/content/en/docs/commands/start.md @@ -90,7 +90,7 @@ minikube start [flags] --uuid string Provide VM UUID to restore MAC address (hyperkit driver only) --vm Filter to use only VM Drivers --vm-driver driver DEPRECATED, use driver instead. - --wait strings comma separated list of Kubernetes components to verify and wait for after starting a cluster. defaults to "apiserver,system_pods", available options: "apiserver,system_pods,default_sa,apps_running,node_ready" . other acceptable values are 'all' or 'none', 'true' and 'false' (default [apiserver,system_pods]) + --wait strings comma separated list of Kubernetes components to verify and wait for after starting a cluster. defaults to "apiserver,system_pods", available options: "apiserver,system_pods,default_sa,apps_running,node_ready,kubelet" . other acceptable values are 'all' or 'none', 'true' and 'false' (default [apiserver,system_pods]) --wait-timeout duration max time to wait per Kubernetes or host to be healthy. (default 6m0s) ```