Merge pull request #3401 from tstromberg/cherry-2f81d

Add apiserver check to "status", and block "start" until it's healthy.

cmd/minikube/cmd/start.go

@@ -30,6 +30,7 @@ import (
 
 	"github.com/blang/semver"
 	"github.com/docker/machine/libmachine/host"
+	"github.com/docker/machine/libmachine/state"
 	"github.com/golang/glog"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -349,6 +350,36 @@ func runStart(cmd *cobra.Command, args []string) {
 		}
 	}
 
+	// Block until the cluster is healthy.
+	fmt.Print("Verifying kubelet health ...")
+	kStat := func() (err error) {
+		st, err := k8sBootstrapper.GetKubeletStatus()
+		if err != nil || st != state.Running.String() {
+			fmt.Printf(".")
+			return &pkgutil.RetriableError{Err: fmt.Errorf("kubelet unhealthy: %v: %s", err, st)}
+		}
+		return nil
+	}
+	err = pkgutil.RetryAfter(20, kStat, 3*time.Second)
+	if err != nil {
+		fmt.Printf("error: %v", err)
+		cmdutil.MaybeReportErrorAndExit(err)
+	}
+	fmt.Print("\nVerifying apiserver health ...")
+	aStat := func() (err error) {
+		st, err := k8sBootstrapper.GetApiServerStatus(net.ParseIP(ip))
+		if err != nil || st != state.Running.String() {
+			fmt.Print(".")
+			return &pkgutil.RetriableError{Err: fmt.Errorf("apiserver unhealthy: %v: %s", err, st)}
+		}
+		return nil
+	}
+	err = pkgutil.RetryAfter(20, aStat, 3*time.Second)
+	if err != nil {
+		fmt.Printf("error: %v", err)
+		cmdutil.MaybeReportErrorAndExit(err)
+	}
+
 	// start 9p server mount
 	if viper.GetBool(createMount) {
 		fmt.Printf("Setting up hostmount on %s...\n", viper.GetString(mountString))
@@ -416,6 +447,8 @@ This can also be done automatically by setting the env var CHANGE_MINIKUBE_NONE_
 	if err != nil {
 		fmt.Println("Unable to load cached images from config file.")
 	}
+	fmt.Println("\n\nEverything looks great. Please enjoy minikube!")
+	return
 }
 
 func init() {

cmd/minikube/cmd/status.go

@@ -37,9 +37,10 @@ import (
 var statusFormat string
 
 type Status struct {
-	MinikubeStatus   string
-	ClusterStatus    string
-	KubeconfigStatus string
+	Host       string
+	Kubelet    string
+	ApiServer  string
+	Kubeconfig string
 }
 
 const internalErrorCode = -1
@@ -66,25 +67,27 @@ var statusCmd = &cobra.Command{
 		}
 		defer api.Close()
 
-		ms, err := cluster.GetHostStatus(api)
+		hostSt, err := cluster.GetHostStatus(api)
 		if err != nil {
 			glog.Errorln("Error getting machine status:", err)
 			cmdUtil.MaybeReportErrorAndExitWithCode(err, internalErrorCode)
 		}
 
-		cs := state.None.String()
-		ks := state.None.String()
-		if ms == state.Running.String() {
+		kubeletSt := state.None.String()
+		kubeconfigSt := state.None.String()
+		apiserverSt := state.None.String()
+
+		if hostSt == state.Running.String() {
 			clusterBootstrapper, err := GetClusterBootstrapper(api, viper.GetString(cmdcfg.Bootstrapper))
 			if err != nil {
 				glog.Errorf("Error getting cluster bootstrapper: %v", err)
 				cmdUtil.MaybeReportErrorAndExitWithCode(err, internalErrorCode)
 			}
-			cs, err = clusterBootstrapper.GetClusterStatus()
+			kubeletSt, err = clusterBootstrapper.GetKubeletStatus()
 			if err != nil {
-				glog.Errorln("Error cluster status:", err)
+				glog.Errorln("Error kubelet status:", err)
 				cmdUtil.MaybeReportErrorAndExitWithCode(err, internalErrorCode)
-			} else if cs != state.Running.String() {
+			} else if kubeletSt != state.Running.String() {
 				returnCode |= clusterNotRunningStatusFlag
 			}
 
@@ -93,15 +96,24 @@ var statusCmd = &cobra.Command{
 				glog.Errorln("Error host driver ip status:", err)
 				cmdUtil.MaybeReportErrorAndExitWithCode(err, internalErrorCode)
 			}
-			kstatus, err := kubeconfig.GetKubeConfigStatus(ip, cmdUtil.GetKubeConfigPath(), config.GetMachineName())
+
+			apiserverSt, err = clusterBootstrapper.GetApiServerStatus(ip)
+			if err != nil {
+				glog.Errorln("Error apiserver status:", err)
+				cmdUtil.MaybeReportErrorAndExitWithCode(err, internalErrorCode)
+			} else if apiserverSt != state.Running.String() {
+				returnCode |= clusterNotRunningStatusFlag
+			}
+
+			ks, err := kubeconfig.GetKubeConfigStatus(ip, cmdUtil.GetKubeConfigPath(), config.GetMachineName())
 			if err != nil {
 				glog.Errorln("Error kubeconfig status:", err)
 				cmdUtil.MaybeReportErrorAndExitWithCode(err, internalErrorCode)
 			}
-			if kstatus {
-				ks = "Correctly Configured: pointing to minikube-vm at " + ip.String()
+			if ks {
+				kubeconfigSt = "Correctly Configured: pointing to minikube-vm at " + ip.String()
 			} else {
-				ks = "Misconfigured: pointing to stale minikube-vm." +
+				kubeconfigSt = "Misconfigured: pointing to stale minikube-vm." +
 					"\nTo fix the kubectl context, run minikube update-context"
 				returnCode |= k8sNotRunningStatusFlag
 			}
@@ -109,8 +121,12 @@ var statusCmd = &cobra.Command{
 			returnCode |= minikubeNotRunningStatusFlag
 		}
 
-		status := Status{ms, cs, ks}
-
+		status := Status{
+			Host:       hostSt,
+			Kubelet:    kubeletSt,
+			ApiServer:  apiserverSt,
+			Kubeconfig: kubeconfigSt,
+		}
 		tmpl, err := template.New("status").Parse(statusFormat)
 		if err != nil {
 			glog.Errorln("Error creating status template:", err)

pkg/minikube/bootstrapper/bootstrapper.go

@@ -18,6 +18,7 @@ package bootstrapper
 
 import (
 	"io"
+	"net"
 
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -30,7 +31,8 @@ type Bootstrapper interface {
 	RestartCluster(config.KubernetesConfig) error
 	GetClusterLogsTo(follow bool, out io.Writer) error
 	SetupCerts(cfg config.KubernetesConfig) error
-	GetClusterStatus() (string, error)
+	GetKubeletStatus() (string, error)
+	GetApiServerStatus(net.IP) (string, error)
 }
 
 const (

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -19,8 +19,11 @@ package kubeadm
 import (
 	"bytes"
 	"crypto"
+	"crypto/tls"
 	"fmt"
 	"io"
+	"net"
+	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -67,8 +70,7 @@ func NewKubeadmBootstrapper(api libmachine.API) (*KubeadmBootstrapper, error) {
 	}, nil
 }
 
-//TODO(r2d4): This should most likely check the health of the apiserver
-func (k *KubeadmBootstrapper) GetClusterStatus() (string, error) {
+func (k *KubeadmBootstrapper) GetKubeletStatus() (string, error) {
 	statusCmd := `sudo systemctl is-active kubelet`
 	status, err := k.c.CombinedOutput(statusCmd)
 	if err != nil {
@@ -80,10 +82,31 @@ func (k *KubeadmBootstrapper) GetClusterStatus() (string, error) {
 		return state.Running.String(), nil
 	case "inactive":
 		return state.Stopped.String(), nil
+	case "activating":
+		return state.Starting.String(), nil
 	}
 	return state.Error.String(), nil
 }
 
+func (k *KubeadmBootstrapper) GetApiServerStatus(ip net.IP) (string, error) {
+	url := fmt.Sprintf("https://%s:%d/healthz", ip, util.APIServerPort)
+	// To avoid: x509: certificate signed by unknown authority
+	tr := &http.Transport{
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(url)
+	glog.Infof("%s response: %v %+v", url, err, resp)
+	// Connection refused, usually.
+	if err != nil {
+		return state.Stopped.String(), nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		return state.Error.String(), nil
+	}
+	return state.Running.String(), nil
+}
+
 // TODO(r2d4): Should this aggregate all the logs from the control plane?
 // Maybe subcommands for each component? minikube logs apiserver?
 func (k *KubeadmBootstrapper) GetClusterLogsTo(follow bool, out io.Writer) error {

pkg/minikube/constants/constants.go

@@ -110,8 +110,10 @@ const (
 	DefaultDiskSize     = "20g"
 	MinimumDiskSizeMB   = 2000
 	DefaultVMDriver     = "virtualbox"
-	DefaultStatusFormat = "minikube: {{.MinikubeStatus}}\n" +
-		"cluster: {{.ClusterStatus}}\n" + "kubectl: {{.KubeconfigStatus}}\n"
+	DefaultStatusFormat = `host: {{.Host}}
+kubelet: {{.Kubelet}}
+apiserver: {{.ApiServer}}
+kubectl: {{.Kubeconfig}}`
 	DefaultAddonListFormat     = "- {{.AddonName}}: {{.AddonStatus}}\n"
 	DefaultConfigViewFormat    = "- {{.ConfigKey}}: {{.ConfigValue}}\n"
 	DefaultCacheListFormat     = "{{.CacheImage}}\n"

test/integration/util/util.go

@@ -151,7 +151,7 @@ func (m *MinikubeRunner) ParseEnvCmdOutput(out string) map[string]string {
 }
 
 func (m *MinikubeRunner) GetStatus() string {
-	return m.RunCommand(fmt.Sprintf("status --format={{.MinikubeStatus}} %s", m.Args), false)
+	return m.RunCommand(fmt.Sprintf("status --format={{.Host}} %s", m.Args), false)
 }
 
 func (m *MinikubeRunner) GetLogs() string {