Merge pull request #6151 from medyagh/kic_exp

Experimental verison of kic

Makefile

@@ -483,6 +483,13 @@ else
 	docker build -t $(REGISTRY)/storage-provisioner-$(GOARCH):$(STORAGE_PROVISIONER_TAG) -f deploy/storage-provisioner/Dockerfile-$(GOARCH) .
 endif
 
+.PHONY: kic-base-image
+kic-base-image: ## builds the base image used for kic.
+	docker rmi -f $(REGISTRY)/kicbase:v0.0.1-snapshot || true
+	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:v0.0.1-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT)  .
+
+
+
 .PHONY: push-storage-provisioner-image
 push-storage-provisioner-image: storage-provisioner-image ## Push storage-provisioner docker image using gcloud
 ifeq ($(GOARCH),amd64)
@@ -591,6 +598,7 @@ out/mkcmp:
 out/performance-monitor:
 	GOOS=$(GOOS) GOARCH=$(GOARCH) go build -o $@ cmd/performance/monitor/monitor.go
 
+
 .PHONY: help
 help:
 	@printf "\033[1mAvailable targets for minikube ${VERSION}\033[21m\n"

cmd/minikube/cmd/start.go

@@ -232,7 +232,7 @@ func initNetworkingFlags() {
 	startCmd.Flags().StringSliceVar(&registryMirror, "registry-mirror", nil, "Registry mirrors to pass to the Docker daemon")
 	startCmd.Flags().String(imageRepository, "", "Alternative image repository to pull docker images from. This can be used when you have limited access to gcr.io. Set it to \"auto\" to let minikube decide one for you. For Chinese mainland users, you may use local gcr.io mirrors such as registry.cn-hangzhou.aliyuncs.com/google_containers")
 	startCmd.Flags().String(imageMirrorCountry, "", "Country code of the image mirror to be used. Leave empty to use the global one. For Chinese mainland users, set it to cn.")
-	startCmd.Flags().String(serviceCIDR, pkgutil.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
+	startCmd.Flags().String(serviceCIDR, constants.DefaultServiceCIDR, "The CIDR to be used for service cluster IPs.")
 	startCmd.Flags().StringArrayVar(&dockerEnv, "docker-env", nil, "Environment variables to pass to the Docker daemon. (format: key=value)")
 	startCmd.Flags().StringArrayVar(&dockerOpt, "docker-opt", nil, "Specify arbitrary flags to pass to the Docker daemon. (format: key=value)")
 }
@@ -430,16 +430,22 @@ func displayEnviron(env []string) {
 }
 
 func setupKubeconfig(h *host.Host, c *cfg.MachineConfig, clusterName string) (*kubeconfig.Settings, error) {
-	addr, err := h.Driver.GetURL()
-	if err != nil {
-		exit.WithError("Failed to get driver URL", err)
+	addr := ""
+	var err error
+	if driver.IsKIC(h.DriverName) {
+		addr = fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(c.KubernetesConfig.NodePort)))
+	} else {
+		addr, err = h.Driver.GetURL()
+		if err != nil {
+			exit.WithError("Failed to get driver URL", err)
+		}
+		addr = strings.Replace(addr, "tcp://", "https://", -1)
+		addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1)
 	}
-	addr = strings.Replace(addr, "tcp://", "https://", -1)
-	addr = strings.Replace(addr, ":2376", ":"+strconv.Itoa(c.KubernetesConfig.NodePort), -1)
+
 	if c.KubernetesConfig.APIServerName != constants.APIServerName {
 		addr = strings.Replace(addr, c.KubernetesConfig.NodeIP, c.KubernetesConfig.APIServerName, -1)
 	}
-
 	kcs := &kubeconfig.Settings{
 		ClusterName:          clusterName,
 		ClusterServerAddress: addr,
@@ -980,29 +986,35 @@ func setDockerProxy() {
 }
 
 // autoSetDriverOptions sets the options needed for specific vm-driver automatically.
-func autoSetDriverOptions(cmd *cobra.Command, drvName string) error {
+func autoSetDriverOptions(cmd *cobra.Command, drvName string) (err error) {
+	err = nil
 	hints := driver.FlagDefaults(drvName)
-	if !cmd.Flags().Changed("extra-config") && hints.ExtraOptions != "" {
-		return extraOptions.Set(hints.ExtraOptions)
+	if !cmd.Flags().Changed("extra-config") && len(hints.ExtraOptions) > 0 {
+		for _, eo := range hints.ExtraOptions {
+			glog.Infof("auto setting extra-config to %q.", eo)
+			err = extraOptions.Set(eo)
+			if err != nil {
+				err = errors.Wrapf(err, "setting extra option %s", eo)
+			}
+		}
 	}
 
 	if !cmd.Flags().Changed(cacheImages) {
 		viper.Set(cacheImages, hints.CacheImages)
 	}
 
-	// currently only used for kic
 	if !cmd.Flags().Changed(containerRuntime) && hints.ContainerRuntime != "" {
 		viper.Set(containerRuntime, hints.ContainerRuntime)
-		glog.Infof("auto set container runtime to %s for kic driver.", hints.ContainerRuntime)
-
+		glog.Infof("auto set %s to %q.", containerRuntime, hints.ContainerRuntime)
 	}
-	if !cmd.Flags().Changed("bootstrapper") && hints.Bootstrapper != "" {
+
+	if !cmd.Flags().Changed(cmdcfg.Bootstrapper) && hints.Bootstrapper != "" {
 		viper.Set(cmdcfg.Bootstrapper, hints.Bootstrapper)
-		glog.Infof("auto set bootstrapper to %s for kic driver.", hints.Bootstrapper)
+		glog.Infof("auto set %s to %q.", cmdcfg.Bootstrapper, hints.Bootstrapper)
 
 	}
 
-	return nil
+	return err
 }
 
 // prepareNone prepares the user and host for the joy of the "none" driver

hack/images/kicbase.Dockerfile

+ARG COMMIT_SHA
+FROM kindest/node:v1.16.2
+USER root
+RUN apt-get update && apt-get install -y \
+  sudo \
+  dnsutils \
+  && apt-get clean -y 
+RUN rm -rf \
+    /var/cache/debconf/* \
+    /var/lib/apt/lists/* \
+    /var/log/* \
+    /tmp/* \
+    /var/tmp/* \
+    /usr/share/doc/* \
+    /usr/share/man/* \
+    /usr/share/local/* \
+    /kind/bin/kubeadm /kind/bin/kubelet /kind/systemd /kind/images /kind/manifests
+RUN echo "kic! Build: ${COMMIT_SHA} Time :$(date)" > "/kic.txt"

pkg/drivers/kic/kic.go

@@ -29,9 +29,19 @@ import (
 	"k8s.io/minikube/pkg/drivers/kic/node"
 	"k8s.io/minikube/pkg/drivers/kic/oci"
 	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/constants"
 )
 
-// https://minikube.sigs.k8s.io/docs/reference/drivers/kic/
+// DefaultPodCIDR is The CIDR to be used for pods inside the node.
+const DefaultPodCIDR = "10.244.0.0/16"
+
+// DefaultBindIPV4 is The default IP the container will bind to.
+const DefaultBindIPV4 = "127.0.0.1"
+
+// BaseImage is the base image is used to spin up kic containers
+const BaseImage = "gcr.io/k8s-minikube/kicbase:v0.0.1@sha256:c4ad2938877d2ae0d5b7248a5e7182ff58c0603165c3bedfe9d503e2d380a0db"
+
+// Driver represents a kic driver https://minikube.sigs.k8s.io/docs/reference/drivers/kic/
 type Driver struct {
 	*drivers.BaseDriver
 	*pkgdrivers.CommonDriver
@@ -43,16 +53,16 @@ type Driver struct {
 
 // Config is configuration for the kic driver used by registry
 type Config struct {
-	MachineName   string            // maps to the container name being created
-	CPU           int               // Number of CPU cores assigned to the container
-	Memory        int               // max memory in MB
-	StorePath     string            // lib machine store path
-	OCIBinary     string            // oci tool to use (docker, podman,...)
-	ImageDigest   string            // image name with sha to use for the node
-	APIServerPort int32             // port to connect to forward from container to user's machine
-	Mounts        []oci.Mount       // mounts
-	PortMappings  []oci.PortMapping // container port mappings
-	Envs          map[string]string // key,value of environment variables passed to the node
+	MachineName  string            // maps to the container name being created
+	CPU          int               // Number of CPU cores assigned to the container
+	Memory       int               // max memory in MB
+	StorePath    string            // libmachine store path
+	OCIBinary    string            // oci tool to use (docker, podman,...)
+	ImageDigest  string            // image name with sha to use for the node
+	HostBindPort int               // port to connect to forward from container to user's machine
+	Mounts       []oci.Mount       // mounts
+	PortMappings []oci.PortMapping // container port mappings
+	Envs         map[string]string // key,value of environment variables passed to the node
 }
 
 // NewDriver returns a fully configured Kic driver
@@ -64,6 +74,7 @@ func NewDriver(c Config) *Driver {
 		},
 		exec:       command.NewKICRunner(c.MachineName, c.OCIBinary),
 		NodeConfig: c,
+		OCIBinary:  c.OCIBinary,
 	}
 	return d
 }
@@ -77,15 +88,15 @@ func (d *Driver) Create() error {
 		CPUs:         strconv.Itoa(d.NodeConfig.CPU),
 		Memory:       strconv.Itoa(d.NodeConfig.Memory) + "mb",
 		Envs:         d.NodeConfig.Envs,
-		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.APIServerPort)},
+		ExtraArgs:    []string{"--expose", fmt.Sprintf("%d", d.NodeConfig.HostBindPort)},
 		OCIBinary:    d.NodeConfig.OCIBinary,
 	}
 
 	// control plane specific options
 	params.PortMappings = append(params.PortMappings, oci.PortMapping{
 		ListenAddress: "127.0.0.1",
-		HostPort:      d.NodeConfig.APIServerPort,
-		ContainerPort: 6443,
+		HostPort:      int32(d.NodeConfig.HostBindPort),
+		ContainerPort: constants.APIServerPort,
 	})
 
 	_, err := node.CreateNode(params)
@@ -247,32 +258,3 @@ func (d *Driver) nodeID(nameOrID string) (string, error) {
 	}
 	return string(id), err
 }
-
-func ImageForVersion(ver string) (string, error) {
-	switch ver {
-	case "v1.11.10":
-		return "medyagh/kic:v1.11.10@sha256:23bb7f5e8dd2232ec829132172e87f7b9d8de65269630989e7dac1e0fe993b74", nil
-	case "v1.12.8":
-		return "medyagh/kic:v1.12.8@sha256:c74bc5f3efe3539f6e1ad7f11bf7c09f3091c0547cb28071f4e43067053e5898", nil
-	case "v1.12.9":
-		return "medyagh/kic:v1.12.9@sha256:ff82f58e18dcb22174e8eb09dae14f7edd82d91a83c7ef19e33298d0eba6a0e3", nil
-	case "v1.12.10":
-		return "medyagh/kic:v1.12.10@sha256:2d174bae7c20698e59791e7cca9b6db234053d1a92a009d5bb124e482540c70b", nil
-	case "v1.13.6":
-		return "medyagh/kic:v1.13.6@sha256:cf63e50f824fe17b90374d38d64c5964eb9fe6b3692669e1201fcf4b29af4964", nil
-	case "v1.13.7":
-		return "medyagh/kic:v1.13.7@sha256:1a6a5e1c7534cf3012655e99df680496df9bcf0791a304adb00617d5061233fa", nil
-	case "v1.14.3":
-		return "medyagh/kic:v1.14.3@sha256:cebec21f6af23d5dfa3465b88ddf4a1acb94c2c20a0a6ff8cc1c027b0a4e2cec", nil
-	case "v1.15.0":
-		return "medyagh/kic:v1.15.0@sha256:40d433d00a2837c8be829bd3cb0576988e377472062490bce0b18281c7f85303", nil
-	case "v1.15.3":
-		return "medyagh/kic:v1.15.3@sha256:f05ce52776a86c6ead806942d424de7076af3f115b0999332981a446329e6cf1", nil
-	case "v1.16.1":
-		return "medyagh/kic:v1.16.1@sha256:e74530d22e6a04442a97a09bdbba885ad693fcc813a0d1244da32666410d1ad1", nil
-	case "v1.16.2":
-		return "medyagh/kic:v1.16.2@sha256:3374a30971bf5b0011441a227fa56ef990b76125b36ca0ab8316a3c7e4f137a3", nil
-	default:
-		return "medyagh/kic:v1.16.2@sha256:3374a30971bf5b0011441a227fa56ef990b76125b36ca0ab8316a3c7e4f137a3", nil
-	}
-}

pkg/drivers/kic/node/node.go

@@ -39,8 +39,8 @@ const (
 // Node represents a handle to a kic node
 // This struct must be created by one of: CreateControlPlane
 type Node struct {
-	// must be one of docker container ID or name
-	name      string
+	id        string         // container id
+	name      string         // container name
 	r         command.Runner // Runner
 	ociBinary string
 }
@@ -123,13 +123,15 @@ func CreateNode(p CreateConfig) (*Node, error) {
 
 // Find finds a node
 func Find(ociBinary string, name string, cmder command.Runner) (*Node, error) {
-	_, err := oci.Inspect(ociBinary, name, "{{.Id}}")
+	n, err := oci.Inspect(ociBinary, name, "{{.Id}}")
 	if err != nil {
 		return nil, fmt.Errorf("can't find node %v", err)
 	}
 	return &Node{
-		name: name,
-		r:    cmder,
+		ociBinary: ociBinary,
+		id:        n[0],
+		name:      name,
+		r:         cmder,
 	}, nil
 }
 

pkg/drivers/kic/oci/oci.go

@@ -108,7 +108,6 @@ func Pause(ociBinary string, ociID string) error {
 
 // Inspect return low-level information on containers
 func Inspect(ociBinary string, containerNameOrID, format string) ([]string, error) {
-
 	cmd := exec.Command(ociBinary, "inspect",
 		"-f", format,
 		containerNameOrID) // ... against the "node" container
@@ -352,7 +351,7 @@ func CreateContainer(ociBinary string, image string, opts ...CreateOpt) ([]strin
 	}
 
 	if err != nil {
-		return output, errors.Wrapf(err, "CreateContainer %v ", args)
+		return output, errors.Wrapf(err, "args: %v  output: %s ", args, output)
 	}
 	return output, nil
 }

pkg/drivers/kic/oci/types.go

@@ -16,6 +16,11 @@ limitations under the License.
 
 package oci
 
+const (
+	Docker = "docker"
+	Podman = "podman"
+)
+
 /*
 These types are from
 https://github.com/kubernetes/kubernetes/blob/063e7ff358fdc8b0916e6f39beedc0d025734cb1/pkg/kubelet/apis/cri/runtime/v1alpha2/api.pb.go#L183

pkg/minikube/bootstrapper/bootstrapper.go

@@ -70,7 +70,7 @@ func GetCachedImageList(imageRepository string, version string, bootstrapper str
 	case Kubeadm:
 		return images.Kubeadm(imageRepository, version)
 	case KIC:
-		return []string{"alpine"}, nil // for testing purpose just caching alpine for kicbs
+		return images.KIC(imageRepository, version)
 	default:
 		return []string{}, nil
 	}

pkg/minikube/bootstrapper/bsutil/binaries.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil package will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (

pkg/minikube/bootstrapper/bsutil/extraconfig.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (

pkg/minikube/bootstrapper/bsutil/featuregates.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (

pkg/minikube/bootstrapper/bsutil/featuregates_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (

pkg/minikube/bootstrapper/bsutil/files.go

@@ -14,12 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (
 	"path"
 
+	"github.com/pkg/errors"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/vmpath"
@@ -42,7 +43,6 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt
 		assets.NewMemoryAssetTarget(kubeadm, KubeadmYamlPath, "0640"),
 		assets.NewMemoryAssetTarget(kubelet, KubeletSystemdConfFile, "0644"),
 		assets.NewMemoryAssetTarget(kubeletSvc, KubeletServiceFile, "0644"),
-		assets.NewMemoryAssetTarget(defaultCNIConfig, DefaultCNIConfigPath, "0644"),
 	}
 	// Copy the default CNI config (k8s.conf), so that kubelet can successfully
 	// start a Pod in the case a user hasn't manually installed any CNI plugin
@@ -52,3 +52,33 @@ func ConfigFileAssets(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byt
 	}
 	return fs
 }
+
+// AddAddons adds addons to list of files
+func AddAddons(files *[]assets.CopyableFile, data interface{}) error {
+	// add addons to file list
+	// custom addons
+	if err := assets.AddMinikubeDirAssets(files); err != nil {
+		return errors.Wrap(err, "adding minikube dir assets")
+	}
+	// bundled addons
+	for _, addonBundle := range assets.Addons {
+		if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled {
+			for _, addon := range addonBundle.Assets {
+				if addon.IsTemplate() {
+					addonFile, err := addon.Evaluate(data)
+					if err != nil {
+						return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
+					}
+
+					*files = append(*files, addonFile)
+				} else {
+					*files = append(*files, addon)
+				}
+			}
+		} else if err != nil {
+			return nil
+		}
+	}
+
+	return nil
+}

pkg/minikube/bootstrapper/bsutil/template/kubelet.go → pkg/minikube/bootstrapper/bsutil/ktmpl/kubelet.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package template
+package ktmpl
 
 import "text/template"
 

pkg/minikube/bootstrapper/bsutil/ktmpl/template.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import (
+	"fmt"
+	"sort"
+)
+
+// printMapInOrder sorts the keys and prints the map in order, combining key
+// value pairs with the separator character
+//
+// Note: this is not necessary, but makes testing easy
+func printMapInOrder(m map[string]string, sep string) []string {
+	if m == nil {
+		return nil
+	}
+	keys := []string{}
+	for k := range m {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for i, k := range keys {
+		keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k])
+	}
+	return keys
+}

pkg/minikube/bootstrapper/bsutil/template/template_test.go → pkg/minikube/bootstrapper/bsutil/ktmpl/template_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package template
+package ktmpl
 
 import (
 	"reflect"

pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha1.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import "text/template"
+
+// V1Alpha1 is for Kubernetes v1.11
+var V1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
+kind: MasterConfiguration
+{{if .NoTaintMaster}}noTaintMaster: true{{end}}
+api:
+  advertiseAddress: {{.AdvertiseAddress}}
+  bindPort: {{.APIServerPort}}
+  controlPlaneEndpoint: localhost
+kubernetesVersion: {{.KubernetesVersion}}
+certificatesDir: {{.CertDir}}
+networking:
+  serviceSubnet: {{.ServiceCIDR}}
+etcd:
+  dataDir: {{.EtcdDataDir}}
+nodeName: {{.NodeName}}
+apiServerCertSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
+{{if .ImageRepository}}imageRepository: {{.ImageRepository}}
+{{end}}{{if .CRISocket}}criSocket: {{.CRISocket}}
+{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }}
+  {{$val}}{{end}}
+{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
+  {{$i}}: {{$val}}{{end}}
+{{end}}`))

pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ktmpl
+
+import "text/template"
+
+// V1Alpha3 is for Kubernetes v1.12
+var V1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{
+	"printMapInOrder": printMapInOrder,
+}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3
+kind: InitConfiguration
+apiEndpoint:
+  advertiseAddress: {{.AdvertiseAddress}}
+  bindPort: {{.APIServerPort}}
+bootstrapTokens:
+  - groups:
+      - system:bootstrappers:kubeadm:default-node-token
+    ttl: 24h0m0s
+    usages:
+      - signing
+      - authentication
+nodeRegistration:
+  criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
+  name: {{.NodeName}}
+  taints: []
+---
+apiVersion: kubeadm.k8s.io/v1alpha3
+kind: ClusterConfiguration
+{{if .ImageRepository}}imageRepository: {{.ImageRepository}}
+{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }}
+  {{$val}}{{end}}
+{{end -}}
+{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
+  {{$i}}: {{$val}}{{end}}
+{{end -}}
+certificatesDir: {{.CertDir}}
+clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
+controlPlaneEndpoint: localhost:{{.APIServerPort}}
+etcd:
+  local:
+    dataDir: {{.EtcdDataDir}}
+kubernetesVersion: {{.KubernetesVersion}}
+networking:
+  dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
+  podSubnet: "{{ .PodSubnet }}"
+  serviceSubnet: {{.ServiceCIDR}}
+---
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+evictionHard:
+  nodefs.available: "0%"
+  nodefs.inodesFree: "0%"
+  imagefs.available: "0%"
+`))

pkg/minikube/bootstrapper/bsutil/template/template.go → pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go

@@ -14,90 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package template
+package ktmpl
 
-import (
-	"fmt"
-	"sort"
-	"text/template"
-)
+import "text/template"
 
-// KubeAdmConfigTmplV1Alpha1 is for Kubernetes v1.11
-var KubeAdmConfigTmplV1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{
-	"printMapInOrder": printMapInOrder,
-}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
-kind: MasterConfiguration
-{{if .NoTaintMaster}}noTaintMaster: true{{end}}
-api:
-  advertiseAddress: {{.AdvertiseAddress}}
-  bindPort: {{.APIServerPort}}
-  controlPlaneEndpoint: localhost
-kubernetesVersion: {{.KubernetesVersion}}
-certificatesDir: {{.CertDir}}
-networking:
-  serviceSubnet: {{.ServiceCIDR}}
-etcd:
-  dataDir: {{.EtcdDataDir}}
-nodeName: {{.NodeName}}
-{{if .ImageRepository}}imageRepository: {{.ImageRepository}}
-{{end}}{{if .CRISocket}}criSocket: {{.CRISocket}}
-{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }}
-  {{$val}}{{end}}
-{{end}}{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
-  {{$i}}: {{$val}}{{end}}
-{{end}}`))
-
-// KubeAdmConfigTmplV1Alpha3 is for Kubernetes v1.12
-var KubeAdmConfigTmplV1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{
-	"printMapInOrder": printMapInOrder,
-}).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3
-kind: InitConfiguration
-apiEndpoint:
-  advertiseAddress: {{.AdvertiseAddress}}
-  bindPort: {{.APIServerPort}}
-bootstrapTokens:
-  - groups:
-      - system:bootstrappers:kubeadm:default-node-token
-    ttl: 24h0m0s
-    usages:
-      - signing
-      - authentication
-nodeRegistration:
-  criSocket: {{if .CRISocket}}{{.CRISocket}}{{else}}/var/run/dockershim.sock{{end}}
-  name: {{.NodeName}}
-  taints: []
----
-apiVersion: kubeadm.k8s.io/v1alpha3
-kind: ClusterConfiguration
-{{if .ImageRepository}}imageRepository: {{.ImageRepository}}
-{{end}}{{range .ExtraArgs}}{{.Component}}ExtraArgs:{{range $i, $val := printMapInOrder .Options ": " }}
-  {{$val}}{{end}}
-{{end -}}
-{{if .FeatureArgs}}featureGates: {{range $i, $val := .FeatureArgs}}
-  {{$i}}: {{$val}}{{end}}
-{{end -}}
-certificatesDir: {{.CertDir}}
-clusterName: kubernetes
-controlPlaneEndpoint: localhost:{{.APIServerPort}}
-etcd:
-  local:
-    dataDir: {{.EtcdDataDir}}
-kubernetesVersion: {{.KubernetesVersion}}
-networking:
-  dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
-  podSubnet: {{if .PodSubnet}}{{.PodSubnet}}{{else}}""{{end}}
-  serviceSubnet: {{.ServiceCIDR}}
----
-apiVersion: kubelet.config.k8s.io/v1beta1
-kind: KubeletConfiguration
-evictionHard:
-  nodefs.available: "0%"
-  nodefs.inodesFree: "0%"
-  imagefs.available: "0%"
-`))
-
-// KubeAdmConfigTmplV1Beta1 is for Kubernetes v1.13+
-var KubeAdmConfigTmplV1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{
+// V1Beta1 is kubeadm config template for Kubernetes v1.13+
+var V1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{
 	"printMapInOrder": printMapInOrder,
 }).Parse(`apiVersion: kubeadm.k8s.io/v1beta1
 kind: InitConfiguration
@@ -130,6 +52,8 @@ kind: ClusterConfiguration
 {{end -}}{{end -}}
 certificatesDir: {{.CertDir}}
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "{{.AdvertiseAddress}}"]
 controlPlaneEndpoint: localhost:{{.APIServerPort}}
 dns:
   type: CoreDNS
@@ -139,7 +63,7 @@ etcd:
 kubernetesVersion: {{.KubernetesVersion}}
 networking:
   dnsDomain: {{if .DNSDomain}}{{.DNSDomain}}{{else}}cluster.local{{end}}
-  podSubnet: ""
+  podSubnet: "{{.PodSubnet }}"
   serviceSubnet: {{.ServiceCIDR}}
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
@@ -150,22 +74,3 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 `))
-
-// printMapInOrder sorts the keys and prints the map in order, combining key
-// value pairs with the separator character
-//
-// Note: this is not necessary, but makes testing easy
-func printMapInOrder(m map[string]string, sep string) []string {
-	if m == nil {
-		return nil
-	}
-	keys := []string{}
-	for k := range m {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	for i, k := range keys {
-		keys[i] = fmt.Sprintf("%s%s\"%s\"", k, sep, m[k])
-	}
-	return keys
-}

pkg/minikube/bootstrapper/bsutil/kubeadm.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (
@@ -24,12 +24,11 @@ import (
 
 	"github.com/blang/semver"
 	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/template"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/vmpath"
-	"k8s.io/minikube/pkg/util"
 )
 
 // Container runtimes
@@ -53,7 +52,7 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt
 		return nil, errors.Wrap(err, "generating extra component config for kubeadm")
 	}
 
-	// In case of no port assigned, use util.APIServerPort
+	// In case of no port assigned, use default
 	nodePort := k8s.NodePort
 	if nodePort <= 0 {
 		nodePort = constants.APIServerPort
@@ -76,7 +75,7 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt
 		NoTaintMaster     bool
 	}{
 		CertDir:           vmpath.GuestCertsDir,
-		ServiceCIDR:       util.DefaultServiceCIDR,
+		ServiceCIDR:       constants.DefaultServiceCIDR,
 		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
 		AdvertiseAddress:  k8s.NodeIP,
 		APIServerPort:     nodePort,
@@ -97,13 +96,13 @@ func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byt
 
 	opts.NoTaintMaster = true
 	b := bytes.Buffer{}
-	configTmpl := template.KubeAdmConfigTmplV1Alpha1
+	configTmpl := ktmpl.V1Alpha1
 	if version.GTE(semver.MustParse("1.12.0")) {
-		configTmpl = template.KubeAdmConfigTmplV1Alpha3
+		configTmpl = ktmpl.V1Alpha3
 	}
 	// v1beta1 works in v1.13, but isn't required until v1.14.
 	if version.GTE(semver.MustParse("1.14.0-alpha.0")) {
-		configTmpl = template.KubeAdmConfigTmplV1Beta1
+		configTmpl = ktmpl.V1Beta1
 	}
 	if err := configTmpl.Execute(&b, opts); err != nil {
 		return nil, err

pkg/minikube/bootstrapper/bsutil/kubeadm_test.go

@@ -101,7 +101,7 @@ This test case has only 1 thing to test and that is the
 nnetworking/dnsDomain value
 */
 func TestGenerateKubeadmYAMLDNS(t *testing.T) {
-	versions := []string{"v1.16", "v1.15", "v1.14", "v1.13", "v1.12"}
+	versions := []string{"v1.17", "v1.16", "v1.15", "v1.14", "v1.13", "v1.12"}
 	tests := []struct {
 		name      string
 		runtime   string

pkg/minikube/bootstrapper/bsutil/kubelet.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (
@@ -22,7 +22,7 @@ import (
 	"path"
 
 	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/template"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/ktmpl"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@@ -76,7 +76,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte,
 		ContainerRuntime: k8s.ContainerRuntime,
 		KubeletPath:      path.Join(binRoot(k8s.KubernetesVersion), "kubelet"),
 	}
-	if err := template.KubeletSystemdTemplate.Execute(&b, opts); err != nil {
+	if err := ktmpl.KubeletSystemdTemplate.Execute(&b, opts); err != nil {
 		return nil, err
 	}
 
@@ -87,7 +87,7 @@ func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte,
 func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) {
 	var b bytes.Buffer
 	opts := struct{ KubeletPath string }{KubeletPath: path.Join(binRoot(cfg.KubernetesVersion), "kubelet")}
-	if err := template.KubeletServiceTemplate.Execute(&b, opts); err != nil {
+	if err := ktmpl.KubeletServiceTemplate.Execute(&b, opts); err != nil {
 		return nil, errors.Wrap(err, "template execute")
 	}
 	return b.Bytes(), nil

pkg/minikube/bootstrapper/bsutil/kubelet_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
 package bsutil
 
 import (

pkg/minikube/bootstrapper/bsutil/kverify/kverify.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package kverify verifies a running kubernetes cluster is healthy
+package kverify
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"os/exec"
+	"time"
+
+	"github.com/docker/machine/libmachine/state"
+	"github.com/golang/glog"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/minikube/pkg/minikube/command"
+)
+
+// APIServerProcess waits for api server to be healthy returns error if it doesn't
+func APIServerProcess(runner command.Runner, start time.Time, timeout time.Duration) error {
+	glog.Infof("waiting for apiserver process to appear ...")
+	err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during process check")
+		}
+		rr, ierr := runner.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
+		if ierr != nil {
+			glog.Warningf("pgrep apiserver: %v cmd: %s", ierr, rr.Command())
+			return false, nil
+		}
+		return true, nil
+	})
+	if err != nil {
+		return fmt.Errorf("apiserver process never appeared")
+	}
+	glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start))
+	return nil
+}
+
+// SystemPods verifies essential pods for running kurnetes is running
+func SystemPods(client *kubernetes.Clientset, start time.Time, ip string, port int, timeout time.Duration) error {
+	glog.Infof("waiting for kube-system pods to appear %s...", net.JoinHostPort(ip, fmt.Sprint(port)))
+	pStart := time.Now()
+	podStart := time.Time{}
+	podList := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during pod check")
+		}
+		// Wait for any system pod, as waiting for apiserver may block until etcd
+		pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{})
+		if len(pods.Items) < 2 {
+			podStart = time.Time{}
+			return false, nil
+		}
+		if err != nil {
+			podStart = time.Time{}
+			return false, nil
+		}
+		if podStart.IsZero() {
+			podStart = time.Now()
+		}
+
+		glog.Infof("%d kube-system pods found since %s", len(pods.Items), podStart)
+		if time.Since(podStart) > 2*kconst.APICallRetryInterval {
+			glog.Infof("stability requirement met, returning")
+			return true, nil
+		}
+		return false, nil
+	}
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, podList); err != nil {
+		return fmt.Errorf("apiserver never returned a pod list")
+	}
+	glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
+	return nil
+}
+
+// APIServerIsRunning waits for api server status to be running
+func APIServerIsRunning(start time.Time, ip string, port int, timeout time.Duration) error {
+	glog.Infof("waiting for apiserver healthz status ...")
+	hStart := time.Now()
+	healthz := func() (bool, error) {
+		if time.Since(start) > timeout {
+			return false, fmt.Errorf("cluster wait timed out during healthz check")
+		}
+
+		status, err := APIServerStatus(net.ParseIP(ip), port)
+		if err != nil {
+			glog.Warningf("status: %v", err)
+			return false, nil
+		}
+		if status != "Running" {
+			return false, nil
+		}
+		return true, nil
+	}
+
+	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil {
+		return fmt.Errorf("apiserver healthz never reported healthy")
+	}
+	glog.Infof("duration metric: took %s to wait for apiserver healthz status ...", time.Since(hStart))
+	return nil
+}
+
+// APIServerStatus hits the /healthz endpoint and returns libmachine style state.State
+func APIServerStatus(ip net.IP, apiserverPort int) (string, error) {
+	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), fmt.Sprint(apiserverPort)))
+	// To avoid: x509: certificate signed by unknown authority
+	tr := &http.Transport{
+		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
+		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	}
+	client := &http.Client{Transport: tr}
+	resp, err := client.Get(url)
+	// Connection refused, usually.
+	if err != nil {
+		return state.Stopped.String(), nil
+	}
+	if resp.StatusCode != http.StatusOK {
+		glog.Warningf("%s response: %v %+v", url, err, resp)
+		return state.Error.String(), nil
+	}
+	return state.Running.String(), nil
+}

pkg/minikube/bootstrapper/kubeadm/rbac.go → pkg/minikube/bootstrapper/bsutil/rbac.go

@@ -14,10 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubeadm
+package bsutil
 
 import (
+	"fmt"
 	"net"
+	"os/exec"
+	"strings"
 	"time"
 
 	"github.com/golang/glog"
@@ -25,6 +28,7 @@ import (
 	rbac "k8s.io/api/rbac/v1beta1"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
@@ -32,9 +36,9 @@ const (
 	rbacName = "minikube-rbac"
 )
 
-// elevateKubeSystemPrivileges gives the kube-system service account
+// ElevateKubeSystemPrivileges gives the kube-system service account
 // cluster admin privileges to work with RBAC.
-func elevateKubeSystemPrivileges(client kubernetes.Interface) error {
+func ElevateKubeSystemPrivileges(client kubernetes.Interface) error {
 	start := time.Now()
 	clusterRoleBinding := &rbac.ClusterRoleBinding{
 		ObjectMeta: meta.ObjectMeta{
@@ -67,3 +71,31 @@ func elevateKubeSystemPrivileges(client kubernetes.Interface) error {
 	glog.Infof("duration metric: took %s to wait for elevateKubeSystemPrivileges.", time.Since(start))
 	return nil
 }
+
+// AdjustResourceLimits makes fine adjustments to pod resources that aren't possible via kubeadm config.
+func AdjustResourceLimits(c command.Runner) error {
+	rr, err := c.RunCmd(exec.Command("/bin/bash", "-c", "cat /proc/$(pgrep kube-apiserver)/oom_adj"))
+	if err != nil {
+		return errors.Wrapf(err, "oom_adj check cmd %s. ", rr.Command())
+	}
+	glog.Infof("apiserver oom_adj: %s", rr.Stdout.String())
+	// oom_adj is already a negative number
+	if strings.HasPrefix(rr.Stdout.String(), "-") {
+		return nil
+	}
+	glog.Infof("adjusting apiserver oom_adj to -10")
+
+	// Prevent the apiserver from OOM'ing before other pods, as it is our gateway into the cluster.
+	// It'd be preferable to do this via Kubernetes, but kubeadm doesn't have a way to set pod QoS.
+	if _, err = c.RunCmd(exec.Command("/bin/bash", "-c", "echo -10 | sudo tee /proc/$(pgrep kube-apiserver)/oom_adj")); err != nil {
+		return errors.Wrap(err, fmt.Sprintf("oom_adj adjust"))
+	}
+	return nil
+}
+
+// ExistingConfig checks if there are config files from possible previous kubernets cluster
+func ExistingConfig(c command.Runner) error {
+	args := append([]string{"ls"}, expectedRemoteArtifacts...)
+	_, err := c.RunCmd(exec.Command("sudo", args...))
+	return err
+}

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-api-port.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd-pod-network-cidr.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/containerd.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /run/containerd/containerd.sock
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio-options-gates.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /var/run/crio/crio.sock
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/crio.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 criSocket: /var/run/crio/crio.sock
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/default.yaml

@@ -12,5 +12,6 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/image-repository.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 imageRepository: test/repo
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"

pkg/minikube/bootstrapper/bsutil/testdata/v1.11/options.yaml

@@ -12,6 +12,7 @@ networking:
 etcd:
   dataDir: /var/lib/minikube/etcd
 nodeName: mk
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
   fail-no-swap: "true"

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
@@ -28,7 +29,7 @@ etcd:
 kubernetesVersion: v1.12.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: 192.168.32.0/20
+  podSubnet: "192.168.32.0/20"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml

@@ -29,6 +29,7 @@ schedulerExtraArgs:
   scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml

@@ -22,6 +22,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml

@@ -26,6 +26,7 @@ schedulerExtraArgs:
   scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:
@@ -28,7 +29,7 @@ etcd:
 kubernetesVersion: v1.13.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: 192.168.32.0/20
+  podSubnet: "192.168.32.0/20"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml

@@ -29,6 +29,7 @@ schedulerExtraArgs:
   scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml

@@ -21,6 +21,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml

@@ -22,6 +22,7 @@ apiServerExtraArgs:
   enable-admission-plugins: "Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml

@@ -26,6 +26,7 @@ schedulerExtraArgs:
   scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServerCertSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 etcd:
   local:

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
@@ -31,7 +33,7 @@ etcd:
 kubernetesVersion: v1.14.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: "192.168.32.0/20"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml

@@ -32,6 +32,8 @@ scheduler:
     scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml

@@ -23,6 +23,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml

@@ -29,6 +29,8 @@ scheduler:
     scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
@@ -31,7 +33,7 @@ etcd:
 kubernetesVersion: v1.15.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: "192.168.32.0/20"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml

@@ -32,6 +32,8 @@ scheduler:
     scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml

@@ -23,6 +23,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml

@@ -29,6 +29,8 @@ scheduler:
     scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
@@ -31,7 +33,7 @@ etcd:
 kubernetesVersion: v1.16.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: "192.168.32.0/20"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml

@@ -32,6 +32,8 @@ scheduler:
     scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml

@@ -23,6 +23,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml

@@ -29,6 +29,8 @@ scheduler:
     scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:12345
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS
@@ -31,7 +33,7 @@ etcd:
 kubernetesVersion: v1.17.0
 networking:
   dnsDomain: cluster.local
-  podSubnet: ""
+  podSubnet: "192.168.32.0/20"
   serviceSubnet: 10.96.0.0/12
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml

@@ -32,6 +32,8 @@ scheduler:
     scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml

@@ -22,6 +22,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml

@@ -23,6 +23,8 @@ apiServer:
     enable-admission-plugins: "NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,MutatingAdmissionWebhook,ValidatingAdmissionWebhook,ResourceQuota"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml

@@ -29,6 +29,8 @@ scheduler:
     scheduler-name: "mini-scheduler"
 certificatesDir: /var/lib/minikube/certs
 clusterName: kubernetes
+apiServer:
+  certSANs: ["127.0.0.1", "localhost", "1.1.1.1"]
 controlPlaneEndpoint: localhost:8443
 dns:
   type: CoreDNS

pkg/minikube/bootstrapper/bsutil/types.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package bsutil will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+// SkipAdditionalPreflights are additional preflights we skip depending on the runtime in use.
+var SkipAdditionalPreflights = map[string][]string{}
+
+// expectedRemoteArtifacts remote artifacts that must exist for minikube to function properly. The sign of a previously working installation.
+// NOTE: /etc is not persistent across restarts, so don't bother checking there
+var expectedRemoteArtifacts = []string{
+	"/var/lib/kubelet/kubeadm-flags.env",
+	"/var/lib/kubelet/config.yaml",
+	EtcdDataDir(),
+}

pkg/minikube/bootstrapper/certs.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/clientcmd/api"
 	"k8s.io/client-go/tools/clientcmd/api/latest"
+	"k8s.io/minikube/pkg/drivers/kic"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -110,7 +111,7 @@ func SetupCerts(cmd command.Runner, k8s config.KubernetesConfig) error {
 
 	kcs := &kubeconfig.Settings{
 		ClusterName:          k8s.NodeName,
-		ClusterServerAddress: fmt.Sprintf("https://localhost:%d", k8s.NodePort),
+		ClusterServerAddress: fmt.Sprintf("https://%s", net.JoinHostPort("localhost", fmt.Sprint(k8s.NodePort))),
 		ClientCertificate:    path.Join(vmpath.GuestCertsDir, "apiserver.crt"),
 		ClientKey:            path.Join(vmpath.GuestCertsDir, "apiserver.key"),
 		CertificateAuthority: path.Join(vmpath.GuestCertsDir, "ca.crt"),
@@ -175,7 +176,7 @@ func generateCerts(k8s config.KubernetesConfig) error {
 
 	apiServerIPs := append(
 		k8s.APIServerIPs,
-		[]net.IP{net.ParseIP(k8s.NodeIP), serviceIP, net.ParseIP("10.0.0.1")}...)
+		[]net.IP{net.ParseIP(k8s.NodeIP), serviceIP, net.ParseIP(kic.DefaultBindIPV4), net.ParseIP("10.0.0.1")}...)
 	apiServerNames := append(k8s.APIServerNames, k8s.APIServerName)
 	apiServerAlternateNames := append(
 		apiServerNames,

pkg/minikube/bootstrapper/certs_test.go

@@ -38,7 +38,7 @@ func TestSetupCerts(t *testing.T) {
 	k8s := config.KubernetesConfig{
 		APIServerName: constants.APIServerName,
 		DNSDomain:     constants.ClusterDNSDomain,
-		ServiceCIDR:   util.DefaultServiceCIDR,
+		ServiceCIDR:   constants.DefaultServiceCIDR,
 	}
 
 	if err := os.Mkdir(filepath.Join(tempDir, "certs"), 0777); err != nil {

pkg/minikube/bootstrapper/images/images.go

@@ -18,20 +18,99 @@ limitations under the License.
 package images
 
 import (
+	"fmt"
 	"path"
 	"runtime"
+
+	"github.com/blang/semver"
 )
 
-// ArchTag returns a CPU architecture suffix for images
-func ArchTag(hasTag bool) string {
+// Pause returns the image name to pull for the pause image
+func Pause(mirror string) string {
+	// Should match `PauseVersion` in:
+	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
+	return path.Join(kubernetesRepo(mirror), "pause"+archTag(false)+"3.1")
+}
+
+// essentials returns images needed too bootstrap a kubenretes
+func essentials(mirror string, v semver.Version) []string {
+	imgs := []string{
+		componentImage("kube-proxy", v, mirror),
+		componentImage("kube-scheduler", v, mirror),
+		componentImage("kube-controller-manager", v, mirror),
+		componentImage("kube-apiserver", v, mirror),
+		coreDNS(v, mirror),
+		etcd(v, mirror),
+		Pause(mirror),
+	}
+	return imgs
+}
+
+// componentImage returns a Kubernetes component image to pull
+func componentImage(name string, v semver.Version, mirror string) string {
+	needsArchSuffix := false
+	ancient := semver.MustParseRange("<1.12.0")
+	if ancient(v) {
+		needsArchSuffix = true
+	}
+
+	return fmt.Sprintf("%sv%s", path.Join(kubernetesRepo(mirror), name+archTag(needsArchSuffix)), v)
+}
+
+// coreDNS returns the images used for CoreDNS
+func coreDNS(v semver.Version, mirror string) string {
+	// Should match `CoreDNSVersion` in
+	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
+	cv := "1.6.5"
+	switch v.Minor {
+	case 16:
+		cv = "1.6.2"
+	case 15, 14:
+		cv = "1.3.1"
+	case 13:
+		cv = "1.2.6"
+	case 12:
+		cv = "1.2.2"
+	case 11:
+		cv = "1.1.3"
+	}
+	return path.Join(kubernetesRepo(mirror), "coredns"+archTag(false)+cv)
+}
+
+// etcd returns the image used for etcd
+func etcd(v semver.Version, mirror string) string {
+	needsArchSuffix := false
+	ancient := semver.MustParseRange("<1.12.0")
+	if ancient(v) {
+		needsArchSuffix = true
+	}
+
+	// Should match `DefaultEtcdVersion` in:
+	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
+	ev := "3.4.3-0"
+	switch v.Minor {
+	case 16:
+		ev = "3.3.15-0"
+	case 14, 15:
+		ev = "3.3.10"
+	case 12, 13:
+		ev = "3.2.24"
+	case 11:
+		ev = "3.2.18"
+	}
+	return path.Join(kubernetesRepo(mirror), "etcd"+archTag(needsArchSuffix)+ev)
+}
+
+// archTag returns a CPU architecture suffix for images
+func archTag(hasTag bool) string {
 	if runtime.GOARCH == "amd64" && !hasTag {
 		return ":"
 	}
 	return "-" + runtime.GOARCH + ":"
 }
 
-// Auxiliary returns images that are helpful for running minikube
-func Auxiliary(mirror string) []string {
+// auxiliary returns images that are helpful for running minikube
+func auxiliary(mirror string) []string {
 	return []string{
 		addonManager(mirror),
 		storageProvisioner(mirror),
@@ -40,21 +119,14 @@ func Auxiliary(mirror string) []string {
 	}
 }
 
-// Pause returns the image name to pull for the pause image
-func Pause(mirror string) string {
-	// Should match `PauseVersion` in:
-	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
-	return path.Join(KubernetesRepo(mirror), "pause"+ArchTag(false)+"3.1")
-}
-
 // storageProvisioner returns the minikube storage provisioner image
 func storageProvisioner(mirror string) string {
-	return path.Join(minikubeRepo(mirror), "storage-provisioner"+ArchTag(false)+"v1.8.1")
+	return path.Join(minikubeRepo(mirror), "storage-provisioner"+archTag(false)+"v1.8.1")
 }
 
 // addonManager returns the Kubernetes addon manager image
 func addonManager(mirror string) string {
-	return path.Join(KubernetesRepo(mirror), "kube-addon-manager"+ArchTag(false)+"v9.0.2")
+	return path.Join(kubernetesRepo(mirror), "kube-addon-manager"+archTag(false)+"v9.0.2")
 }
 
 // dashboardFrontend returns the image used for the dashboard frontend

pkg/minikube/bootstrapper/images/images_test.go

@@ -29,7 +29,7 @@ func TestAuxiliary(t *testing.T) {
 		"kubernetesui/dashboard:v2.0.0-beta8",
 		"kubernetesui/metrics-scraper:v1.0.2",
 	}
-	got := Auxiliary("")
+	got := auxiliary("")
 	if diff := cmp.Diff(want, got); diff != "" {
 		t.Errorf("images mismatch (-want +got):\n%s", diff)
 	}
@@ -42,7 +42,7 @@ func TestAuxiliaryMirror(t *testing.T) {
 		"test.mirror/dashboard:v2.0.0-beta8",
 		"test.mirror/metrics-scraper:v1.0.2",
 	}
-	got := Auxiliary("test.mirror")
+	got := auxiliary("test.mirror")
 	if diff := cmp.Diff(want, got); diff != "" {
 		t.Errorf("images mismatch (-want +got):\n%s", diff)
 	}

pkg/minikube/bootstrapper/images/kic.go

+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package images
+
+import (
+	"strings"
+
+	"github.com/blang/semver"
+	"github.com/pkg/errors"
+)
+
+// KIC returns a list of images necessary to bootstrap kic
+// it is same as vm-kubeadm minus auxulary
+func KIC(mirror string, version string) ([]string, error) {
+	v, err := semver.Make(strings.TrimPrefix(version, "v"))
+	if err != nil {
+		return nil, errors.Wrap(err, "semver")
+	}
+	imgs := essentials(mirror, v)
+	return imgs, nil
+}

pkg/minikube/bootstrapper/images/kubeadm.go

@@ -17,8 +17,6 @@ limitations under the License.
 package images
 
 import (
-	"fmt"
-	"path"
 	"strings"
 
 	"github.com/blang/semver"
@@ -31,71 +29,7 @@ func Kubeadm(mirror string, version string) ([]string, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "semver")
 	}
-
-	imgs := []string{
-		componentImage("kube-proxy", v, mirror),
-		componentImage("kube-scheduler", v, mirror),
-		componentImage("kube-controller-manager", v, mirror),
-		componentImage("kube-apiserver", v, mirror),
-		coreDNS(v, mirror),
-		etcd(v, mirror),
-		Pause(mirror),
-	}
-	imgs = append(imgs, Auxiliary(mirror)...)
+	imgs := essentials(mirror, v)
+	imgs = append(imgs, auxiliary(mirror)...)
 	return imgs, nil
 }
-
-// componentImage returns a Kubernetes component image to pull
-func componentImage(name string, v semver.Version, mirror string) string {
-	needsArchSuffix := false
-	ancient := semver.MustParseRange("<1.12.0")
-	if ancient(v) {
-		needsArchSuffix = true
-	}
-
-	return fmt.Sprintf("%sv%s", path.Join(KubernetesRepo(mirror), name+ArchTag(needsArchSuffix)), v)
-}
-
-// coreDNS returns the images used for CoreDNS
-func coreDNS(v semver.Version, mirror string) string {
-	// Should match `CoreDNSVersion` in
-	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
-	cv := "1.6.5"
-	switch v.Minor {
-	case 16:
-		cv = "1.6.2"
-	case 15, 14:
-		cv = "1.3.1"
-	case 13:
-		cv = "1.2.6"
-	case 12:
-		cv = "1.2.2"
-	case 11:
-		cv = "1.1.3"
-	}
-	return path.Join(KubernetesRepo(mirror), "coredns"+ArchTag(false)+cv)
-}
-
-// etcd returns the image used for etcd
-func etcd(v semver.Version, mirror string) string {
-	needsArchSuffix := false
-	ancient := semver.MustParseRange("<1.12.0")
-	if ancient(v) {
-		needsArchSuffix = true
-	}
-
-	// Should match `DefaultEtcdVersion` in:
-	// https://github.com/kubernetes/kubernetes/blob/master/cmd/kubeadm/app/constants/constants.go
-	ev := "3.4.3-0"
-	switch v.Minor {
-	case 16:
-		ev = "3.3.15-0"
-	case 14, 15:
-		ev = "3.3.10"
-	case 12, 13:
-		ev = "3.2.24"
-	case 11:
-		ev = "3.2.18"
-	}
-	return path.Join(KubernetesRepo(mirror), "etcd"+ArchTag(needsArchSuffix)+ev)
-}

pkg/minikube/bootstrapper/images/repo.go

@@ -19,8 +19,8 @@ package images
 // DefaultKubernetesRepo is the default Kubernetes repository
 const DefaultKubernetesRepo = "k8s.gcr.io"
 
-// KubernetesRepo returns the official Kubernetes repository, or an alternate
-func KubernetesRepo(mirror string) string {
+// kubernetesRepo returns the official Kubernetes repository, or an alternate
+func kubernetesRepo(mirror string) string {
 	if mirror != "" {
 		return mirror
 	}

pkg/minikube/bootstrapper/kicbs/cni.go

@@ -14,12 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bootstrapper for kic
+// Package kicbs bootstrapper for kic
 package kicbs
 
 const defaultCNIManifest = `
-# kindnetd networking manifest
-# would you kindly template this file
 ---
 kind: ClusterRole
 apiVersion: rbac.authorization.k8s.io/v1
@@ -99,7 +97,7 @@ spec:
             fieldRef:
               fieldPath: status.podIP
         - name: POD_SUBNET
-          value: {{ .PodSubnet }}
+          value: 10.244.0.0/16
         volumeMounts:
         - name: cni-cfg
           mountPath: /etc/cni/net.d

pkg/minikube/bootstrapper/kicbs/kicbs.go

@@ -14,21 +14,28 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-// bootstrapper for kic
+// Package kicbs bootstrapper for kic
 package kicbs
 
 import (
 	"fmt"
 	"net"
 	"os/exec"
+	"strings"
 	"time"
 
+	"github.com/blang/semver"
 	"github.com/docker/machine/libmachine"
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
+	"k8s.io/client-go/kubernetes"
+	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
+	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -36,11 +43,13 @@ import (
 	"k8s.io/minikube/pkg/minikube/cruntime"
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
+	"k8s.io/minikube/pkg/minikube/vmpath"
 )
 
 // Bootstrapper is a bootstrapper using kicbs
 type Bootstrapper struct {
 	c           command.Runner
+	k8sClient   *kubernetes.Clientset // kubernetes client used to verify pods inside cluster
 	contextName string
 }
 
@@ -60,9 +69,9 @@ func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) {
 
 // UpdateCluster updates the cluster
 func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
-	images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion)
+	images, err := images.KIC(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion)
 	if err != nil {
-		return errors.Wrap(err, "kubeadm images")
+		return errors.Wrap(err, "kic images")
 	}
 
 	if cfg.KubernetesConfig.ShouldLoadCachedImages {
@@ -101,15 +110,15 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 		return errors.Wrap(err, "downloading binaries")
 	}
 
-	var cniFile []byte = nil
-	if cfg.KubernetesConfig.EnableDefaultCNI {
-		cniFile = []byte(defaultCNIManifest)
-	}
+	cniFile := []byte(defaultCNIManifest)
+
 	files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile)
 
-	// if err := addAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil {
+	// TODO: add addons for kic later
+	// if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil {
 	// 	return errors.Wrap(err, "adding addons")
 	// }
+
 	for _, f := range files {
 		if err := k.c.Copy(f); err != nil {
 			return errors.Wrapf(err, "copy")
@@ -122,28 +131,216 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	return nil
 }
 
-func (k *Bootstrapper) PullImages(config.KubernetesConfig) error {
-	return fmt.Errorf("the PullImages is not implemented in kicbs yet")
+// SetupCerts generates the certs the cluster
+func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig) error {
+	return bootstrapper.SetupCerts(k.c, cfg)
 }
-func (k *Bootstrapper) StartCluster(config.KubernetesConfig) error {
-	return fmt.Errorf("the StartCluster is not implemented in kicbs yet")
+
+// PullImages downloads images that will be used by Kubernetes
+func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error {
+	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
+	if err != nil {
+		return errors.Wrap(err, "parsing kubernetes version")
+	}
+	if version.LT(semver.MustParse("1.11.0")) {
+		return fmt.Errorf("pull command is not supported by kubeadm v%s", version)
+	}
+
+	rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s config images pull --config %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath)))
+	if err != nil {
+		return errors.Wrapf(err, "running cmd: %q", rr.Command())
+	}
+	return nil
+}
+
+// StartCluster starts the cluster
+func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
+	k8s.NodeIP = kic.DefaultBindIPV4
+	err := bsutil.ExistingConfig(k.c)
+	if err == nil { // if there is an existing cluster don't reconfigure it
+		return k.restartCluster(k8s)
+	}
+	glog.Infof("existence check: %v", err)
+
+	start := time.Now()
+	glog.Infof("StartCluster: %+v", k8s)
+	defer func() {
+		glog.Infof("StartCluster complete in %s", time.Since(start))
+	}()
+
+	extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions)
+	r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime})
+	if err != nil {
+		return err
+	}
+
+	ignore := []string{
+		fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestManifestsDir, "/", "-", -1)),
+		fmt.Sprintf("DirAvailable-%s", strings.Replace(vmpath.GuestPersistentDir, "/", "-", -1)),
+		fmt.Sprintf("DirAvailable-%s", strings.Replace(bsutil.EtcdDataDir(), "/", "-", -1)),
+		"FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml",
+		"FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml",
+		"FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml",
+		"FileAvailable--etc-kubernetes-manifests-etcd.yaml",
+		"FileContent--proc-sys-net-bridge-bridge-nf-call-iptables", // for kic only
+		"Port-10250",         // For "none" users who already have a kubelet online
+		"Swap",               // For "none" users who have swap configured
+		"SystemVerification", // For kic on linux example error: "modprobe: FATAL: Module configs not found in directory /lib/modules/5.2.17-1rodete3-amd64"
+	}
+	ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...)
+
+	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), bsutil.KubeadmYamlPath, extraFlags, strings.Join(ignore, ",")))
+	glog.Infof("starting kubeadm init")
+	if rr, err := k.c.RunCmd(c); err != nil {
+		return errors.Wrapf(err, "init failed. cmd: %q output: %q", rr.Command(), rr.Output())
+	}
+
+	glog.Infof("applying kic overlay network")
+	if err := k.applyOverlayNetwork(); err != nil {
+		return errors.Wrap(err, "applying kic overlay network")
+	}
+
+	glog.Infof("Skipping Configuring cluster permissions for kic...")
+
+	if err := bsutil.AdjustResourceLimits(k.c); err != nil {
+		glog.Warningf("unable to adjust resource limits: %v", err)
+	}
+
+	return nil
+}
+
+// restartCluster restarts the Kubernetes cluster configured by kubeadm
+func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
+	glog.Infof("restartCluster start")
+
+	start := time.Now()
+	defer func() {
+		glog.Infof("restartCluster took %s", time.Since(start))
+	}()
+
+	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
+	if err != nil {
+		return errors.Wrap(err, "parsing kubernetes version")
+	}
+
+	phase := "alpha"
+	controlPlane := "controlplane"
+	if version.GTE(semver.MustParse("1.13.0")) {
+		phase = "init"
+		controlPlane = "control-plane"
+	}
+
+	baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase)
+	cmds := []string{
+		fmt.Sprintf("%s phase certs all --config %s", baseCmd, bsutil.KubeadmYamlPath),
+		fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, bsutil.KubeadmYamlPath),
+		fmt.Sprintf("%s phase %s all --config %s", baseCmd, controlPlane, bsutil.KubeadmYamlPath),
+		fmt.Sprintf("%s phase etcd local --config %s", baseCmd, bsutil.KubeadmYamlPath),
+	}
+
+	// Run commands one at a time so that it is easier to root cause failures.
+	for _, c := range cmds {
+		rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", c))
+		if err != nil {
+			return errors.Wrapf(err, "running cmd: %s", rr.Command())
+		}
+	}
+
+	// We must ensure that the apiserver is healthy before proceeding
+	if err := kverify.APIServerProcess(k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
+		return errors.Wrap(err, "apiserver healthz")
+	}
+
+	client, err := k.client(k8s)
+	if err != nil {
+		return errors.Wrap(err, "getting k8s client")
+	}
+
+	if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil {
+		return errors.Wrap(err, "system pods")
+	}
+
+	// Explicitly re-enable kubeadm addons (proxy, coredns) so that they will check for IP or configuration changes.
+	if rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s phase addon all --config %s", baseCmd, bsutil.KubeadmYamlPath))); err != nil {
+		return errors.Wrapf(err, fmt.Sprintf("addon phase cmd:%q", rr.Command()))
+	}
+
+	if err := bsutil.AdjustResourceLimits(k.c); err != nil {
+		glog.Warningf("unable to adjust resource limits: %v", err)
+	}
+	return nil
+}
+
+// WaitForCluster blocks until the cluster appears to be healthy
+func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error {
+	start := time.Now()
+	out.T(out.Waiting, "Waiting for cluster to come online ...")
+	if err := kverify.APIServerProcess(k.c, start, timeout); err != nil {
+		return errors.Wrap(err, "wait for api proc")
+	}
+
+	if err := kverify.APIServerIsRunning(start, "127.0.0.1", k8s.NodePort, timeout); err != nil {
+		return err
+	}
+
+	c, err := k.client(k8s) // getting kubernetes client before polling.
+	if err != nil {
+		return errors.Wrap(err, "get k8s client")
+	}
+
+	if err := kverify.SystemPods(c, start, "127.0.0.1", k8s.NodePort, timeout); err != nil {
+		return errors.Wrap(err, "wait for system pods")
+	}
+
+	return nil
 }
 
 func (k *Bootstrapper) DeleteCluster(config.KubernetesConfig) error {
 	return fmt.Errorf("the DeleteCluster is not implemented in kicbs yet")
 }
-func (k *Bootstrapper) WaitForCluster(config.KubernetesConfig, time.Duration) error {
-	return fmt.Errorf("the WaitForCluster is not implemented in kicbs yet")
-}
+
 func (k *Bootstrapper) LogCommands(bootstrapper.LogOptions) map[string]string {
 	return map[string]string{}
 }
-func (k *Bootstrapper) SetupCerts(cfg config.KubernetesConfig) error {
-	return fmt.Errorf("the SetupCerts is not implemented in kicbs yet")
-}
+
 func (k *Bootstrapper) GetKubeletStatus() (string, error) {
 	return "", fmt.Errorf("the GetKubeletStatus is not implemented in kicbs yet")
 }
 func (k *Bootstrapper) GetAPIServerStatus(net.IP, int) (string, error) {
 	return "", fmt.Errorf("the GetAPIServerStatus is not implemented in kicbs yet")
 }
+
+// client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver
+func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) {
+	if k.k8sClient != nil {
+		return k.k8sClient, nil
+	}
+
+	config, err := kapi.ClientConfig(k.contextName)
+	if err != nil {
+		return nil, errors.Wrap(err, "client config")
+	}
+
+	endpoint := fmt.Sprintf("https://%s", net.JoinHostPort("127.0.0.1", fmt.Sprint(k8s.NodePort)))
+	if config.Host != endpoint {
+		glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint)
+		config.Host = endpoint
+	}
+	c, err := kubernetes.NewForConfig(config)
+	if err == nil {
+		k.k8sClient = c
+	}
+	return c, err
+}
+
+// applyOverlayNetwork applies the CNI plugin needed to make kic work
+func (k *Bootstrapper) applyOverlayNetwork() error {
+	cmd := exec.Command(
+		"kubectl", "create", "--kubeconfig=/etc/kubernetes/admin.conf",
+		"-f", bsutil.DefaultCNIConfigPath,
+	)
+	if rr, err := k.c.RunCmd(cmd); err != nil {
+		return errors.Wrapf(err, "cmd: %s output: %s", rr.Command(), rr.Output())
+	}
+	return nil
+}

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -17,12 +17,10 @@ limitations under the License.
 package kubeadm
 
 import (
-	"crypto/tls"
 	"os/exec"
 
 	"fmt"
 	"net"
-	"net/http"
 
 	// WARNING: Do not use path/filepath in this package unless you want bizarre Windows paths
 
@@ -36,14 +34,13 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
-	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	kconst "k8s.io/kubernetes/cmd/kubeadm/app/constants"
 	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil/kverify"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -55,20 +52,10 @@ import (
 	"k8s.io/minikube/pkg/util/retry"
 )
 
-// remote artifacts that must exist for minikube to function properly. The sign of a previously working installation.
-// NOTE: /etc is not persistent across restarts, so don't bother checking there
-var expectedArtifacts = []string{
-	"/var/lib/kubelet/kubeadm-flags.env",
-	"/var/lib/kubelet/config.yaml",
-	bsutil.EtcdDataDir(),
-}
-
-// SkipAdditionalPreflights are additional preflights we skip depending on the runtime in use.
-var SkipAdditionalPreflights = map[string][]string{}
-
 // Bootstrapper is a bootstrapper using kubeadm
 type Bootstrapper struct {
 	c           command.Runner
+	k8sClient   *kubernetes.Clientset // kubernetes client used to verify pods inside cluster
 	contextName string
 }
 
@@ -83,7 +70,7 @@ func NewBootstrapper(api libmachine.API) (*Bootstrapper, error) {
 	if err != nil {
 		return nil, errors.Wrap(err, "command runner")
 	}
-	return &Bootstrapper{c: runner, contextName: name}, nil
+	return &Bootstrapper{c: runner, contextName: name, k8sClient: nil}, nil
 }
 
 // GetKubeletStatus returns the kubelet status
@@ -106,23 +93,7 @@ func (k *Bootstrapper) GetKubeletStatus() (string, error) {
 
 // GetAPIServerStatus returns the api-server status
 func (k *Bootstrapper) GetAPIServerStatus(ip net.IP, apiserverPort int) (string, error) {
-	url := fmt.Sprintf("https://%s/healthz", net.JoinHostPort(ip.String(), strconv.Itoa(apiserverPort)))
-	// To avoid: x509: certificate signed by unknown authority
-	tr := &http.Transport{
-		Proxy:           nil, // To avoid connectiv issue if http(s)_proxy is set.
-		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-	}
-	client := &http.Client{Transport: tr}
-	resp, err := client.Get(url)
-	// Connection refused, usually.
-	if err != nil {
-		return state.Stopped.String(), nil
-	}
-	if resp.StatusCode != http.StatusOK {
-		glog.Warningf("%s response: %v %+v", url, err, resp)
-		return state.Error.String(), nil
-	}
-	return state.Running.String(), nil
+	return kverify.APIServerStatus(ip, apiserverPort)
 }
 
 // LogCommands returns a map of log type to a command which will display that log.
@@ -167,16 +138,10 @@ func (k *Bootstrapper) createCompatSymlinks() error {
 	return nil
 }
 
-func (k *Bootstrapper) existingConfig() error {
-	args := append([]string{"ls"}, expectedArtifacts...)
-	_, err := k.c.RunCmd(exec.Command("sudo", args...))
-	return err
-}
-
 // StartCluster starts the cluster
 func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
-	err := k.existingConfig()
-	if err == nil {
+	err := bsutil.ExistingConfig(k.c)
+	if err == nil { // if there is an existing cluster don't reconfigure it
 		return k.restartCluster(k8s)
 	}
 	glog.Infof("existence check: %v", err)
@@ -209,7 +174,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 		"Port-10250", // For "none" users who already have a kubelet online
 		"Swap",       // For "none" users who have swap configured
 	}
-	ignore = append(ignore, SkipAdditionalPreflights[r.Name()]...)
+	ignore = append(ignore, bsutil.SkipAdditionalPreflights[r.Name()]...)
 
 	// Allow older kubeadm versions to function with newer Docker releases.
 	if version.LT(semver.MustParse("1.13.0")) {
@@ -223,79 +188,31 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 	}
 
 	glog.Infof("Configuring cluster permissions ...")
-
 	elevate := func() error {
 		client, err := k.client(k8s)
 		if err != nil {
 			return err
 		}
-		return elevateKubeSystemPrivileges(client)
+		return bsutil.ElevateKubeSystemPrivileges(client)
 	}
 
 	if err := retry.Expo(elevate, time.Millisecond*500, 120*time.Second); err != nil {
 		return errors.Wrap(err, "timed out waiting to elevate kube-system RBAC privileges")
 	}
 
-	if err := k.adjustResourceLimits(); err != nil {
+	if err := bsutil.AdjustResourceLimits(k.c); err != nil {
 		glog.Warningf("unable to adjust resource limits: %v", err)
 	}
 
 	return nil
 }
 
-// adjustResourceLimits makes fine adjustments to pod resources that aren't possible via kubeadm config.
-func (k *Bootstrapper) adjustResourceLimits() error {
-	rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", "cat /proc/$(pgrep kube-apiserver)/oom_adj"))
-	if err != nil {
-		return errors.Wrapf(err, "oom_adj check cmd %s. ", rr.Command())
-	}
-	glog.Infof("apiserver oom_adj: %s", rr.Stdout.String())
-	// oom_adj is already a negative number
-	if strings.HasPrefix(rr.Stdout.String(), "-") {
-		return nil
-	}
-	glog.Infof("adjusting apiserver oom_adj to -10")
-
-	// Prevent the apiserver from OOM'ing before other pods, as it is our gateway into the cluster.
-	// It'd be preferable to do this via Kubernetes, but kubeadm doesn't have a way to set pod QoS.
-	if _, err = k.c.RunCmd(exec.Command("/bin/bash", "-c", "echo -10 | sudo tee /proc/$(pgrep kube-apiserver)/oom_adj")); err != nil {
-		return errors.Wrap(err, fmt.Sprintf("oom_adj adjust"))
-	}
-
-	return nil
-}
-
-func addAddons(files *[]assets.CopyableFile, data interface{}) error {
-	// add addons to file list
-	// custom addons
-	if err := assets.AddMinikubeDirAssets(files); err != nil {
-		return errors.Wrap(err, "adding minikube dir assets")
-	}
-	// bundled addons
-	for _, addonBundle := range assets.Addons {
-		if isEnabled, err := addonBundle.IsEnabled(); err == nil && isEnabled {
-			for _, addon := range addonBundle.Assets {
-				if addon.IsTemplate() {
-					addonFile, err := addon.Evaluate(data)
-					if err != nil {
-						return errors.Wrapf(err, "evaluate bundled addon %s asset", addon.GetAssetName())
-					}
-
-					*files = append(*files, addonFile)
-				} else {
-					*files = append(*files, addon)
-				}
-			}
-		} else if err != nil {
-			return nil
-		}
+// client sets and returns a Kubernetes client to use to speak to a kubeadm launched apiserver
+func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) {
+	if k.k8sClient != nil {
+		return k.k8sClient, nil
 	}
 
-	return nil
-}
-
-// client returns a Kubernetes client to use to speak to a kubeadm launched apiserver
-func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientset, error) {
 	config, err := kapi.ClientConfig(k.contextName)
 	if err != nil {
 		return nil, errors.Wrap(err, "client config")
@@ -306,108 +223,30 @@ func (k *Bootstrapper) client(k8s config.KubernetesConfig) (*kubernetes.Clientse
 		glog.Errorf("Overriding stale ClientConfig host %s with %s", config.Host, endpoint)
 		config.Host = endpoint
 	}
-
-	return kubernetes.NewForConfig(config)
-}
-
-func (k *Bootstrapper) waitForAPIServerProcess(start time.Time, timeout time.Duration) error {
-	glog.Infof("waiting for apiserver process to appear ...")
-	err := wait.PollImmediate(time.Second*1, timeout, func() (bool, error) {
-		if time.Since(start) > timeout {
-			return false, fmt.Errorf("cluster wait timed out during process check")
-		}
-		rr, ierr := k.c.RunCmd(exec.Command("sudo", "pgrep", "kube-apiserver"))
-		if ierr != nil {
-			glog.Warningf("pgrep apiserver: %v cmd: %s", ierr, rr.Command())
-			return false, nil
-		}
-		return true, nil
-	})
-	if err != nil {
-		return fmt.Errorf("apiserver process never appeared")
-	}
-	glog.Infof("duration metric: took %s to wait for apiserver process to appear ...", time.Since(start))
-	return nil
-}
-
-func (k *Bootstrapper) waitForAPIServerHealthz(start time.Time, k8s config.KubernetesConfig, timeout time.Duration) error {
-	glog.Infof("waiting for apiserver healthz status ...")
-	hStart := time.Now()
-	healthz := func() (bool, error) {
-		if time.Since(start) > timeout {
-			return false, fmt.Errorf("cluster wait timed out during healthz check")
-		}
-
-		status, err := k.GetAPIServerStatus(net.ParseIP(k8s.NodeIP), k8s.NodePort)
-		if err != nil {
-			glog.Warningf("status: %v", err)
-			return false, nil
-		}
-		if status != "Running" {
-			return false, nil
-		}
-		return true, nil
-	}
-
-	if err := wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, healthz); err != nil {
-		return fmt.Errorf("apiserver healthz never reported healthy")
-	}
-	glog.Infof("duration metric: took %s to wait for apiserver healthz status ...", time.Since(hStart))
-	return nil
-}
-
-func (k *Bootstrapper) waitForSystemPods(start time.Time, k8s config.KubernetesConfig, timeout time.Duration) error {
-	glog.Infof("waiting for kube-system pods to appear ...")
-	pStart := time.Now()
-	client, err := k.client(k8s)
-	if err != nil {
-		return errors.Wrap(err, "client")
-	}
-
-	podStart := time.Time{}
-	podList := func() (bool, error) {
-		if time.Since(start) > timeout {
-			return false, fmt.Errorf("cluster wait timed out during pod check")
-		}
-		// Wait for any system pod, as waiting for apiserver may block until etcd
-		pods, err := client.CoreV1().Pods("kube-system").List(meta.ListOptions{})
-		if len(pods.Items) < 2 {
-			podStart = time.Time{}
-			return false, nil
-		}
-		if err != nil {
-			podStart = time.Time{}
-			return false, nil
-		}
-		if podStart.IsZero() {
-			podStart = time.Now()
-		}
-
-		glog.Infof("%d kube-system pods found since %s", len(pods.Items), podStart)
-		if time.Since(podStart) > 2*kconst.APICallRetryInterval {
-			glog.Infof("stability requirement met, returning")
-			return true, nil
-		}
-		return false, nil
-	}
-	if err = wait.PollImmediate(kconst.APICallRetryInterval, kconst.DefaultControlPlaneTimeout, podList); err != nil {
-		return fmt.Errorf("apiserver never returned a pod list")
+	c, err := kubernetes.NewForConfig(config)
+	if err == nil {
+		k.k8sClient = c
 	}
-	glog.Infof("duration metric: took %s to wait for pod list to return data ...", time.Since(pStart))
-	return nil
+	return c, err
 }
 
 // WaitForCluster blocks until the cluster appears to be healthy
 func (k *Bootstrapper) WaitForCluster(k8s config.KubernetesConfig, timeout time.Duration) error {
 	start := time.Now()
 	out.T(out.Waiting, "Waiting for cluster to come online ...")
-	if err := k.waitForAPIServerProcess(start, timeout); err != nil {
+	if err := kverify.APIServerProcess(k.c, start, timeout); err != nil {
 		return err
 	}
-	if err := k.waitForAPIServerHealthz(start, k8s, timeout); err != nil {
+	if err := kverify.APIServerIsRunning(start, k8s.NodeIP, k8s.NodePort, timeout); err != nil {
 		return err
 	}
-	return k.waitForSystemPods(start, k8s, timeout)
+
+	c, err := k.client(k8s)
+	if err != nil {
+		return errors.Wrap(err, "get k8s client")
+	}
+
+	return kverify.SystemPods(c, start, k8s.NodeIP, k8s.NodePort, timeout)
 }
 
 // restartCluster restarts the Kubernetes cluster configured by kubeadm
@@ -452,10 +291,16 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 	}
 
 	// We must ensure that the apiserver is healthy before proceeding
-	if err := k.waitForAPIServerHealthz(time.Now(), k8s, kconst.DefaultControlPlaneTimeout); err != nil {
+	if err := kverify.APIServerProcess(k.c, time.Now(), kconst.DefaultControlPlaneTimeout); err != nil {
 		return errors.Wrap(err, "apiserver healthz")
 	}
-	if err := k.waitForSystemPods(time.Now(), k8s, kconst.DefaultControlPlaneTimeout); err != nil {
+
+	client, err := k.client(k8s)
+	if err != nil {
+		return errors.Wrap(err, "getting k8s client")
+	}
+
+	if err := kverify.SystemPods(client, time.Now(), k8s.NodeIP, k8s.NodePort, kconst.DefaultControlPlaneTimeout); err != nil {
 		return errors.Wrap(err, "system pods")
 	}
 
@@ -464,7 +309,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 		return errors.Wrapf(err, fmt.Sprintf("addon phase cmd:%q", rr.Command()))
 	}
 
-	if err := k.adjustResourceLimits(); err != nil {
+	if err := bsutil.AdjustResourceLimits(k.c); err != nil {
 		glog.Warningf("unable to adjust resource limits: %v", err)
 	}
 	return nil
@@ -560,7 +405,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	}
 	files := bsutil.ConfigFileAssets(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService, cniFile)
 
-	if err := addAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil {
+	if err := bsutil.AddAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil {
 		return errors.Wrap(err, "adding addons")
 	}
 	for _, f := range files {

pkg/minikube/cluster/cluster.go

@@ -52,7 +52,6 @@ import (
 	"k8s.io/minikube/pkg/minikube/localpath"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/registry"
-	pkgutil "k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
@@ -343,7 +342,7 @@ func GetHostDriverIP(api libmachine.API, machineName string) (net.IP, error) {
 func engineOptions(config cfg.MachineConfig) *engine.Options {
 	o := engine.Options{
 		Env:              config.DockerEnv,
-		InsecureRegistry: append([]string{pkgutil.DefaultServiceCIDR}, config.InsecureRegistry...),
+		InsecureRegistry: append([]string{constants.DefaultServiceCIDR}, config.InsecureRegistry...),
 		RegistryMirror:   config.RegistryMirror,
 		ArbitraryFlags:   config.DockerOpt,
 		InstallURL:       drivers.DefaultEngineInstallURL,

pkg/minikube/command/kic_runner.go

@@ -23,6 +23,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
+	"path"
 	"time"
 
 	"github.com/golang/glog"
@@ -159,9 +160,9 @@ func (k *kicRunner) Copy(f assets.CopyableFile) error {
 	if out, err := exec.Command(k.ociBin, "cp", assetName, destination).CombinedOutput(); err != nil {
 		return errors.Wrapf(err, "error copying %s into node, output: %s", f.GetAssetName(), string(out))
 	}
-
-	if _, err := k.RunCmd(exec.Command("chmod", f.GetPermissions(), f.GetTargetDir())); err != nil {
-		return errors.Wrap(err, "failed to chmod file permissions")
+	fp := path.Join(f.GetTargetDir(), f.GetTargetName())
+	if _, err := k.RunCmd(exec.Command("sudo", "chmod", f.GetPermissions(), fp)); err != nil {
+		return errors.Wrapf(err, "failed to chmod file permissions %s", fp)
 	}
 	return nil
 }