Merge pull request #8468 from prasadkatti/mv_kubelet_params_to_cfg_file

Move kubelet params to config file

pkg/minikube/bootstrapper/bsutil/ktmpl/v1alpha3.go

@@ -68,6 +68,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: {{.CgroupDriver}}
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -76,4 +77,5 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: {{.StaticPodPath}}
 `))

pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta1.go

@@ -79,6 +79,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: {{.ClientCAFile}}
+cgroupDriver: {{.CgroupDriver}}
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -87,6 +91,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: {{.StaticPodPath}}
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/ktmpl/v1beta2.go

@@ -83,6 +83,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: {{.ClientCAFile}}
+cgroupDriver: {{.CgroupDriver}}
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -91,6 +95,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: {{.StaticPodPath}}
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/kubeadm.go

@@ -61,6 +61,11 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 		nodePort = constants.APIServerPort
 	}
 
+	cgroupDriver, err := r.CGroupDriver()
+	if err != nil {
+		return nil, errors.Wrap(err, "getting cgroup driver")
+	}
+
 	componentOpts, err := createExtraComponentConfig(k8s.ExtraOptions, version, componentFeatureArgs, cp)
 	if err != nil {
 		return nil, errors.Wrap(err, "generating extra component config for kubeadm")
@@ -96,6 +101,9 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 		FeatureArgs         map[string]bool
 		NoTaintMaster       bool
 		NodeIP              string
+		CgroupDriver        string
+		ClientCAFile        string
+		StaticPodPath       string
 		ControlPlaneAddress string
 		KubeProxyOptions    map[string]string
 	}{
@@ -117,6 +125,9 @@ func GenerateKubeadmYAML(cc config.ClusterConfig, n config.Node, r cruntime.Mana
 		NoTaintMaster:       false, // That does not work with k8s 1.12+
 		DNSDomain:           k8s.DNSDomain,
 		NodeIP:              n.IP,
+		CgroupDriver:        cgroupDriver,
+		ClientCAFile:        path.Join(vmpath.GuestKubernetesCertsDir, "ca.crt"),
+		StaticPodPath:       vmpath.GuestManifestsDir,
 		ControlPlaneAddress: constants.ControlPlaneAlias,
 		KubeProxyOptions:    createKubeProxyOptions(k8s.ExtraOptions),
 	}

pkg/minikube/bootstrapper/bsutil/kubeadm_test.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/google/go-cmp/cmp"
 	"github.com/pmezard/go-difflib/difflib"
+	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@@ -108,6 +109,10 @@ nnetworking/dnsDomain value
 */
 func TestGenerateKubeadmYAMLDNS(t *testing.T) {
 	versions := []string{"v1.19", "v1.18", "v1.17", "v1.16", "v1.15", "v1.14", "v1.13", "v1.12"}
+	fcr := command.NewFakeCommandRunner()
+	fcr.SetCommandToOutput(map[string]string{
+		"docker info --format {{.CgroupDriver}}": "systemd\n",
+	})
 	tests := []struct {
 		name      string
 		runtime   string
@@ -118,7 +123,7 @@ func TestGenerateKubeadmYAMLDNS(t *testing.T) {
 	}
 	for _, version := range versions {
 		for _, tc := range tests {
-			runtime, err := cruntime.New(cruntime.Config{Type: tc.runtime})
+			runtime, err := cruntime.New(cruntime.Config{Type: tc.runtime, Runner: fcr})
 			if err != nil {
 				t.Fatalf("runtime: %v", err)
 			}
@@ -174,6 +179,12 @@ func TestGenerateKubeadmYAML(t *testing.T) {
 	if err != nil {
 		t.Errorf("versions: %v", err)
 	}
+	fcr := command.NewFakeCommandRunner()
+	fcr.SetCommandToOutput(map[string]string{
+		"docker info --format {{.CgroupDriver}}": "systemd\n",
+		"crio config":                            "cgroup_manager = \"systemd\"\n",
+		"sudo crictl info":                       "{\"config\": {\"systemdCgroup\": true}}",
+	})
 	tests := []struct {
 		name      string
 		runtime   string
@@ -192,7 +203,7 @@ func TestGenerateKubeadmYAML(t *testing.T) {
 	}
 	for _, version := range versions {
 		for _, tc := range tests {
-			runtime, err := cruntime.New(cruntime.Config{Type: tc.runtime})
+			runtime, err := cruntime.New(cruntime.Config{Type: tc.runtime, Runner: fcr})
 			if err != nil {
 				t.Fatalf("runtime: %v", err)
 			}

pkg/minikube/bootstrapper/bsutil/kubelet.go

@@ -44,11 +44,6 @@ func extraKubeletOpts(mc config.ClusterConfig, nc config.Node, r cruntime.Manage
 		return nil, errors.Wrap(err, "generating extra configuration for kubelet")
 	}
 
-	cgroupDriver, err := r.CGroupDriver()
-	if err == nil {
-		extraOpts["cgroup-driver"] = cgroupDriver
-	}
-
 	for k, v := range r.KubeletOptions() {
 		extraOpts[k] = v
 	}

pkg/minikube/bootstrapper/bsutil/kubelet_test.go

@@ -21,7 +21,6 @@ import (
 	"testing"
 
 	"github.com/pmezard/go-difflib/difflib"
-	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
 	"k8s.io/minikube/pkg/minikube/constants"
 	"k8s.io/minikube/pkg/minikube/cruntime"
@@ -176,8 +175,7 @@ ExecStart=/var/lib/minikube/binaries/v1.18.2/kubelet --authorization-mode=Webhoo
 
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			runtime, err := cruntime.New(cruntime.Config{Type: tc.cfg.KubernetesConfig.ContainerRuntime,
-				Runner: command.NewFakeCommandRunner()})
+			runtime, err := cruntime.New(cruntime.Config{Type: tc.cfg.KubernetesConfig.ContainerRuntime})
 			if err != nil {
 				t.Fatalf("runtime: %v", err)
 			}

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-api-port.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd-pod-network-cidr.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/containerd.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio-options-gates.yaml

@@ -48,6 +48,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -56,3 +57,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/crio.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/default.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/dns.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/image-repository.yaml

@@ -41,6 +41,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -49,3 +50,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.12/options.yaml

@@ -45,6 +45,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,3 +54,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-api-port.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd-pod-network-cidr.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/containerd.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio-options-gates.yaml

@@ -48,6 +48,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -56,3 +57,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/crio.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/default.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/dns.yaml

@@ -40,6 +40,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -48,3 +49,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/image-repository.yaml

@@ -41,6 +41,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -49,3 +50,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.13/options.yaml

@@ -45,6 +45,7 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,3 +54,4 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-api-port.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd-pod-network-cidr.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/containerd.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio-options-gates.yaml

@@ -57,6 +57,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -65,6 +69,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/crio.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/default.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/dns.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/image-repository.yaml

@@ -48,6 +48,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -56,6 +60,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.14/options.yaml

@@ -54,6 +54,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -62,6 +66,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-api-port.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd-pod-network-cidr.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/containerd.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio-options-gates.yaml

@@ -57,6 +57,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -65,6 +69,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/crio.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/default.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/dns.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/image-repository.yaml

@@ -48,6 +48,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -56,6 +60,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.15/options.yaml

@@ -54,6 +54,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -62,6 +66,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-api-port.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd-pod-network-cidr.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/containerd.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio-options-gates.yaml

@@ -57,6 +57,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -65,6 +69,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/crio.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/default.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/dns.yaml

@@ -47,6 +47,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -55,6 +59,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/image-repository.yaml

@@ -48,6 +48,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -56,6 +60,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.16/options.yaml

@@ -54,6 +54,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -62,6 +66,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-api-port.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd-pod-network-cidr.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/containerd.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio-options-gates.yaml

@@ -55,6 +55,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -63,6 +67,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/crio.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/default.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/dns.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/image-repository.yaml

@@ -46,6 +46,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -54,6 +58,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.17/options.yaml

@@ -52,6 +52,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -60,6 +64,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd-api-port.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd-pod-network-cidr.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.18/containerd.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio-options-gates.yaml

@@ -55,6 +55,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -63,6 +67,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.18/crio.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.18/default.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.18/dns.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.18/image-repository.yaml

@@ -46,6 +46,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -54,6 +58,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.18/options.yaml

@@ -52,6 +52,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -60,6 +64,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd-api-port.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd-pod-network-cidr.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.19/containerd.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio-options-gates.yaml

@@ -55,6 +55,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -63,6 +67,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.19/crio.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.19/default.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.19/dns.yaml

@@ -45,6 +45,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -53,6 +57,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.19/image-repository.yaml

@@ -46,6 +46,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -54,6 +58,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/testdata/v1.19/options.yaml

@@ -52,6 +52,10 @@ networking:
 ---
 apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
+authentication:
+  x509:
+    clientCAFile: /var/lib/minikube/certs/ca.crt
+cgroupDriver: systemd
 clusterDomain: "cluster.local"
 # disable disk resource management by default
 imageGCHighThresholdPercent: 100
@@ -60,6 +64,7 @@ evictionHard:
   nodefs.inodesFree: "0%"
   imagefs.available: "0%"
 failSwapOn: false
+staticPodPath: /etc/kubernetes/manifests
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration

pkg/minikube/bootstrapper/bsutil/versions.go

@@ -40,9 +40,9 @@ func versionIsBetween(version, gte, lte semver.Version) bool {
 
 var versionSpecificOpts = []config.VersionedExtraOption{
 
-	// Kubeconfig args
-	config.NewUnversionedOption(Kubelet, "kubeconfig", "/etc/kubernetes/kubelet.conf"),
 	config.NewUnversionedOption(Kubelet, "bootstrap-kubeconfig", "/etc/kubernetes/bootstrap-kubelet.conf"),
+	config.NewUnversionedOption(Kubelet, "config", "/var/lib/kubelet/config.yaml"),
+	config.NewUnversionedOption(Kubelet, "kubeconfig", "/etc/kubernetes/kubelet.conf"),
 	{
 		Option: config.ExtraOption{
 			Component: Kubelet,
@@ -52,8 +52,6 @@ var versionSpecificOpts = []config.VersionedExtraOption{
 		LessThanOrEqual: semver.MustParse("1.9.10"),
 	},
 
-	// System pods args
-	config.NewUnversionedOption(Kubelet, "pod-manifest-path", vmpath.GuestManifestsDir),
 	{
 		Option: config.ExtraOption{
 			Component: Kubelet,
@@ -63,15 +61,17 @@ var versionSpecificOpts = []config.VersionedExtraOption{
 		LessThanOrEqual: semver.MustParse("1.15.0-alpha.3"),
 	},
 
-	// Kubelet config file
-	config.NewUnversionedOption(Kubelet, "config", "/var/lib/kubelet/config.yaml"),
-
-	// Auth args
-	config.NewUnversionedOption(Kubelet, "authorization-mode", "Webhook"),
-	config.NewUnversionedOption(Kubelet, "client-ca-file", path.Join(vmpath.GuestKubernetesCertsDir, "ca.crt")),
+	// before 1.16.0-beta.2, kubeadm bug did not allow overriding this via config file, so this has
+	// to be passed in as a kubelet flag. See https://github.com/kubernetes/kubernetes/pull/81903 for more details.
+	{
+		Option: config.ExtraOption{
+			Component: Kubelet,
+			Key:       "client-ca-file",
+			Value:     path.Join(vmpath.GuestKubernetesCertsDir, "ca.crt"),
+		},
+		LessThanOrEqual: semver.MustParse("1.16.0-beta.1"),
+	},
 
-	// Cgroup args
-	config.NewUnversionedOption(Kubelet, "cgroup-driver", "cgroupfs"),
 	{
 		Option: config.ExtraOption{
 			Component: Apiserver,