move reusabe parts of kubeadm to a package

cmd/minikube/cmd/start.go

@@ -47,8 +47,8 @@ import (
 	"golang.org/x/sync/errgroup"
 	cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm"
 	"k8s.io/minikube/pkg/minikube/cluster"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -186,7 +186,7 @@ func initKubernetesFlags() {
 		`A set of key=value pairs that describe configuration that may be passed to different components.
 		The key should be '.' separated, and the first part before the dot is the component to apply the configuration to.
 		Valid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler
-		Valid kubeadm parameters: `+fmt.Sprintf("%s, %s", strings.Join(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], ", "), strings.Join(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], ",")))
+		Valid kubeadm parameters: `+fmt.Sprintf("%s, %s", strings.Join(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], ", "), strings.Join(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], ",")))
 	startCmd.Flags().String(featureGates, "", "A set of key=value pairs that describe feature gates for alpha/experimental features.")
 	startCmd.Flags().String(dnsDomain, constants.ClusterDNSDomain, "The cluster dns domain name used in the kubernetes cluster")
 	startCmd.Flags().Int(apiServerPort, constants.APIServerPort, "The apiserver listening port")
@@ -789,9 +789,9 @@ func validateFlags(cmd *cobra.Command, drvName string) {
 	}
 
 	// check that kubeadm extra args contain only whitelisted parameters
-	for param := range extraOptions.AsMap().Get(kubeadm.Kubeadm) {
-		if !cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) &&
-			!cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) {
+	for param := range extraOptions.AsMap().Get(bsutil.Kubeadm) {
+		if !cfg.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], param) &&
+			!cfg.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], param) {
 			exit.UsageT("Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config", out.V{"parameter_name": param})
 		}
 	}

pkg/minikube/bootstrapper/bsutil/binaries.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+import (
+	"path"
+	"runtime"
+
+	"github.com/pkg/errors"
+	"golang.org/x/sync/errgroup"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/machine"
+)
+
+// TransferBinaries transfers all required Kubernetes binaries
+func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
+	var g errgroup.Group
+	for _, name := range constants.KubeadmBinaries {
+		name := name
+		g.Go(func() error {
+			src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
+			if err != nil {
+				return errors.Wrapf(err, "downloading %s", name)
+			}
+
+			dst := path.Join(binRoot(cfg.KubernetesVersion), name)
+			if err := machine.CopyBinary(c, src, dst); err != nil {
+				return errors.Wrapf(err, "copybinary %s -> %s", src, dst)
+			}
+			return nil
+		})
+	}
+	return g.Wait()
+}

pkg/minikube/bootstrapper/bsutil/bsutil.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+import (
+	"bytes"
+	"fmt"
+	"path"
+
+	"github.com/blang/semver"
+	"github.com/golang/glog"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/cruntime"
+	"k8s.io/minikube/pkg/minikube/vmpath"
+	"k8s.io/minikube/pkg/util"
+)
+
+// Container runtimes
+const remoteContainerRuntime = "remote"
+
+// GenerateKubeadmYAML generates the kubeadm.yaml file
+func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
+	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing kubernetes version")
+	}
+
+	// parses a map of the feature gates for kubeadm and component
+	kubeadmFeatureArgs, componentFeatureArgs, err := parseFeatureArgs(k8s.FeatureGates)
+	if err != nil {
+		return nil, errors.Wrap(err, "parses feature gate config for kubeadm and component")
+	}
+
+	extraComponentConfig, err := createExtraComponentConfig(k8s.ExtraOptions, version, componentFeatureArgs)
+	if err != nil {
+		return nil, errors.Wrap(err, "generating extra component config for kubeadm")
+	}
+
+	// In case of no port assigned, use util.APIServerPort
+	nodePort := k8s.NodePort
+	if nodePort <= 0 {
+		nodePort = constants.APIServerPort
+	}
+
+	opts := struct {
+		CertDir           string
+		ServiceCIDR       string
+		PodSubnet         string
+		AdvertiseAddress  string
+		APIServerPort     int
+		KubernetesVersion string
+		EtcdDataDir       string
+		NodeName          string
+		DNSDomain         string
+		CRISocket         string
+		ImageRepository   string
+		ExtraArgs         []ComponentExtraArgs
+		FeatureArgs       map[string]bool
+		NoTaintMaster     bool
+	}{
+		CertDir:           vmpath.GuestCertsDir,
+		ServiceCIDR:       util.DefaultServiceCIDR,
+		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
+		AdvertiseAddress:  k8s.NodeIP,
+		APIServerPort:     nodePort,
+		KubernetesVersion: k8s.KubernetesVersion,
+		EtcdDataDir:       etcdDataDir(),
+		NodeName:          k8s.NodeName,
+		CRISocket:         r.SocketPath(),
+		ImageRepository:   k8s.ImageRepository,
+		ExtraArgs:         extraComponentConfig,
+		FeatureArgs:       kubeadmFeatureArgs,
+		NoTaintMaster:     false, // That does not work with k8s 1.12+
+		DNSDomain:         k8s.DNSDomain,
+	}
+
+	if k8s.ServiceCIDR != "" {
+		opts.ServiceCIDR = k8s.ServiceCIDR
+	}
+
+	opts.NoTaintMaster = true
+	b := bytes.Buffer{}
+	configTmpl := ConfigTmplV1Alpha1
+	if version.GTE(semver.MustParse("1.12.0")) {
+		configTmpl = ConfigTmplV1Alpha3
+	}
+	// v1beta1 works in v1.13, but isn't required until v1.14.
+	if version.GTE(semver.MustParse("1.14.0-alpha.0")) {
+		configTmpl = ConfigTmplV1Beta1
+	}
+	if err := configTmpl.Execute(&b, opts); err != nil {
+		return nil, err
+	}
+
+	return b.Bytes(), nil
+}
+
+// NewKubeletConfig generates a new systemd unit containing a configured kubelet
+// based on the options present in the KubernetesConfig.
+func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
+	version, err := ParseKubernetesVersion(k8s.KubernetesVersion)
+	if err != nil {
+		return nil, errors.Wrap(err, "parsing kubernetes version")
+	}
+
+	extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version)
+	if err != nil {
+		return nil, errors.Wrap(err, "generating extra configuration for kubelet")
+	}
+
+	for k, v := range r.KubeletOptions() {
+		extraOpts[k] = v
+	}
+	if k8s.NetworkPlugin != "" {
+		extraOpts["network-plugin"] = k8s.NetworkPlugin
+	}
+	if _, ok := extraOpts["node-ip"]; !ok {
+		extraOpts["node-ip"] = k8s.NodeIP
+	}
+
+	pauseImage := images.Pause(k8s.ImageRepository)
+	if _, ok := extraOpts["pod-infra-container-image"]; !ok && k8s.ImageRepository != "" && pauseImage != "" && k8s.ContainerRuntime != remoteContainerRuntime {
+		extraOpts["pod-infra-container-image"] = pauseImage
+	}
+
+	// parses a map of the feature gates for kubelet
+	_, kubeletFeatureArgs, err := parseFeatureArgs(k8s.FeatureGates)
+	if err != nil {
+		return nil, errors.Wrap(err, "parses feature gate config for kubelet")
+	}
+
+	if kubeletFeatureArgs != "" {
+		extraOpts["feature-gates"] = kubeletFeatureArgs
+	}
+
+	b := bytes.Buffer{}
+	opts := struct {
+		ExtraOptions     string
+		ContainerRuntime string
+		KubeletPath      string
+	}{
+		ExtraOptions:     convertToFlags(extraOpts),
+		ContainerRuntime: k8s.ContainerRuntime,
+		KubeletPath:      path.Join(binRoot(k8s.KubernetesVersion), "kubelet"),
+	}
+	if err := KubeletSystemdTemplate.Execute(&b, opts); err != nil {
+		return nil, err
+	}
+
+	return b.Bytes(), nil
+}
+
+// NewKubeletService returns a generated systemd unit file for the kubelet
+func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) {
+	var b bytes.Buffer
+	opts := struct{ KubeletPath string }{KubeletPath: path.Join(binRoot(cfg.KubernetesVersion), "kubelet")}
+	if err := kubeletServiceTemplate.Execute(&b, opts); err != nil {
+		return nil, errors.Wrap(err, "template execute")
+	}
+	return b.Bytes(), nil
+}
+
+// These are the components that can be configured
+// through the "extra-config"
+const (
+	Kubelet           = "kubelet"
+	Kubeadm           = "kubeadm"
+	Apiserver         = "apiserver"
+	Scheduler         = "scheduler"
+	ControllerManager = "controller-manager"
+)
+
+// ExtraConfigForComponent generates a map of flagname-value pairs for a k8s
+// component.
+func ExtraConfigForComponent(component string, opts config.ExtraOptionSlice, version semver.Version) (map[string]string, error) {
+	versionedOpts, err := DefaultOptionsForComponentAndVersion(component, version)
+	if err != nil {
+		return nil, errors.Wrapf(err, "setting version specific options for %s", component)
+	}
+
+	for _, opt := range opts {
+		if opt.Component == component {
+			if val, ok := versionedOpts[opt.Key]; ok {
+				glog.Infof("Overwriting default %s=%s with user provided %s=%s for component %s", opt.Key, val, opt.Key, opt.Value, component)
+			}
+			versionedOpts[opt.Key] = opt.Value
+		}
+	}
+
+	return versionedOpts, nil
+}
+
+// binRoot returns the persistent path binaries are stored in
+func binRoot(version string) string {
+	return path.Join(vmpath.GuestPersistentDir, "binaries", version)
+}
+
+// InvokeKubeadm returns the invocation command for Kubeadm
+func InvokeKubeadm(version string) string {
+	return fmt.Sprintf("sudo env PATH=%s:$PATH kubeadm", binRoot(version))
+}
+
+// etcdDataDir is where etcd data is stored.
+func etcdDataDir() string {
+	return path.Join(vmpath.GuestPersistentDir, "etcd")
+}

pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go → pkg/minikube/bootstrapper/bsutil/bsutil_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubeadm
+package bsutil
 
 import (
 	"fmt"
@@ -241,7 +241,7 @@ as v1.11 yaml file is very different compared to v1.12+.
 This test case has only 1 thing to test and that is the
 nnetworking/dnsDomain value
 */
-func TestGenerateConfigDNS(t *testing.T) {
+func TestGenerateKubeadmYAMLDNS(t *testing.T) {
 	versions := []string{"v1.16", "v1.15", "v1.14", "v1.13", "v1.12"}
 	tests := []struct {
 		name      string
@@ -264,7 +264,7 @@ func TestGenerateConfigDNS(t *testing.T) {
 				cfg.NodeName = "mk"
 				cfg.KubernetesVersion = version + ".0"
 
-				got, err := generateConfig(cfg, runtime)
+				got, err := GenerateKubeadmYAML(cfg, runtime)
 				if err != nil && !tc.shouldErr {
 					t.Fatalf("got unexpected error generating config: %v", err)
 				}
@@ -296,7 +296,7 @@ func TestGenerateConfigDNS(t *testing.T) {
 	}
 }
 
-func TestGenerateConfig(t *testing.T) {
+func TestGenerateKubeadmYAML(t *testing.T) {
 	extraOpts := getExtraOpts()
 	extraOptsPodCidr := getExtraOptsPodCidr()
 	versions, err := recentReleases()
@@ -332,7 +332,7 @@ func TestGenerateConfig(t *testing.T) {
 				cfg.NodeName = "mk"
 				cfg.KubernetesVersion = version + ".0"
 
-				got, err := generateConfig(cfg, runtime)
+				got, err := GenerateKubeadmYAML(cfg, runtime)
 				if err != nil && !tc.shouldErr {
 					t.Fatalf("got unexpected error generating config: %v", err)
 				}

pkg/minikube/bootstrapper/bsutil/extraconfig.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil
+
+import (
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/blang/semver"
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/config"
+)
+
+const (
+	KubeadmCmdParam    = iota
+	KubeadmConfigParam = iota
+)
+
+// ComponentExtraArgs holds extra args for a component
+type ComponentExtraArgs struct {
+	Component string
+	Options   map[string]string
+}
+
+// mapping of component to the section name in kubeadm.
+var componentToKubeadmConfigKey = map[string]string{
+	Apiserver:         "apiServer",
+	ControllerManager: "controllerManager",
+	Scheduler:         "scheduler",
+	Kubeadm:           "kubeadm",
+	// The Kubelet is not configured in kubeadm, only in systemd.
+	Kubelet: "",
+}
+
+// KubeadmExtraArgsWhitelist is a whitelist of supported kubeadm params that can be supplied to kubeadm through
+// minikube's ExtraArgs parameter. The list is split into two parts - params that can be supplied as flags on the
+// command line and params that have to be inserted into the kubeadm config file. This is because of a kubeadm
+// constraint which allows only certain params to be provided from the command line when the --config parameter
+// is specified
+var KubeadmExtraArgsWhitelist = map[int][]string{
+	KubeadmCmdParam: {
+		"ignore-preflight-errors",
+		"dry-run",
+		"kubeconfig",
+		"kubeconfig-dir",
+		"node-name",
+		"cri-socket",
+		"experimental-upload-certs",
+		"certificate-key",
+		"rootfs",
+	},
+	KubeadmConfigParam: {
+		"pod-network-cidr",
+	},
+}
+
+// NewComponentExtraArgs creates a new ComponentExtraArgs
+func NewComponentExtraArgs(opts config.ExtraOptionSlice, version semver.Version, featureGates string) ([]ComponentExtraArgs, error) {
+	var kubeadmExtraArgs []ComponentExtraArgs
+	for _, extraOpt := range opts {
+		if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok {
+			return nil, fmt.Errorf("unknown component %q. valid components are: %v", componentToKubeadmConfigKey, componentToKubeadmConfigKey)
+		}
+	}
+
+	keys := []string{}
+	for k := range componentToKubeadmConfigKey {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	for _, component := range keys {
+		kubeadmComponentKey := componentToKubeadmConfigKey[component]
+		if kubeadmComponentKey == "" {
+			continue
+		}
+		extraConfig, err := ExtraConfigForComponent(component, opts, version)
+		if err != nil {
+			return nil, errors.Wrapf(err, "getting kubeadm extra args for %s", component)
+		}
+		if featureGates != "" {
+			extraConfig["feature-gates"] = featureGates
+		}
+		if len(extraConfig) > 0 {
+			kubeadmExtraArgs = append(kubeadmExtraArgs, ComponentExtraArgs{
+				Component: kubeadmComponentKey,
+				Options:   extraConfig,
+			})
+		}
+	}
+
+	return kubeadmExtraArgs, nil
+}
+
+// CreateFlagsFromExtraArgs converts kubeadm extra args into flags to be supplied from the command linne
+func CreateFlagsFromExtraArgs(extraOptions config.ExtraOptionSlice) string {
+	kubeadmExtraOpts := extraOptions.AsMap().Get(Kubeadm)
+
+	// kubeadm allows only a small set of parameters to be supplied from the command line when the --config param
+	// is specified, here we remove those that are not allowed
+	for opt := range kubeadmExtraOpts {
+		if !config.ContainsParam(KubeadmExtraArgsWhitelist[KubeadmCmdParam], opt) {
+			// kubeadmExtraOpts is a copy so safe to delete
+			delete(kubeadmExtraOpts, opt)
+		}
+	}
+	return convertToFlags(kubeadmExtraOpts)
+}
+
+// createExtraComponentConfig generates a map of component to extra args for all of the components except kubeadm
+func createExtraComponentConfig(extraOptions config.ExtraOptionSlice, version semver.Version, componentFeatureArgs string) ([]ComponentExtraArgs, error) {
+	extraArgsSlice, err := NewComponentExtraArgs(extraOptions, version, componentFeatureArgs)
+	if err != nil {
+		return nil, err
+	}
+
+	// kubeadm extra args should not be included in the kubeadm config in the extra args section (instead, they must
+	// be inserted explicitly in the appropriate places or supplied from the command line); here we remove all of the
+	// kubeadm extra args from the slice
+	for i, extraArgs := range extraArgsSlice {
+		if extraArgs.Component == Kubeadm {
+			extraArgsSlice = append(extraArgsSlice[:i], extraArgsSlice[i+1:]...)
+			break
+		}
+	}
+	return extraArgsSlice, nil
+}
+
+func convertToFlags(opts map[string]string) string {
+	var flags []string
+	var keys []string
+	for k := range opts {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		flags = append(flags, fmt.Sprintf("--%s=%s", k, opts[k]))
+	}
+	return strings.Join(flags, " ")
+}

pkg/minikube/bootstrapper/bsutil/featuregates.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// bsutil package will eventually be renamed to kubeadm package after getting rid of older one
+package bsutil

pkg/minikube/bootstrapper/kubeadm/templates.go → pkg/minikube/bootstrapper/bsutil/templates.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubeadm
+package bsutil
 
 import (
 	"fmt"
@@ -22,8 +22,8 @@ import (
 	"text/template"
 )
 
-// configTmplV1Alpha1 is for Kubernetes v1.11
-var configTmplV1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{
+// ConfigTmplV1Alpha1 is for Kubernetes v1.11
+var ConfigTmplV1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{
 	"printMapInOrder": printMapInOrder,
 }).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1
 kind: MasterConfiguration
@@ -47,8 +47,8 @@ nodeName: {{.NodeName}}
   {{$i}}: {{$val}}{{end}}
 {{end}}`))
 
-// configTmplV1Alpha3 is for Kubernetes v1.12
-var configTmplV1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{
+// ConfigTmplV1Alpha3 is for Kubernetes v1.12
+var ConfigTmplV1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{
 	"printMapInOrder": printMapInOrder,
 }).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3
 kind: InitConfiguration
@@ -96,8 +96,8 @@ evictionHard:
   imagefs.available: "0%"
 `))
 
-// configTmplV1Beta1 is for Kubernetes v1.13+
-var configTmplV1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{
+// ConfigTmplV1Beta1 is for Kubernetes v1.13+
+var ConfigTmplV1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{
 	"printMapInOrder": printMapInOrder,
 }).Parse(`apiVersion: kubeadm.k8s.io/v1beta1
 kind: InitConfiguration
@@ -151,8 +151,8 @@ evictionHard:
   imagefs.available: "0%"
 `))
 
-// kubeletSystemdTemplate hosts the override kubelet flags, written to kubeletSystemdConfFile
-var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`[Unit]
+// KubeletSystemdTemplate hosts the override kubelet flags, written to kubeletSystemdConfFile
+var KubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`[Unit]
 {{if or (eq .ContainerRuntime "cri-o") (eq .ContainerRuntime "cri")}}Wants=crio.service{{else if eq .ContainerRuntime "containerd"}}Wants=containerd.service{{else}}Wants=docker.socket{{end}}
 
 [Service]

pkg/minikube/bootstrapper/bsutil/templates/templates.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package templates
+
+import "fmt"
+
+func Dum() {
+	fmt.Println("hello")
+}

pkg/minikube/bootstrapper/kubeadm/templates_test.go → pkg/minikube/bootstrapper/bsutil/templates_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubeadm
+package bsutil
 
 import (
 	"reflect"

pkg/minikube/bootstrapper/kubeadm/versions.go → pkg/minikube/bootstrapper/bsutil/versions.go

@@ -14,17 +14,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubeadm
+package bsutil
 
 import (
 	"fmt"
 	"path"
-	"sort"
 	"strconv"
 	"strings"
 
 	"github.com/blang/semver"
-	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"k8s.io/kubernetes/cmd/kubeadm/app/features"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -33,92 +31,19 @@ import (
 	"k8s.io/minikube/pkg/util"
 )
 
-// These are the components that can be configured
-// through the "extra-config"
-const (
-	Kubelet           = "kubelet"
-	Kubeadm           = "kubeadm"
-	Apiserver         = "apiserver"
-	Scheduler         = "scheduler"
-	ControllerManager = "controller-manager"
-)
-
-// ExtraConfigForComponent generates a map of flagname-value pairs for a k8s
-// component.
-func ExtraConfigForComponent(component string, opts config.ExtraOptionSlice, version semver.Version) (map[string]string, error) {
-	versionedOpts, err := DefaultOptionsForComponentAndVersion(component, version)
+// ParseKubernetesVersion parses the kubernetes version
+func ParseKubernetesVersion(version string) (semver.Version, error) {
+	// Strip leading 'v' prefix from version for semver parsing
+	v, err := semver.Make(version[1:])
 	if err != nil {
-		return nil, errors.Wrapf(err, "setting version specific options for %s", component)
-	}
-
-	for _, opt := range opts {
-		if opt.Component == component {
-			if val, ok := versionedOpts[opt.Key]; ok {
-				glog.Infof("Overwriting default %s=%s with user provided %s=%s for component %s", opt.Key, val, opt.Key, opt.Value, component)
-			}
-			versionedOpts[opt.Key] = opt.Value
-		}
-	}
-
-	return versionedOpts, nil
-}
-
-// ComponentExtraArgs holds extra args for a component
-type ComponentExtraArgs struct {
-	Component string
-	Options   map[string]string
-}
-
-// mapping of component to the section name in kubeadm.
-var componentToKubeadmConfigKey = map[string]string{
-	Apiserver:         "apiServer",
-	ControllerManager: "controllerManager",
-	Scheduler:         "scheduler",
-	Kubeadm:           "kubeadm",
-	// The Kubelet is not configured in kubeadm, only in systemd.
-	Kubelet: "",
-}
-
-// NewComponentExtraArgs creates a new ComponentExtraArgs
-func NewComponentExtraArgs(opts config.ExtraOptionSlice, version semver.Version, featureGates string) ([]ComponentExtraArgs, error) {
-	var kubeadmExtraArgs []ComponentExtraArgs
-	for _, extraOpt := range opts {
-		if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok {
-			return nil, fmt.Errorf("unknown component %q. valid components are: %v", componentToKubeadmConfigKey, componentToKubeadmConfigKey)
-		}
-	}
-
-	keys := []string{}
-	for k := range componentToKubeadmConfigKey {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-
-	for _, component := range keys {
-		kubeadmComponentKey := componentToKubeadmConfigKey[component]
-		if kubeadmComponentKey == "" {
-			continue
-		}
-		extraConfig, err := ExtraConfigForComponent(component, opts, version)
-		if err != nil {
-			return nil, errors.Wrapf(err, "getting kubeadm extra args for %s", component)
-		}
-		if featureGates != "" {
-			extraConfig["feature-gates"] = featureGates
-		}
-		if len(extraConfig) > 0 {
-			kubeadmExtraArgs = append(kubeadmExtraArgs, ComponentExtraArgs{
-				Component: kubeadmComponentKey,
-				Options:   extraConfig,
-			})
-		}
+		return semver.Version{}, errors.Wrap(err, "invalid version, must begin with 'v'")
 	}
 
-	return kubeadmExtraArgs, nil
+	return v, nil
 }
 
-// ParseFeatureArgs parses feature args into extra args
-func ParseFeatureArgs(featureGates string) (map[string]bool, string, error) {
+// parseFeatureArgs parses feature args into extra args
+func parseFeatureArgs(featureGates string) (map[string]bool, string, error) {
 	kubeadmFeatureArgs := map[string]bool{}
 	componentFeatureArgs := ""
 	for _, s := range strings.Split(featureGates, ",") {
@@ -160,30 +85,6 @@ func Supports(featureName string) bool {
 	return false
 }
 
-// parseKubernetesVersion parses the kubernetes version
-func parseKubernetesVersion(version string) (semver.Version, error) {
-	// Strip leading 'v' prefix from version for semver parsing
-	v, err := semver.Make(version[1:])
-	if err != nil {
-		return semver.Version{}, errors.Wrap(err, "invalid version, must begin with 'v'")
-	}
-
-	return v, nil
-}
-
-func convertToFlags(opts map[string]string) string {
-	var flags []string
-	var keys []string
-	for k := range opts {
-		keys = append(keys, k)
-	}
-	sort.Strings(keys)
-	for _, k := range keys {
-		flags = append(flags, fmt.Sprintf("--%s=%s", k, opts[k]))
-	}
-	return strings.Join(flags, " ")
-}
-
 var versionSpecificOpts = []config.VersionedExtraOption{
 	{
 		Option: config.ExtraOption{

pkg/minikube/bootstrapper/kubeadm/versions_test.go → pkg/minikube/bootstrapper/bsutil/versions_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package kubeadm
+package bsutil
 
 import (
 	"reflect"
@@ -94,7 +94,7 @@ func TestVersionIsBetween(t *testing.T) {
 }
 
 func TestParseKubernetesVersion(t *testing.T) {
-	version, err := parseKubernetesVersion("v1.8.0-alpha.5")
+	version, err := ParseKubernetesVersion("v1.8.0-alpha.5")
 	if err != nil {
 		t.Fatalf("Error parsing version: %v", err)
 	}
@@ -130,7 +130,7 @@ func TestParseFeatureArgs(t *testing.T) {
 
 	for _, test := range tests {
 		t.Run(test.description, func(t *testing.T) {
-			kubeadm, component, err := ParseFeatureArgs(test.featureGates)
+			kubeadm, component, err := parseFeatureArgs(test.featureGates)
 
 			if err != nil {
 				t.Fatalf("Error parsing feature args: %v", err)

pkg/minikube/bootstrapper/kicbs/kicbs.go

@@ -57,8 +57,63 @@ func (k *Bootstrapper) PullImages(config.KubernetesConfig) error {
 func (k *Bootstrapper) StartCluster(config.KubernetesConfig) error {
 	return fmt.Errorf("the StartCluster is not implemented in kicbs yet")
 }
-func (k *Bootstrapper) UpdateCluster(config.MachineConfig) error {
-	return fmt.Errorf("the UpdateCluster is not implemented in kicbs yet")
+
+// UpdateCluster generated kubeadm.yaml and other configs and loads the imags
+func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
+	// images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion)
+	// if err != nil {
+	// 	return errors.Wrap(err, "kubeadm images")
+	// }
+
+	// if cfg.KubernetesConfig.ShouldLoadCachedImages {
+	// 	if err := machine.LoadImages(&cfg, k.c, images, constants.ImageCacheDir); err != nil {
+	// 		out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err})
+	// 	}
+	// }
+	// r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, Socket: cfg.KubernetesConfig.CRISocket})
+	// if err != nil {
+	// 	return errors.Wrap(err, "runtime")
+	// }
+	// kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg.KubernetesConfig, r)
+	// if err != nil {
+	// 	return errors.Wrap(err, "generating kubeadm cfg")
+	// }
+
+	// kubeletCfg, err := NewKubeletConfig(cfg.KubernetesConfig, r)
+	// if err != nil {
+	// 	return errors.Wrap(err, "generating kubelet config")
+	// }
+
+	// kubeletService, err := NewKubeletService(cfg.KubernetesConfig)
+	// if err != nil {
+	// 	return errors.Wrap(err, "generating kubelet service")
+	// }
+
+	// glog.Infof("kubelet %s config:\n%+v", kubeletCfg, cfg.KubernetesConfig)
+
+	// stopCmd := exec.Command("/bin/bash", "-c", "pgrep kubelet && sudo systemctl stop kubelet")
+	// // stop kubelet to avoid "Text File Busy" error
+	// if rr, err := k.c.RunCmd(stopCmd); err != nil {
+	// 	glog.Warningf("unable to stop kubelet: %s command: %q output: %q", err, rr.Command(), rr.Output())
+	// }
+
+	// if err := transferBinaries(cfg.KubernetesConfig, k.c); err != nil {
+	// 	return errors.Wrap(err, "downloading binaries")
+	// }
+	// files := configFiles(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService)
+	// if err := addAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil {
+	// 	return errors.Wrap(err, "adding addons")
+	// }
+	// for _, f := range files {
+	// 	if err := k.c.Copy(f); err != nil {
+	// 		return errors.Wrapf(err, "copy")
+	// 	}
+	// }
+
+	// if _, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", "sudo systemctl daemon-reload && sudo systemctl start kubelet")); err != nil {
+	// 	return errors.Wrap(err, "starting kubelet")
+	// }
+	return nil
 }
 func (k *Bootstrapper) DeleteCluster(config.KubernetesConfig) error {
 	return fmt.Errorf("the DeleteCluster is not implemented in kicbs yet")

pkg/minikube/bootstrapper/kubeadm/kubeadm.go

@@ -17,7 +17,6 @@ limitations under the License.
 package kubeadm
 
 import (
-	"bytes"
 	"crypto/tls"
 	"os/exec"
 
@@ -27,7 +26,6 @@ import (
 
 	// WARNING: Do not use path/filepath in this package unless you want bizarre Windows paths
 	"path"
-	"runtime"
 	"strings"
 	"time"
 
@@ -37,7 +35,6 @@ import (
 	"github.com/golang/glog"
 	"github.com/pkg/errors"
 	"github.com/spf13/viper"
-	"golang.org/x/sync/errgroup"
 	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
@@ -45,6 +42,7 @@ import (
 	"k8s.io/minikube/pkg/kapi"
 	"k8s.io/minikube/pkg/minikube/assets"
 	"k8s.io/minikube/pkg/minikube/bootstrapper"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/command"
 	"k8s.io/minikube/pkg/minikube/config"
@@ -53,46 +51,18 @@ import (
 	"k8s.io/minikube/pkg/minikube/machine"
 	"k8s.io/minikube/pkg/minikube/out"
 	"k8s.io/minikube/pkg/minikube/vmpath"
-	"k8s.io/minikube/pkg/util"
 	"k8s.io/minikube/pkg/util/retry"
 )
 
 // enum to differentiate kubeadm command line parameters from kubeadm config file parameters (see the
 // KubeadmExtraArgsWhitelist variable below for more info)
 const (
-	KubeadmCmdParam        = iota
-	KubeadmConfigParam     = iota
 	defaultCNIConfigPath   = "/etc/cni/net.d/k8s.conf"
 	kubeletServiceFile     = "/lib/systemd/system/kubelet.service"
 	kubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf"
 )
 
-const (
-	// Container runtimes
-	remoteContainerRuntime = "remote"
-)
-
-// KubeadmExtraArgsWhitelist is a whitelist of supported kubeadm params that can be supplied to kubeadm through
-// minikube's ExtraArgs parameter. The list is split into two parts - params that can be supplied as flags on the
-// command line and params that have to be inserted into the kubeadm config file. This is because of a kubeadm
-// constraint which allows only certain params to be provided from the command line when the --config parameter
-// is specified
-var KubeadmExtraArgsWhitelist = map[int][]string{
-	KubeadmCmdParam: {
-		"ignore-preflight-errors",
-		"dry-run",
-		"kubeconfig",
-		"kubeconfig-dir",
-		"node-name",
-		"cri-socket",
-		"experimental-upload-certs",
-		"certificate-key",
-		"rootfs",
-	},
-	KubeadmConfigParam: {
-		"pod-network-cidr",
-	},
-}
+const ()
 
 // remote artifacts that must exist for minikube to function properly. The sign of a previously working installation.
 // NOTE: /etc is not persistent across restarts, so don't bother checking there
@@ -192,21 +162,6 @@ func (k *Bootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string
 	}
 }
 
-// createFlagsFromExtraArgs converts kubeadm extra args into flags to be supplied from the command linne
-func createFlagsFromExtraArgs(extraOptions config.ExtraOptionSlice) string {
-	kubeadmExtraOpts := extraOptions.AsMap().Get(Kubeadm)
-
-	// kubeadm allows only a small set of parameters to be supplied from the command line when the --config param
-	// is specified, here we remove those that are not allowed
-	for opt := range kubeadmExtraOpts {
-		if !config.ContainsParam(KubeadmExtraArgsWhitelist[KubeadmCmdParam], opt) {
-			// kubeadmExtraOpts is a copy so safe to delete
-			delete(kubeadmExtraOpts, opt)
-		}
-	}
-	return convertToFlags(kubeadmExtraOpts)
-}
-
 // etcdDataDir is where etcd data is stored.
 func etcdDataDir() string {
 	return path.Join(vmpath.GuestPersistentDir, "etcd")
@@ -249,12 +204,12 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 		glog.Infof("StartCluster complete in %s", time.Since(start))
 	}()
 
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
 
-	extraFlags := createFlagsFromExtraArgs(k8s.ExtraOptions)
+	extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions)
 	r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime})
 	if err != nil {
 		return err
@@ -279,7 +234,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error {
 		ignore = append(ignore, "SystemVerification")
 	}
 
-	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", invokeKubeadm(k8s.KubernetesVersion), yamlConfigPath, extraFlags, strings.Join(ignore, ",")))
+	c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), yamlConfigPath, extraFlags, strings.Join(ignore, ",")))
 	if rr, err := k.c.RunCmd(c); err != nil {
 		return errors.Wrapf(err, "init failed. cmd: %q", rr.Command())
 	}
@@ -481,7 +436,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 		glog.Infof("restartCluster took %s", time.Since(start))
 	}()
 
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -497,7 +452,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 		glog.Errorf("failed to create compat symlinks: %v", err)
 	}
 
-	baseCmd := fmt.Sprintf("%s %s", invokeKubeadm(k8s.KubernetesVersion), phase)
+	baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase)
 	cmds := []string{
 		fmt.Sprintf("%s phase certs all --config %s", baseCmd, yamlConfigPath),
 		fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, yamlConfigPath),
@@ -534,14 +489,14 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error {
 
 // DeleteCluster removes the components that were started earlier
 func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error {
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
 
-	cmd := fmt.Sprintf("%s reset --force", invokeKubeadm(k8s.KubernetesVersion))
+	cmd := fmt.Sprintf("%s reset --force", bsutil.InvokeKubeadm(k8s.KubernetesVersion))
 	if version.LT(semver.MustParse("1.11.0")) {
-		cmd = fmt.Sprintf("%s reset", invokeKubeadm(k8s.KubernetesVersion))
+		cmd = fmt.Sprintf("%s reset", bsutil.InvokeKubeadm(k8s.KubernetesVersion))
 	}
 
 	if rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", cmd)); err != nil {
@@ -553,7 +508,7 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error {
 
 // PullImages downloads images that will be used by Kubernetes
 func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error {
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
+	version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion)
 	if err != nil {
 		return errors.Wrap(err, "parsing kubernetes version")
 	}
@@ -561,7 +516,7 @@ func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error {
 		return fmt.Errorf("pull command is not supported by kubeadm v%s", version)
 	}
 
-	rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s config images pull --config %s", invokeKubeadm(k8s.KubernetesVersion), yamlConfigPath)))
+	rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s config images pull --config %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), yamlConfigPath)))
 	if err != nil {
 		return errors.Wrapf(err, "running cmd: %q", rr.Command())
 	}
@@ -573,61 +528,6 @@ func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig) error {
 	return bootstrapper.SetupCerts(k.c, k8s)
 }
 
-// NewKubeletConfig generates a new systemd unit containing a configured kubelet
-// based on the options present in the KubernetesConfig.
-func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
-	if err != nil {
-		return nil, errors.Wrap(err, "parsing kubernetes version")
-	}
-
-	extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version)
-	if err != nil {
-		return nil, errors.Wrap(err, "generating extra configuration for kubelet")
-	}
-
-	for k, v := range r.KubeletOptions() {
-		extraOpts[k] = v
-	}
-	if k8s.NetworkPlugin != "" {
-		extraOpts["network-plugin"] = k8s.NetworkPlugin
-	}
-	if _, ok := extraOpts["node-ip"]; !ok {
-		extraOpts["node-ip"] = k8s.NodeIP
-	}
-
-	pauseImage := images.Pause(k8s.ImageRepository)
-	if _, ok := extraOpts["pod-infra-container-image"]; !ok && k8s.ImageRepository != "" && pauseImage != "" && k8s.ContainerRuntime != remoteContainerRuntime {
-		extraOpts["pod-infra-container-image"] = pauseImage
-	}
-
-	// parses a map of the feature gates for kubelet
-	_, kubeletFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates)
-	if err != nil {
-		return nil, errors.Wrap(err, "parses feature gate config for kubelet")
-	}
-
-	if kubeletFeatureArgs != "" {
-		extraOpts["feature-gates"] = kubeletFeatureArgs
-	}
-
-	b := bytes.Buffer{}
-	opts := struct {
-		ExtraOptions     string
-		ContainerRuntime string
-		KubeletPath      string
-	}{
-		ExtraOptions:     convertToFlags(extraOpts),
-		ContainerRuntime: k8s.ContainerRuntime,
-		KubeletPath:      path.Join(binRoot(k8s.KubernetesVersion), "kubelet"),
-	}
-	if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil {
-		return nil, err
-	}
-
-	return b.Bytes(), nil
-}
-
 // UpdateCluster updates the cluster
 func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion)
@@ -644,17 +544,17 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	if err != nil {
 		return errors.Wrap(err, "runtime")
 	}
-	kubeadmCfg, err := generateConfig(cfg.KubernetesConfig, r)
+	kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg.KubernetesConfig, r)
 	if err != nil {
 		return errors.Wrap(err, "generating kubeadm cfg")
 	}
 
-	kubeletCfg, err := NewKubeletConfig(cfg.KubernetesConfig, r)
+	kubeletCfg, err := bsutil.NewKubeletConfig(cfg.KubernetesConfig, r)
 	if err != nil {
 		return errors.Wrap(err, "generating kubelet config")
 	}
 
-	kubeletService, err := NewKubeletService(cfg.KubernetesConfig)
+	kubeletService, err := bsutil.NewKubeletService(cfg.KubernetesConfig)
 	if err != nil {
 		return errors.Wrap(err, "generating kubelet service")
 	}
@@ -667,7 +567,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 		glog.Warningf("unable to stop kubelet: %s command: %q output: %q", err, rr.Command(), rr.Output())
 	}
 
-	if err := transferBinaries(cfg.KubernetesConfig, k.c); err != nil {
+	if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c); err != nil {
 		return errors.Wrap(err, "downloading binaries")
 	}
 	files := configFiles(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService)
@@ -686,112 +586,6 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error {
 	return nil
 }
 
-// createExtraComponentConfig generates a map of component to extra args for all of the components except kubeadm
-func createExtraComponentConfig(extraOptions config.ExtraOptionSlice, version semver.Version, componentFeatureArgs string) ([]ComponentExtraArgs, error) {
-	extraArgsSlice, err := NewComponentExtraArgs(extraOptions, version, componentFeatureArgs)
-	if err != nil {
-		return nil, err
-	}
-
-	// kubeadm extra args should not be included in the kubeadm config in the extra args section (instead, they must
-	// be inserted explicitly in the appropriate places or supplied from the command line); here we remove all of the
-	// kubeadm extra args from the slice
-	for i, extraArgs := range extraArgsSlice {
-		if extraArgs.Component == Kubeadm {
-			extraArgsSlice = append(extraArgsSlice[:i], extraArgsSlice[i+1:]...)
-			break
-		}
-	}
-	return extraArgsSlice, nil
-}
-
-// generateConfig generates the kubeadm.yaml file
-func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) {
-	version, err := parseKubernetesVersion(k8s.KubernetesVersion)
-	if err != nil {
-		return nil, errors.Wrap(err, "parsing kubernetes version")
-	}
-
-	// parses a map of the feature gates for kubeadm and component
-	kubeadmFeatureArgs, componentFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates)
-	if err != nil {
-		return nil, errors.Wrap(err, "parses feature gate config for kubeadm and component")
-	}
-
-	extraComponentConfig, err := createExtraComponentConfig(k8s.ExtraOptions, version, componentFeatureArgs)
-	if err != nil {
-		return nil, errors.Wrap(err, "generating extra component config for kubeadm")
-	}
-
-	// In case of no port assigned, use util.APIServerPort
-	nodePort := k8s.NodePort
-	if nodePort <= 0 {
-		nodePort = constants.APIServerPort
-	}
-
-	opts := struct {
-		CertDir           string
-		ServiceCIDR       string
-		PodSubnet         string
-		AdvertiseAddress  string
-		APIServerPort     int
-		KubernetesVersion string
-		EtcdDataDir       string
-		NodeName          string
-		DNSDomain         string
-		CRISocket         string
-		ImageRepository   string
-		ExtraArgs         []ComponentExtraArgs
-		FeatureArgs       map[string]bool
-		NoTaintMaster     bool
-	}{
-		CertDir:           vmpath.GuestCertsDir,
-		ServiceCIDR:       util.DefaultServiceCIDR,
-		PodSubnet:         k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm),
-		AdvertiseAddress:  k8s.NodeIP,
-		APIServerPort:     nodePort,
-		KubernetesVersion: k8s.KubernetesVersion,
-		EtcdDataDir:       etcdDataDir(),
-		NodeName:          k8s.NodeName,
-		CRISocket:         r.SocketPath(),
-		ImageRepository:   k8s.ImageRepository,
-		ExtraArgs:         extraComponentConfig,
-		FeatureArgs:       kubeadmFeatureArgs,
-		NoTaintMaster:     false, // That does not work with k8s 1.12+
-		DNSDomain:         k8s.DNSDomain,
-	}
-
-	if k8s.ServiceCIDR != "" {
-		opts.ServiceCIDR = k8s.ServiceCIDR
-	}
-
-	opts.NoTaintMaster = true
-	b := bytes.Buffer{}
-	configTmpl := configTmplV1Alpha1
-	if version.GTE(semver.MustParse("1.12.0")) {
-		configTmpl = configTmplV1Alpha3
-	}
-	// v1beta1 works in v1.13, but isn't required until v1.14.
-	if version.GTE(semver.MustParse("1.14.0-alpha.0")) {
-		configTmpl = configTmplV1Beta1
-	}
-	if err := configTmpl.Execute(&b, opts); err != nil {
-		return nil, err
-	}
-
-	return b.Bytes(), nil
-}
-
-// NewKubeletService returns a generated systemd unit file for the kubelet
-func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) {
-	var b bytes.Buffer
-	opts := struct{ KubeletPath string }{KubeletPath: path.Join(binRoot(cfg.KubernetesVersion), "kubelet")}
-	if err := kubeletServiceTemplate.Execute(&b, opts); err != nil {
-		return nil, errors.Wrap(err, "template execute")
-	}
-	return b.Bytes(), nil
-}
-
 // configFiles returns configuration file assets
 func configFiles(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byte, kubeletSvc []byte) []assets.CopyableFile {
 	fs := []assets.CopyableFile{
@@ -807,34 +601,3 @@ func configFiles(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byte, ku
 	}
 	return fs
 }
-
-// binDir returns the persistent path binaries are stored in
-func binRoot(version string) string {
-	return path.Join(vmpath.GuestPersistentDir, "binaries", version)
-}
-
-// invokeKubeadm returns the invocation command for Kubeadm
-func invokeKubeadm(version string) string {
-	return fmt.Sprintf("sudo env PATH=%s:$PATH kubeadm", binRoot(version))
-}
-
-// transferBinaries transfers all required Kubernetes binaries
-func transferBinaries(cfg config.KubernetesConfig, c command.Runner) error {
-	var g errgroup.Group
-	for _, name := range constants.KubeadmBinaries {
-		name := name
-		g.Go(func() error {
-			src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH)
-			if err != nil {
-				return errors.Wrapf(err, "downloading %s", name)
-			}
-
-			dst := path.Join(binRoot(cfg.KubernetesVersion), name)
-			if err := machine.CopyBinary(c, src, dst); err != nil {
-				return errors.Wrapf(err, "copybinary %s -> %s", src, dst)
-			}
-			return nil
-		})
-	}
-	return g.Wait()
-}

pkg/minikube/cluster/cluster.go

@@ -173,7 +173,6 @@ func configureHost(h *host.Host, e *engine.Options) error {
 
 // ensureGuestClockSync ensures that the guest system clock is relatively in-sync
 func ensureSyncedGuestClock(h hostRunner) error {
-	fmt.Println("Inside ensureSyncedGuestClock")
 	d, err := guestClockDelta(h, time.Now())
 	if err != nil {
 		glog.Warningf("Unable to measure system clock delta: %v", err)