From b8d07daef3596fed20b9e620a7164b5abaa45ae3 Mon Sep 17 00:00:00 2001 From: Medya Gh Date: Wed, 18 Dec 2019 17:22:42 -0800 Subject: [PATCH] move reusabe parts of kubeadm to a package --- cmd/minikube/cmd/start.go | 10 +- pkg/minikube/bootstrapper/bsutil/binaries.go | 51 ++++ pkg/minikube/bootstrapper/bsutil/bsutil.go | 224 +++++++++++++++ .../kubeadm_test.go => bsutil/bsutil_test.go} | 10 +- .../bootstrapper/bsutil/extraconfig.go | 156 ++++++++++ .../bootstrapper/bsutil/featuregates.go | 18 ++ .../{kubeadm => bsutil}/templates.go | 18 +- .../bsutil/templates/templates.go | 23 ++ .../{kubeadm => bsutil}/templates_test.go | 2 +- .../{kubeadm => bsutil}/versions.go | 117 +------- .../{kubeadm => bsutil}/versions_test.go | 6 +- pkg/minikube/bootstrapper/kicbs/kicbs.go | 59 +++- pkg/minikube/bootstrapper/kubeadm/kubeadm.go | 269 ++---------------- pkg/minikube/cluster/cluster.go | 1 - 14 files changed, 577 insertions(+), 387 deletions(-) create mode 100644 pkg/minikube/bootstrapper/bsutil/binaries.go create mode 100644 pkg/minikube/bootstrapper/bsutil/bsutil.go rename pkg/minikube/bootstrapper/{kubeadm/kubeadm_test.go => bsutil/bsutil_test.go} (98%) create mode 100644 pkg/minikube/bootstrapper/bsutil/extraconfig.go create mode 100644 pkg/minikube/bootstrapper/bsutil/featuregates.go rename pkg/minikube/bootstrapper/{kubeadm => bsutil}/templates.go (92%) create mode 100644 pkg/minikube/bootstrapper/bsutil/templates/templates.go rename pkg/minikube/bootstrapper/{kubeadm => bsutil}/templates_test.go (98%) rename pkg/minikube/bootstrapper/{kubeadm => bsutil}/versions.go (62%) rename pkg/minikube/bootstrapper/{kubeadm => bsutil}/versions_test.go (96%) diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index a4d4db00e..20a86a699 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -47,8 +47,8 @@ import ( "golang.org/x/sync/errgroup" cmdcfg "k8s.io/minikube/cmd/minikube/cmd/config" "k8s.io/minikube/pkg/minikube/bootstrapper" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" "k8s.io/minikube/pkg/minikube/bootstrapper/images" - "k8s.io/minikube/pkg/minikube/bootstrapper/kubeadm" "k8s.io/minikube/pkg/minikube/cluster" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" @@ -186,7 +186,7 @@ func initKubernetesFlags() { `A set of key=value pairs that describe configuration that may be passed to different components. The key should be '.' separated, and the first part before the dot is the component to apply the configuration to. Valid components are: kubelet, kubeadm, apiserver, controller-manager, etcd, proxy, scheduler - Valid kubeadm parameters: `+fmt.Sprintf("%s, %s", strings.Join(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], ", "), strings.Join(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], ","))) + Valid kubeadm parameters: `+fmt.Sprintf("%s, %s", strings.Join(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], ", "), strings.Join(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], ","))) startCmd.Flags().String(featureGates, "", "A set of key=value pairs that describe feature gates for alpha/experimental features.") startCmd.Flags().String(dnsDomain, constants.ClusterDNSDomain, "The cluster dns domain name used in the kubernetes cluster") startCmd.Flags().Int(apiServerPort, constants.APIServerPort, "The apiserver listening port") @@ -789,9 +789,9 @@ func validateFlags(cmd *cobra.Command, drvName string) { } // check that kubeadm extra args contain only whitelisted parameters - for param := range extraOptions.AsMap().Get(kubeadm.Kubeadm) { - if !cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmCmdParam], param) && - !cfg.ContainsParam(kubeadm.KubeadmExtraArgsWhitelist[kubeadm.KubeadmConfigParam], param) { + for param := range extraOptions.AsMap().Get(bsutil.Kubeadm) { + if !cfg.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmCmdParam], param) && + !cfg.ContainsParam(bsutil.KubeadmExtraArgsWhitelist[bsutil.KubeadmConfigParam], param) { exit.UsageT("Sorry, the kubeadm.{{.parameter_name}} parameter is currently not supported by --extra-config", out.V{"parameter_name": param}) } } diff --git a/pkg/minikube/bootstrapper/bsutil/binaries.go b/pkg/minikube/bootstrapper/bsutil/binaries.go new file mode 100644 index 000000000..0db71752e --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/binaries.go @@ -0,0 +1,51 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +package bsutil + +import ( + "path" + "runtime" + + "github.com/pkg/errors" + "golang.org/x/sync/errgroup" + "k8s.io/minikube/pkg/minikube/command" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/machine" +) + +// TransferBinaries transfers all required Kubernetes binaries +func TransferBinaries(cfg config.KubernetesConfig, c command.Runner) error { + var g errgroup.Group + for _, name := range constants.KubeadmBinaries { + name := name + g.Go(func() error { + src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH) + if err != nil { + return errors.Wrapf(err, "downloading %s", name) + } + + dst := path.Join(binRoot(cfg.KubernetesVersion), name) + if err := machine.CopyBinary(c, src, dst); err != nil { + return errors.Wrapf(err, "copybinary %s -> %s", src, dst) + } + return nil + }) + } + return g.Wait() +} diff --git a/pkg/minikube/bootstrapper/bsutil/bsutil.go b/pkg/minikube/bootstrapper/bsutil/bsutil.go new file mode 100644 index 000000000..a91dc5a34 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/bsutil.go @@ -0,0 +1,224 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +package bsutil + +import ( + "bytes" + "fmt" + "path" + + "github.com/blang/semver" + "github.com/golang/glog" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/config" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/cruntime" + "k8s.io/minikube/pkg/minikube/vmpath" + "k8s.io/minikube/pkg/util" +) + +// Container runtimes +const remoteContainerRuntime = "remote" + +// GenerateKubeadmYAML generates the kubeadm.yaml file +func GenerateKubeadmYAML(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) { + version, err := ParseKubernetesVersion(k8s.KubernetesVersion) + if err != nil { + return nil, errors.Wrap(err, "parsing kubernetes version") + } + + // parses a map of the feature gates for kubeadm and component + kubeadmFeatureArgs, componentFeatureArgs, err := parseFeatureArgs(k8s.FeatureGates) + if err != nil { + return nil, errors.Wrap(err, "parses feature gate config for kubeadm and component") + } + + extraComponentConfig, err := createExtraComponentConfig(k8s.ExtraOptions, version, componentFeatureArgs) + if err != nil { + return nil, errors.Wrap(err, "generating extra component config for kubeadm") + } + + // In case of no port assigned, use util.APIServerPort + nodePort := k8s.NodePort + if nodePort <= 0 { + nodePort = constants.APIServerPort + } + + opts := struct { + CertDir string + ServiceCIDR string + PodSubnet string + AdvertiseAddress string + APIServerPort int + KubernetesVersion string + EtcdDataDir string + NodeName string + DNSDomain string + CRISocket string + ImageRepository string + ExtraArgs []ComponentExtraArgs + FeatureArgs map[string]bool + NoTaintMaster bool + }{ + CertDir: vmpath.GuestCertsDir, + ServiceCIDR: util.DefaultServiceCIDR, + PodSubnet: k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm), + AdvertiseAddress: k8s.NodeIP, + APIServerPort: nodePort, + KubernetesVersion: k8s.KubernetesVersion, + EtcdDataDir: etcdDataDir(), + NodeName: k8s.NodeName, + CRISocket: r.SocketPath(), + ImageRepository: k8s.ImageRepository, + ExtraArgs: extraComponentConfig, + FeatureArgs: kubeadmFeatureArgs, + NoTaintMaster: false, // That does not work with k8s 1.12+ + DNSDomain: k8s.DNSDomain, + } + + if k8s.ServiceCIDR != "" { + opts.ServiceCIDR = k8s.ServiceCIDR + } + + opts.NoTaintMaster = true + b := bytes.Buffer{} + configTmpl := ConfigTmplV1Alpha1 + if version.GTE(semver.MustParse("1.12.0")) { + configTmpl = ConfigTmplV1Alpha3 + } + // v1beta1 works in v1.13, but isn't required until v1.14. + if version.GTE(semver.MustParse("1.14.0-alpha.0")) { + configTmpl = ConfigTmplV1Beta1 + } + if err := configTmpl.Execute(&b, opts); err != nil { + return nil, err + } + + return b.Bytes(), nil +} + +// NewKubeletConfig generates a new systemd unit containing a configured kubelet +// based on the options present in the KubernetesConfig. +func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) { + version, err := ParseKubernetesVersion(k8s.KubernetesVersion) + if err != nil { + return nil, errors.Wrap(err, "parsing kubernetes version") + } + + extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version) + if err != nil { + return nil, errors.Wrap(err, "generating extra configuration for kubelet") + } + + for k, v := range r.KubeletOptions() { + extraOpts[k] = v + } + if k8s.NetworkPlugin != "" { + extraOpts["network-plugin"] = k8s.NetworkPlugin + } + if _, ok := extraOpts["node-ip"]; !ok { + extraOpts["node-ip"] = k8s.NodeIP + } + + pauseImage := images.Pause(k8s.ImageRepository) + if _, ok := extraOpts["pod-infra-container-image"]; !ok && k8s.ImageRepository != "" && pauseImage != "" && k8s.ContainerRuntime != remoteContainerRuntime { + extraOpts["pod-infra-container-image"] = pauseImage + } + + // parses a map of the feature gates for kubelet + _, kubeletFeatureArgs, err := parseFeatureArgs(k8s.FeatureGates) + if err != nil { + return nil, errors.Wrap(err, "parses feature gate config for kubelet") + } + + if kubeletFeatureArgs != "" { + extraOpts["feature-gates"] = kubeletFeatureArgs + } + + b := bytes.Buffer{} + opts := struct { + ExtraOptions string + ContainerRuntime string + KubeletPath string + }{ + ExtraOptions: convertToFlags(extraOpts), + ContainerRuntime: k8s.ContainerRuntime, + KubeletPath: path.Join(binRoot(k8s.KubernetesVersion), "kubelet"), + } + if err := KubeletSystemdTemplate.Execute(&b, opts); err != nil { + return nil, err + } + + return b.Bytes(), nil +} + +// NewKubeletService returns a generated systemd unit file for the kubelet +func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) { + var b bytes.Buffer + opts := struct{ KubeletPath string }{KubeletPath: path.Join(binRoot(cfg.KubernetesVersion), "kubelet")} + if err := kubeletServiceTemplate.Execute(&b, opts); err != nil { + return nil, errors.Wrap(err, "template execute") + } + return b.Bytes(), nil +} + +// These are the components that can be configured +// through the "extra-config" +const ( + Kubelet = "kubelet" + Kubeadm = "kubeadm" + Apiserver = "apiserver" + Scheduler = "scheduler" + ControllerManager = "controller-manager" +) + +// ExtraConfigForComponent generates a map of flagname-value pairs for a k8s +// component. +func ExtraConfigForComponent(component string, opts config.ExtraOptionSlice, version semver.Version) (map[string]string, error) { + versionedOpts, err := DefaultOptionsForComponentAndVersion(component, version) + if err != nil { + return nil, errors.Wrapf(err, "setting version specific options for %s", component) + } + + for _, opt := range opts { + if opt.Component == component { + if val, ok := versionedOpts[opt.Key]; ok { + glog.Infof("Overwriting default %s=%s with user provided %s=%s for component %s", opt.Key, val, opt.Key, opt.Value, component) + } + versionedOpts[opt.Key] = opt.Value + } + } + + return versionedOpts, nil +} + +// binRoot returns the persistent path binaries are stored in +func binRoot(version string) string { + return path.Join(vmpath.GuestPersistentDir, "binaries", version) +} + +// InvokeKubeadm returns the invocation command for Kubeadm +func InvokeKubeadm(version string) string { + return fmt.Sprintf("sudo env PATH=%s:$PATH kubeadm", binRoot(version)) +} + +// etcdDataDir is where etcd data is stored. +func etcdDataDir() string { + return path.Join(vmpath.GuestPersistentDir, "etcd") +} diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go b/pkg/minikube/bootstrapper/bsutil/bsutil_test.go similarity index 98% rename from pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go rename to pkg/minikube/bootstrapper/bsutil/bsutil_test.go index a387d7193..b316a8d3d 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm_test.go +++ b/pkg/minikube/bootstrapper/bsutil/bsutil_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadm +package bsutil import ( "fmt" @@ -241,7 +241,7 @@ as v1.11 yaml file is very different compared to v1.12+. This test case has only 1 thing to test and that is the nnetworking/dnsDomain value */ -func TestGenerateConfigDNS(t *testing.T) { +func TestGenerateKubeadmYAMLDNS(t *testing.T) { versions := []string{"v1.16", "v1.15", "v1.14", "v1.13", "v1.12"} tests := []struct { name string @@ -264,7 +264,7 @@ func TestGenerateConfigDNS(t *testing.T) { cfg.NodeName = "mk" cfg.KubernetesVersion = version + ".0" - got, err := generateConfig(cfg, runtime) + got, err := GenerateKubeadmYAML(cfg, runtime) if err != nil && !tc.shouldErr { t.Fatalf("got unexpected error generating config: %v", err) } @@ -296,7 +296,7 @@ func TestGenerateConfigDNS(t *testing.T) { } } -func TestGenerateConfig(t *testing.T) { +func TestGenerateKubeadmYAML(t *testing.T) { extraOpts := getExtraOpts() extraOptsPodCidr := getExtraOptsPodCidr() versions, err := recentReleases() @@ -332,7 +332,7 @@ func TestGenerateConfig(t *testing.T) { cfg.NodeName = "mk" cfg.KubernetesVersion = version + ".0" - got, err := generateConfig(cfg, runtime) + got, err := GenerateKubeadmYAML(cfg, runtime) if err != nil && !tc.shouldErr { t.Fatalf("got unexpected error generating config: %v", err) } diff --git a/pkg/minikube/bootstrapper/bsutil/extraconfig.go b/pkg/minikube/bootstrapper/bsutil/extraconfig.go new file mode 100644 index 000000000..a9722474f --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/extraconfig.go @@ -0,0 +1,156 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +package bsutil + +import ( + "fmt" + "sort" + "strings" + + "github.com/blang/semver" + "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/config" +) + +const ( + KubeadmCmdParam = iota + KubeadmConfigParam = iota +) + +// ComponentExtraArgs holds extra args for a component +type ComponentExtraArgs struct { + Component string + Options map[string]string +} + +// mapping of component to the section name in kubeadm. +var componentToKubeadmConfigKey = map[string]string{ + Apiserver: "apiServer", + ControllerManager: "controllerManager", + Scheduler: "scheduler", + Kubeadm: "kubeadm", + // The Kubelet is not configured in kubeadm, only in systemd. + Kubelet: "", +} + +// KubeadmExtraArgsWhitelist is a whitelist of supported kubeadm params that can be supplied to kubeadm through +// minikube's ExtraArgs parameter. The list is split into two parts - params that can be supplied as flags on the +// command line and params that have to be inserted into the kubeadm config file. This is because of a kubeadm +// constraint which allows only certain params to be provided from the command line when the --config parameter +// is specified +var KubeadmExtraArgsWhitelist = map[int][]string{ + KubeadmCmdParam: { + "ignore-preflight-errors", + "dry-run", + "kubeconfig", + "kubeconfig-dir", + "node-name", + "cri-socket", + "experimental-upload-certs", + "certificate-key", + "rootfs", + }, + KubeadmConfigParam: { + "pod-network-cidr", + }, +} + +// NewComponentExtraArgs creates a new ComponentExtraArgs +func NewComponentExtraArgs(opts config.ExtraOptionSlice, version semver.Version, featureGates string) ([]ComponentExtraArgs, error) { + var kubeadmExtraArgs []ComponentExtraArgs + for _, extraOpt := range opts { + if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok { + return nil, fmt.Errorf("unknown component %q. valid components are: %v", componentToKubeadmConfigKey, componentToKubeadmConfigKey) + } + } + + keys := []string{} + for k := range componentToKubeadmConfigKey { + keys = append(keys, k) + } + sort.Strings(keys) + + for _, component := range keys { + kubeadmComponentKey := componentToKubeadmConfigKey[component] + if kubeadmComponentKey == "" { + continue + } + extraConfig, err := ExtraConfigForComponent(component, opts, version) + if err != nil { + return nil, errors.Wrapf(err, "getting kubeadm extra args for %s", component) + } + if featureGates != "" { + extraConfig["feature-gates"] = featureGates + } + if len(extraConfig) > 0 { + kubeadmExtraArgs = append(kubeadmExtraArgs, ComponentExtraArgs{ + Component: kubeadmComponentKey, + Options: extraConfig, + }) + } + } + + return kubeadmExtraArgs, nil +} + +// CreateFlagsFromExtraArgs converts kubeadm extra args into flags to be supplied from the command linne +func CreateFlagsFromExtraArgs(extraOptions config.ExtraOptionSlice) string { + kubeadmExtraOpts := extraOptions.AsMap().Get(Kubeadm) + + // kubeadm allows only a small set of parameters to be supplied from the command line when the --config param + // is specified, here we remove those that are not allowed + for opt := range kubeadmExtraOpts { + if !config.ContainsParam(KubeadmExtraArgsWhitelist[KubeadmCmdParam], opt) { + // kubeadmExtraOpts is a copy so safe to delete + delete(kubeadmExtraOpts, opt) + } + } + return convertToFlags(kubeadmExtraOpts) +} + +// createExtraComponentConfig generates a map of component to extra args for all of the components except kubeadm +func createExtraComponentConfig(extraOptions config.ExtraOptionSlice, version semver.Version, componentFeatureArgs string) ([]ComponentExtraArgs, error) { + extraArgsSlice, err := NewComponentExtraArgs(extraOptions, version, componentFeatureArgs) + if err != nil { + return nil, err + } + + // kubeadm extra args should not be included in the kubeadm config in the extra args section (instead, they must + // be inserted explicitly in the appropriate places or supplied from the command line); here we remove all of the + // kubeadm extra args from the slice + for i, extraArgs := range extraArgsSlice { + if extraArgs.Component == Kubeadm { + extraArgsSlice = append(extraArgsSlice[:i], extraArgsSlice[i+1:]...) + break + } + } + return extraArgsSlice, nil +} + +func convertToFlags(opts map[string]string) string { + var flags []string + var keys []string + for k := range opts { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + flags = append(flags, fmt.Sprintf("--%s=%s", k, opts[k])) + } + return strings.Join(flags, " ") +} diff --git a/pkg/minikube/bootstrapper/bsutil/featuregates.go b/pkg/minikube/bootstrapper/bsutil/featuregates.go new file mode 100644 index 000000000..89fe894f2 --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/featuregates.go @@ -0,0 +1,18 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// bsutil package will eventually be renamed to kubeadm package after getting rid of older one +package bsutil diff --git a/pkg/minikube/bootstrapper/kubeadm/templates.go b/pkg/minikube/bootstrapper/bsutil/templates.go similarity index 92% rename from pkg/minikube/bootstrapper/kubeadm/templates.go rename to pkg/minikube/bootstrapper/bsutil/templates.go index 5f5325c15..329f97bdd 100644 --- a/pkg/minikube/bootstrapper/kubeadm/templates.go +++ b/pkg/minikube/bootstrapper/bsutil/templates.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadm +package bsutil import ( "fmt" @@ -22,8 +22,8 @@ import ( "text/template" ) -// configTmplV1Alpha1 is for Kubernetes v1.11 -var configTmplV1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{ +// ConfigTmplV1Alpha1 is for Kubernetes v1.11 +var ConfigTmplV1Alpha1 = template.Must(template.New("configTmpl-v1alpha1").Funcs(template.FuncMap{ "printMapInOrder": printMapInOrder, }).Parse(`apiVersion: kubeadm.k8s.io/v1alpha1 kind: MasterConfiguration @@ -47,8 +47,8 @@ nodeName: {{.NodeName}} {{$i}}: {{$val}}{{end}} {{end}}`)) -// configTmplV1Alpha3 is for Kubernetes v1.12 -var configTmplV1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{ +// ConfigTmplV1Alpha3 is for Kubernetes v1.12 +var ConfigTmplV1Alpha3 = template.Must(template.New("configTmpl-v1alpha3").Funcs(template.FuncMap{ "printMapInOrder": printMapInOrder, }).Parse(`apiVersion: kubeadm.k8s.io/v1alpha3 kind: InitConfiguration @@ -96,8 +96,8 @@ evictionHard: imagefs.available: "0%" `)) -// configTmplV1Beta1 is for Kubernetes v1.13+ -var configTmplV1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{ +// ConfigTmplV1Beta1 is for Kubernetes v1.13+ +var ConfigTmplV1Beta1 = template.Must(template.New("configTmpl-v1beta1").Funcs(template.FuncMap{ "printMapInOrder": printMapInOrder, }).Parse(`apiVersion: kubeadm.k8s.io/v1beta1 kind: InitConfiguration @@ -151,8 +151,8 @@ evictionHard: imagefs.available: "0%" `)) -// kubeletSystemdTemplate hosts the override kubelet flags, written to kubeletSystemdConfFile -var kubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`[Unit] +// KubeletSystemdTemplate hosts the override kubelet flags, written to kubeletSystemdConfFile +var KubeletSystemdTemplate = template.Must(template.New("kubeletSystemdTemplate").Parse(`[Unit] {{if or (eq .ContainerRuntime "cri-o") (eq .ContainerRuntime "cri")}}Wants=crio.service{{else if eq .ContainerRuntime "containerd"}}Wants=containerd.service{{else}}Wants=docker.socket{{end}} [Service] diff --git a/pkg/minikube/bootstrapper/bsutil/templates/templates.go b/pkg/minikube/bootstrapper/bsutil/templates/templates.go new file mode 100644 index 000000000..ac268596d --- /dev/null +++ b/pkg/minikube/bootstrapper/bsutil/templates/templates.go @@ -0,0 +1,23 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package templates + +import "fmt" + +func Dum() { + fmt.Println("hello") +} diff --git a/pkg/minikube/bootstrapper/kubeadm/templates_test.go b/pkg/minikube/bootstrapper/bsutil/templates_test.go similarity index 98% rename from pkg/minikube/bootstrapper/kubeadm/templates_test.go rename to pkg/minikube/bootstrapper/bsutil/templates_test.go index 126815b2b..6636976b1 100644 --- a/pkg/minikube/bootstrapper/kubeadm/templates_test.go +++ b/pkg/minikube/bootstrapper/bsutil/templates_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadm +package bsutil import ( "reflect" diff --git a/pkg/minikube/bootstrapper/kubeadm/versions.go b/pkg/minikube/bootstrapper/bsutil/versions.go similarity index 62% rename from pkg/minikube/bootstrapper/kubeadm/versions.go rename to pkg/minikube/bootstrapper/bsutil/versions.go index 82e157fd7..0b2952d90 100644 --- a/pkg/minikube/bootstrapper/kubeadm/versions.go +++ b/pkg/minikube/bootstrapper/bsutil/versions.go @@ -14,17 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadm +package bsutil import ( "fmt" "path" - "sort" "strconv" "strings" "github.com/blang/semver" - "github.com/golang/glog" "github.com/pkg/errors" "k8s.io/kubernetes/cmd/kubeadm/app/features" "k8s.io/minikube/pkg/minikube/config" @@ -33,92 +31,19 @@ import ( "k8s.io/minikube/pkg/util" ) -// These are the components that can be configured -// through the "extra-config" -const ( - Kubelet = "kubelet" - Kubeadm = "kubeadm" - Apiserver = "apiserver" - Scheduler = "scheduler" - ControllerManager = "controller-manager" -) - -// ExtraConfigForComponent generates a map of flagname-value pairs for a k8s -// component. -func ExtraConfigForComponent(component string, opts config.ExtraOptionSlice, version semver.Version) (map[string]string, error) { - versionedOpts, err := DefaultOptionsForComponentAndVersion(component, version) +// ParseKubernetesVersion parses the kubernetes version +func ParseKubernetesVersion(version string) (semver.Version, error) { + // Strip leading 'v' prefix from version for semver parsing + v, err := semver.Make(version[1:]) if err != nil { - return nil, errors.Wrapf(err, "setting version specific options for %s", component) - } - - for _, opt := range opts { - if opt.Component == component { - if val, ok := versionedOpts[opt.Key]; ok { - glog.Infof("Overwriting default %s=%s with user provided %s=%s for component %s", opt.Key, val, opt.Key, opt.Value, component) - } - versionedOpts[opt.Key] = opt.Value - } - } - - return versionedOpts, nil -} - -// ComponentExtraArgs holds extra args for a component -type ComponentExtraArgs struct { - Component string - Options map[string]string -} - -// mapping of component to the section name in kubeadm. -var componentToKubeadmConfigKey = map[string]string{ - Apiserver: "apiServer", - ControllerManager: "controllerManager", - Scheduler: "scheduler", - Kubeadm: "kubeadm", - // The Kubelet is not configured in kubeadm, only in systemd. - Kubelet: "", -} - -// NewComponentExtraArgs creates a new ComponentExtraArgs -func NewComponentExtraArgs(opts config.ExtraOptionSlice, version semver.Version, featureGates string) ([]ComponentExtraArgs, error) { - var kubeadmExtraArgs []ComponentExtraArgs - for _, extraOpt := range opts { - if _, ok := componentToKubeadmConfigKey[extraOpt.Component]; !ok { - return nil, fmt.Errorf("unknown component %q. valid components are: %v", componentToKubeadmConfigKey, componentToKubeadmConfigKey) - } - } - - keys := []string{} - for k := range componentToKubeadmConfigKey { - keys = append(keys, k) - } - sort.Strings(keys) - - for _, component := range keys { - kubeadmComponentKey := componentToKubeadmConfigKey[component] - if kubeadmComponentKey == "" { - continue - } - extraConfig, err := ExtraConfigForComponent(component, opts, version) - if err != nil { - return nil, errors.Wrapf(err, "getting kubeadm extra args for %s", component) - } - if featureGates != "" { - extraConfig["feature-gates"] = featureGates - } - if len(extraConfig) > 0 { - kubeadmExtraArgs = append(kubeadmExtraArgs, ComponentExtraArgs{ - Component: kubeadmComponentKey, - Options: extraConfig, - }) - } + return semver.Version{}, errors.Wrap(err, "invalid version, must begin with 'v'") } - return kubeadmExtraArgs, nil + return v, nil } -// ParseFeatureArgs parses feature args into extra args -func ParseFeatureArgs(featureGates string) (map[string]bool, string, error) { +// parseFeatureArgs parses feature args into extra args +func parseFeatureArgs(featureGates string) (map[string]bool, string, error) { kubeadmFeatureArgs := map[string]bool{} componentFeatureArgs := "" for _, s := range strings.Split(featureGates, ",") { @@ -160,30 +85,6 @@ func Supports(featureName string) bool { return false } -// parseKubernetesVersion parses the kubernetes version -func parseKubernetesVersion(version string) (semver.Version, error) { - // Strip leading 'v' prefix from version for semver parsing - v, err := semver.Make(version[1:]) - if err != nil { - return semver.Version{}, errors.Wrap(err, "invalid version, must begin with 'v'") - } - - return v, nil -} - -func convertToFlags(opts map[string]string) string { - var flags []string - var keys []string - for k := range opts { - keys = append(keys, k) - } - sort.Strings(keys) - for _, k := range keys { - flags = append(flags, fmt.Sprintf("--%s=%s", k, opts[k])) - } - return strings.Join(flags, " ") -} - var versionSpecificOpts = []config.VersionedExtraOption{ { Option: config.ExtraOption{ diff --git a/pkg/minikube/bootstrapper/kubeadm/versions_test.go b/pkg/minikube/bootstrapper/bsutil/versions_test.go similarity index 96% rename from pkg/minikube/bootstrapper/kubeadm/versions_test.go rename to pkg/minikube/bootstrapper/bsutil/versions_test.go index 9b76841b9..846fba24c 100644 --- a/pkg/minikube/bootstrapper/kubeadm/versions_test.go +++ b/pkg/minikube/bootstrapper/bsutil/versions_test.go @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package kubeadm +package bsutil import ( "reflect" @@ -94,7 +94,7 @@ func TestVersionIsBetween(t *testing.T) { } func TestParseKubernetesVersion(t *testing.T) { - version, err := parseKubernetesVersion("v1.8.0-alpha.5") + version, err := ParseKubernetesVersion("v1.8.0-alpha.5") if err != nil { t.Fatalf("Error parsing version: %v", err) } @@ -130,7 +130,7 @@ func TestParseFeatureArgs(t *testing.T) { for _, test := range tests { t.Run(test.description, func(t *testing.T) { - kubeadm, component, err := ParseFeatureArgs(test.featureGates) + kubeadm, component, err := parseFeatureArgs(test.featureGates) if err != nil { t.Fatalf("Error parsing feature args: %v", err) diff --git a/pkg/minikube/bootstrapper/kicbs/kicbs.go b/pkg/minikube/bootstrapper/kicbs/kicbs.go index 21ab12b9e..c5ff7f6dd 100644 --- a/pkg/minikube/bootstrapper/kicbs/kicbs.go +++ b/pkg/minikube/bootstrapper/kicbs/kicbs.go @@ -57,8 +57,63 @@ func (k *Bootstrapper) PullImages(config.KubernetesConfig) error { func (k *Bootstrapper) StartCluster(config.KubernetesConfig) error { return fmt.Errorf("the StartCluster is not implemented in kicbs yet") } -func (k *Bootstrapper) UpdateCluster(config.MachineConfig) error { - return fmt.Errorf("the UpdateCluster is not implemented in kicbs yet") + +// UpdateCluster generated kubeadm.yaml and other configs and loads the imags +func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { + // images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion) + // if err != nil { + // return errors.Wrap(err, "kubeadm images") + // } + + // if cfg.KubernetesConfig.ShouldLoadCachedImages { + // if err := machine.LoadImages(&cfg, k.c, images, constants.ImageCacheDir); err != nil { + // out.FailureT("Unable to load cached images: {{.error}}", out.V{"error": err}) + // } + // } + // r, err := cruntime.New(cruntime.Config{Type: cfg.ContainerRuntime, Socket: cfg.KubernetesConfig.CRISocket}) + // if err != nil { + // return errors.Wrap(err, "runtime") + // } + // kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg.KubernetesConfig, r) + // if err != nil { + // return errors.Wrap(err, "generating kubeadm cfg") + // } + + // kubeletCfg, err := NewKubeletConfig(cfg.KubernetesConfig, r) + // if err != nil { + // return errors.Wrap(err, "generating kubelet config") + // } + + // kubeletService, err := NewKubeletService(cfg.KubernetesConfig) + // if err != nil { + // return errors.Wrap(err, "generating kubelet service") + // } + + // glog.Infof("kubelet %s config:\n%+v", kubeletCfg, cfg.KubernetesConfig) + + // stopCmd := exec.Command("/bin/bash", "-c", "pgrep kubelet && sudo systemctl stop kubelet") + // // stop kubelet to avoid "Text File Busy" error + // if rr, err := k.c.RunCmd(stopCmd); err != nil { + // glog.Warningf("unable to stop kubelet: %s command: %q output: %q", err, rr.Command(), rr.Output()) + // } + + // if err := transferBinaries(cfg.KubernetesConfig, k.c); err != nil { + // return errors.Wrap(err, "downloading binaries") + // } + // files := configFiles(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService) + // if err := addAddons(&files, assets.GenerateTemplateData(cfg.KubernetesConfig)); err != nil { + // return errors.Wrap(err, "adding addons") + // } + // for _, f := range files { + // if err := k.c.Copy(f); err != nil { + // return errors.Wrapf(err, "copy") + // } + // } + + // if _, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", "sudo systemctl daemon-reload && sudo systemctl start kubelet")); err != nil { + // return errors.Wrap(err, "starting kubelet") + // } + return nil } func (k *Bootstrapper) DeleteCluster(config.KubernetesConfig) error { return fmt.Errorf("the DeleteCluster is not implemented in kicbs yet") diff --git a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go index cfe28ea26..1fd6ed652 100644 --- a/pkg/minikube/bootstrapper/kubeadm/kubeadm.go +++ b/pkg/minikube/bootstrapper/kubeadm/kubeadm.go @@ -17,7 +17,6 @@ limitations under the License. package kubeadm import ( - "bytes" "crypto/tls" "os/exec" @@ -27,7 +26,6 @@ import ( // WARNING: Do not use path/filepath in this package unless you want bizarre Windows paths "path" - "runtime" "strings" "time" @@ -37,7 +35,6 @@ import ( "github.com/golang/glog" "github.com/pkg/errors" "github.com/spf13/viper" - "golang.org/x/sync/errgroup" meta "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" @@ -45,6 +42,7 @@ import ( "k8s.io/minikube/pkg/kapi" "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper" + "k8s.io/minikube/pkg/minikube/bootstrapper/bsutil" "k8s.io/minikube/pkg/minikube/bootstrapper/images" "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" @@ -53,46 +51,18 @@ import ( "k8s.io/minikube/pkg/minikube/machine" "k8s.io/minikube/pkg/minikube/out" "k8s.io/minikube/pkg/minikube/vmpath" - "k8s.io/minikube/pkg/util" "k8s.io/minikube/pkg/util/retry" ) // enum to differentiate kubeadm command line parameters from kubeadm config file parameters (see the // KubeadmExtraArgsWhitelist variable below for more info) const ( - KubeadmCmdParam = iota - KubeadmConfigParam = iota defaultCNIConfigPath = "/etc/cni/net.d/k8s.conf" kubeletServiceFile = "/lib/systemd/system/kubelet.service" kubeletSystemdConfFile = "/etc/systemd/system/kubelet.service.d/10-kubeadm.conf" ) -const ( - // Container runtimes - remoteContainerRuntime = "remote" -) - -// KubeadmExtraArgsWhitelist is a whitelist of supported kubeadm params that can be supplied to kubeadm through -// minikube's ExtraArgs parameter. The list is split into two parts - params that can be supplied as flags on the -// command line and params that have to be inserted into the kubeadm config file. This is because of a kubeadm -// constraint which allows only certain params to be provided from the command line when the --config parameter -// is specified -var KubeadmExtraArgsWhitelist = map[int][]string{ - KubeadmCmdParam: { - "ignore-preflight-errors", - "dry-run", - "kubeconfig", - "kubeconfig-dir", - "node-name", - "cri-socket", - "experimental-upload-certs", - "certificate-key", - "rootfs", - }, - KubeadmConfigParam: { - "pod-network-cidr", - }, -} +const () // remote artifacts that must exist for minikube to function properly. The sign of a previously working installation. // NOTE: /etc is not persistent across restarts, so don't bother checking there @@ -192,21 +162,6 @@ func (k *Bootstrapper) LogCommands(o bootstrapper.LogOptions) map[string]string } } -// createFlagsFromExtraArgs converts kubeadm extra args into flags to be supplied from the command linne -func createFlagsFromExtraArgs(extraOptions config.ExtraOptionSlice) string { - kubeadmExtraOpts := extraOptions.AsMap().Get(Kubeadm) - - // kubeadm allows only a small set of parameters to be supplied from the command line when the --config param - // is specified, here we remove those that are not allowed - for opt := range kubeadmExtraOpts { - if !config.ContainsParam(KubeadmExtraArgsWhitelist[KubeadmCmdParam], opt) { - // kubeadmExtraOpts is a copy so safe to delete - delete(kubeadmExtraOpts, opt) - } - } - return convertToFlags(kubeadmExtraOpts) -} - // etcdDataDir is where etcd data is stored. func etcdDataDir() string { return path.Join(vmpath.GuestPersistentDir, "etcd") @@ -249,12 +204,12 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { glog.Infof("StartCluster complete in %s", time.Since(start)) }() - version, err := parseKubernetesVersion(k8s.KubernetesVersion) + version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing kubernetes version") } - extraFlags := createFlagsFromExtraArgs(k8s.ExtraOptions) + extraFlags := bsutil.CreateFlagsFromExtraArgs(k8s.ExtraOptions) r, err := cruntime.New(cruntime.Config{Type: k8s.ContainerRuntime}) if err != nil { return err @@ -279,7 +234,7 @@ func (k *Bootstrapper) StartCluster(k8s config.KubernetesConfig) error { ignore = append(ignore, "SystemVerification") } - c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", invokeKubeadm(k8s.KubernetesVersion), yamlConfigPath, extraFlags, strings.Join(ignore, ","))) + c := exec.Command("/bin/bash", "-c", fmt.Sprintf("%s init --config %s %s --ignore-preflight-errors=%s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), yamlConfigPath, extraFlags, strings.Join(ignore, ","))) if rr, err := k.c.RunCmd(c); err != nil { return errors.Wrapf(err, "init failed. cmd: %q", rr.Command()) } @@ -481,7 +436,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { glog.Infof("restartCluster took %s", time.Since(start)) }() - version, err := parseKubernetesVersion(k8s.KubernetesVersion) + version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing kubernetes version") } @@ -497,7 +452,7 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { glog.Errorf("failed to create compat symlinks: %v", err) } - baseCmd := fmt.Sprintf("%s %s", invokeKubeadm(k8s.KubernetesVersion), phase) + baseCmd := fmt.Sprintf("%s %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), phase) cmds := []string{ fmt.Sprintf("%s phase certs all --config %s", baseCmd, yamlConfigPath), fmt.Sprintf("%s phase kubeconfig all --config %s", baseCmd, yamlConfigPath), @@ -534,14 +489,14 @@ func (k *Bootstrapper) restartCluster(k8s config.KubernetesConfig) error { // DeleteCluster removes the components that were started earlier func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { - version, err := parseKubernetesVersion(k8s.KubernetesVersion) + version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing kubernetes version") } - cmd := fmt.Sprintf("%s reset --force", invokeKubeadm(k8s.KubernetesVersion)) + cmd := fmt.Sprintf("%s reset --force", bsutil.InvokeKubeadm(k8s.KubernetesVersion)) if version.LT(semver.MustParse("1.11.0")) { - cmd = fmt.Sprintf("%s reset", invokeKubeadm(k8s.KubernetesVersion)) + cmd = fmt.Sprintf("%s reset", bsutil.InvokeKubeadm(k8s.KubernetesVersion)) } if rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", cmd)); err != nil { @@ -553,7 +508,7 @@ func (k *Bootstrapper) DeleteCluster(k8s config.KubernetesConfig) error { // PullImages downloads images that will be used by Kubernetes func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error { - version, err := parseKubernetesVersion(k8s.KubernetesVersion) + version, err := bsutil.ParseKubernetesVersion(k8s.KubernetesVersion) if err != nil { return errors.Wrap(err, "parsing kubernetes version") } @@ -561,7 +516,7 @@ func (k *Bootstrapper) PullImages(k8s config.KubernetesConfig) error { return fmt.Errorf("pull command is not supported by kubeadm v%s", version) } - rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s config images pull --config %s", invokeKubeadm(k8s.KubernetesVersion), yamlConfigPath))) + rr, err := k.c.RunCmd(exec.Command("/bin/bash", "-c", fmt.Sprintf("%s config images pull --config %s", bsutil.InvokeKubeadm(k8s.KubernetesVersion), yamlConfigPath))) if err != nil { return errors.Wrapf(err, "running cmd: %q", rr.Command()) } @@ -573,61 +528,6 @@ func (k *Bootstrapper) SetupCerts(k8s config.KubernetesConfig) error { return bootstrapper.SetupCerts(k.c, k8s) } -// NewKubeletConfig generates a new systemd unit containing a configured kubelet -// based on the options present in the KubernetesConfig. -func NewKubeletConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) { - version, err := parseKubernetesVersion(k8s.KubernetesVersion) - if err != nil { - return nil, errors.Wrap(err, "parsing kubernetes version") - } - - extraOpts, err := ExtraConfigForComponent(Kubelet, k8s.ExtraOptions, version) - if err != nil { - return nil, errors.Wrap(err, "generating extra configuration for kubelet") - } - - for k, v := range r.KubeletOptions() { - extraOpts[k] = v - } - if k8s.NetworkPlugin != "" { - extraOpts["network-plugin"] = k8s.NetworkPlugin - } - if _, ok := extraOpts["node-ip"]; !ok { - extraOpts["node-ip"] = k8s.NodeIP - } - - pauseImage := images.Pause(k8s.ImageRepository) - if _, ok := extraOpts["pod-infra-container-image"]; !ok && k8s.ImageRepository != "" && pauseImage != "" && k8s.ContainerRuntime != remoteContainerRuntime { - extraOpts["pod-infra-container-image"] = pauseImage - } - - // parses a map of the feature gates for kubelet - _, kubeletFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates) - if err != nil { - return nil, errors.Wrap(err, "parses feature gate config for kubelet") - } - - if kubeletFeatureArgs != "" { - extraOpts["feature-gates"] = kubeletFeatureArgs - } - - b := bytes.Buffer{} - opts := struct { - ExtraOptions string - ContainerRuntime string - KubeletPath string - }{ - ExtraOptions: convertToFlags(extraOpts), - ContainerRuntime: k8s.ContainerRuntime, - KubeletPath: path.Join(binRoot(k8s.KubernetesVersion), "kubelet"), - } - if err := kubeletSystemdTemplate.Execute(&b, opts); err != nil { - return nil, err - } - - return b.Bytes(), nil -} - // UpdateCluster updates the cluster func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { images, err := images.Kubeadm(cfg.KubernetesConfig.ImageRepository, cfg.KubernetesConfig.KubernetesVersion) @@ -644,17 +544,17 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { if err != nil { return errors.Wrap(err, "runtime") } - kubeadmCfg, err := generateConfig(cfg.KubernetesConfig, r) + kubeadmCfg, err := bsutil.GenerateKubeadmYAML(cfg.KubernetesConfig, r) if err != nil { return errors.Wrap(err, "generating kubeadm cfg") } - kubeletCfg, err := NewKubeletConfig(cfg.KubernetesConfig, r) + kubeletCfg, err := bsutil.NewKubeletConfig(cfg.KubernetesConfig, r) if err != nil { return errors.Wrap(err, "generating kubelet config") } - kubeletService, err := NewKubeletService(cfg.KubernetesConfig) + kubeletService, err := bsutil.NewKubeletService(cfg.KubernetesConfig) if err != nil { return errors.Wrap(err, "generating kubelet service") } @@ -667,7 +567,7 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { glog.Warningf("unable to stop kubelet: %s command: %q output: %q", err, rr.Command(), rr.Output()) } - if err := transferBinaries(cfg.KubernetesConfig, k.c); err != nil { + if err := bsutil.TransferBinaries(cfg.KubernetesConfig, k.c); err != nil { return errors.Wrap(err, "downloading binaries") } files := configFiles(cfg.KubernetesConfig, kubeadmCfg, kubeletCfg, kubeletService) @@ -686,112 +586,6 @@ func (k *Bootstrapper) UpdateCluster(cfg config.MachineConfig) error { return nil } -// createExtraComponentConfig generates a map of component to extra args for all of the components except kubeadm -func createExtraComponentConfig(extraOptions config.ExtraOptionSlice, version semver.Version, componentFeatureArgs string) ([]ComponentExtraArgs, error) { - extraArgsSlice, err := NewComponentExtraArgs(extraOptions, version, componentFeatureArgs) - if err != nil { - return nil, err - } - - // kubeadm extra args should not be included in the kubeadm config in the extra args section (instead, they must - // be inserted explicitly in the appropriate places or supplied from the command line); here we remove all of the - // kubeadm extra args from the slice - for i, extraArgs := range extraArgsSlice { - if extraArgs.Component == Kubeadm { - extraArgsSlice = append(extraArgsSlice[:i], extraArgsSlice[i+1:]...) - break - } - } - return extraArgsSlice, nil -} - -// generateConfig generates the kubeadm.yaml file -func generateConfig(k8s config.KubernetesConfig, r cruntime.Manager) ([]byte, error) { - version, err := parseKubernetesVersion(k8s.KubernetesVersion) - if err != nil { - return nil, errors.Wrap(err, "parsing kubernetes version") - } - - // parses a map of the feature gates for kubeadm and component - kubeadmFeatureArgs, componentFeatureArgs, err := ParseFeatureArgs(k8s.FeatureGates) - if err != nil { - return nil, errors.Wrap(err, "parses feature gate config for kubeadm and component") - } - - extraComponentConfig, err := createExtraComponentConfig(k8s.ExtraOptions, version, componentFeatureArgs) - if err != nil { - return nil, errors.Wrap(err, "generating extra component config for kubeadm") - } - - // In case of no port assigned, use util.APIServerPort - nodePort := k8s.NodePort - if nodePort <= 0 { - nodePort = constants.APIServerPort - } - - opts := struct { - CertDir string - ServiceCIDR string - PodSubnet string - AdvertiseAddress string - APIServerPort int - KubernetesVersion string - EtcdDataDir string - NodeName string - DNSDomain string - CRISocket string - ImageRepository string - ExtraArgs []ComponentExtraArgs - FeatureArgs map[string]bool - NoTaintMaster bool - }{ - CertDir: vmpath.GuestCertsDir, - ServiceCIDR: util.DefaultServiceCIDR, - PodSubnet: k8s.ExtraOptions.Get("pod-network-cidr", Kubeadm), - AdvertiseAddress: k8s.NodeIP, - APIServerPort: nodePort, - KubernetesVersion: k8s.KubernetesVersion, - EtcdDataDir: etcdDataDir(), - NodeName: k8s.NodeName, - CRISocket: r.SocketPath(), - ImageRepository: k8s.ImageRepository, - ExtraArgs: extraComponentConfig, - FeatureArgs: kubeadmFeatureArgs, - NoTaintMaster: false, // That does not work with k8s 1.12+ - DNSDomain: k8s.DNSDomain, - } - - if k8s.ServiceCIDR != "" { - opts.ServiceCIDR = k8s.ServiceCIDR - } - - opts.NoTaintMaster = true - b := bytes.Buffer{} - configTmpl := configTmplV1Alpha1 - if version.GTE(semver.MustParse("1.12.0")) { - configTmpl = configTmplV1Alpha3 - } - // v1beta1 works in v1.13, but isn't required until v1.14. - if version.GTE(semver.MustParse("1.14.0-alpha.0")) { - configTmpl = configTmplV1Beta1 - } - if err := configTmpl.Execute(&b, opts); err != nil { - return nil, err - } - - return b.Bytes(), nil -} - -// NewKubeletService returns a generated systemd unit file for the kubelet -func NewKubeletService(cfg config.KubernetesConfig) ([]byte, error) { - var b bytes.Buffer - opts := struct{ KubeletPath string }{KubeletPath: path.Join(binRoot(cfg.KubernetesVersion), "kubelet")} - if err := kubeletServiceTemplate.Execute(&b, opts); err != nil { - return nil, errors.Wrap(err, "template execute") - } - return b.Bytes(), nil -} - // configFiles returns configuration file assets func configFiles(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byte, kubeletSvc []byte) []assets.CopyableFile { fs := []assets.CopyableFile{ @@ -807,34 +601,3 @@ func configFiles(cfg config.KubernetesConfig, kubeadm []byte, kubelet []byte, ku } return fs } - -// binDir returns the persistent path binaries are stored in -func binRoot(version string) string { - return path.Join(vmpath.GuestPersistentDir, "binaries", version) -} - -// invokeKubeadm returns the invocation command for Kubeadm -func invokeKubeadm(version string) string { - return fmt.Sprintf("sudo env PATH=%s:$PATH kubeadm", binRoot(version)) -} - -// transferBinaries transfers all required Kubernetes binaries -func transferBinaries(cfg config.KubernetesConfig, c command.Runner) error { - var g errgroup.Group - for _, name := range constants.KubeadmBinaries { - name := name - g.Go(func() error { - src, err := machine.CacheBinary(name, cfg.KubernetesVersion, "linux", runtime.GOARCH) - if err != nil { - return errors.Wrapf(err, "downloading %s", name) - } - - dst := path.Join(binRoot(cfg.KubernetesVersion), name) - if err := machine.CopyBinary(c, src, dst); err != nil { - return errors.Wrapf(err, "copybinary %s -> %s", src, dst) - } - return nil - }) - } - return g.Wait() -} diff --git a/pkg/minikube/cluster/cluster.go b/pkg/minikube/cluster/cluster.go index 40f63d748..e116e2d50 100644 --- a/pkg/minikube/cluster/cluster.go +++ b/pkg/minikube/cluster/cluster.go @@ -173,7 +173,6 @@ func configureHost(h *host.Host, e *engine.Options) error { // ensureGuestClockSync ensures that the guest system clock is relatively in-sync func ensureSyncedGuestClock(h hostRunner) error { - fmt.Println("Inside ensureSyncedGuestClock") d, err := guestClockDelta(h, time.Now()) if err != nil { glog.Warningf("Unable to measure system clock delta: %v", err) -- GitLab