Refactor preload_images.go to generate and upload a preloaded tarball

This script is now responsible for making sure that the DefaultK8sVersion, NewestK8sVersion, and OldestK8sVersion supported by minikube have preloaded tarballs. It will be run on every PR as an automated release script for preloaded tarballs.

Makefile

@@ -526,14 +526,8 @@ kic-base-image: ## builds the base image used for kic.
 	docker build -f ./hack/images/kicbase.Dockerfile -t $(REGISTRY)/kicbase:$(KIC_VERSION)-snapshot  --build-arg COMMIT_SHA=${VERSION}-$(COMMIT) --target base .
 
 .PHONY: upload-preloaded-images-tar
-upload-preloaded-images-tar: generate-preloaded-images-tar # Upload the preloaded images tar to the GCS bucket. Specify a specific kubernetes version to build via `KUBERNETES_VERSION=vx.y.z make upload-preloaded-images-tar`.
-	gsutil cp out/preloaded-images-k8s-${PRELOADED_TARBALL_VERSION}-${KUBERNETES_VERSION}-docker-overlay2.tar.lz4 gs://${PRELOADED_VOLUMES_GCS_BUCKET}
-	gsutil acl ch -u AllUsers:R gs://${PRELOADED_VOLUMES_GCS_BUCKET}/preloaded-images-k8s-${PRELOADED_TARBALL_VERSION}-${KUBERNETES_VERSION}-docker-overlay2.tar.lz4 
-
-.PHONY: generate-preloaded-images-tar
-generate-preloaded-images-tar:
-	go run ./hack/preload-images/preload_images.go -kubernetes-version ${KUBERNETES_VERSION} -preloaded-tarball-version ${PRELOADED_TARBALL_VERSION}
-
+upload-preloaded-images-tar: out/minikube # Upload the preloaded images for oldest supported, newest supported, and default kubernetes versions to GCS.
+	go run ./hack/preload-images/*.go 
 
 .PHONY: push-storage-provisioner-image
 push-storage-provisioner-image: storage-provisioner-image ## Push storage-provisioner docker image using gcloud

hack/preload-images/generate.go

+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/drivers/kic"
+	"k8s.io/minikube/pkg/drivers/kic/oci"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
+	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
+	"k8s.io/minikube/pkg/minikube/command"
+	"k8s.io/minikube/pkg/minikube/config"
+	"k8s.io/minikube/pkg/minikube/driver"
+	"k8s.io/minikube/pkg/minikube/localpath"
+)
+
+func generateTarball(kubernetesVersion, tarballFilename string) error {
+	defer func() {
+		if err := deleteMinikube(); err != nil {
+			fmt.Println(err)
+		}
+	}()
+
+	driver := kic.NewDriver(kic.Config{
+		KubernetesVersion: kubernetesVersion,
+		ContainerRuntime:  driver.Docker,
+		OCIBinary:         oci.Docker,
+		MachineName:       profile,
+		ImageDigest:       kic.BaseImage,
+		StorePath:         localpath.MiniPath(),
+		CPU:               2,
+		Memory:            4000,
+		APIServerPort:     8080,
+	})
+
+	baseDir := filepath.Dir(driver.GetSSHKeyPath())
+	defer os.Remove(baseDir)
+
+	if err := os.MkdirAll(baseDir, 0755); err != nil {
+		return errors.Wrap(err, "mkdir")
+	}
+	if err := driver.Create(); err != nil {
+		return errors.Wrap(err, "creating kic driver")
+	}
+
+	// Now, get images to pull
+	imgs, err := images.Kubeadm("", kubernetesVersion)
+	if err != nil {
+		return errors.Wrap(err, "kubeadm images")
+	}
+
+	for _, img := range append(imgs, kic.OverlayImage) {
+		cmd := exec.Command("docker", "exec", profile, "docker", "pull", img)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			return errors.Wrapf(err, "downloading %s", img)
+		}
+	}
+
+	// Transfer in k8s binaries
+	kcfg := config.KubernetesConfig{
+		KubernetesVersion: kubernetesVersion,
+	}
+	runner := command.NewKICRunner(profile, driver.OCIBinary)
+	if err := bsutil.TransferBinaries(kcfg, runner); err != nil {
+		return errors.Wrap(err, "transferring k8s binaries")
+	}
+	// Create image tarball
+	if err := createImageTarball(tarballFilename); err != nil {
+		return errors.Wrap(err, "create tarball")
+	}
+	return copyTarballToHost(tarballFilename)
+}
+
+func createImageTarball(tarballFilename string) error {
+	dirs := []string{
+		fmt.Sprintf("./lib/docker/%s", dockerStorageDriver),
+		"./lib/docker/image",
+		"./lib/minikube/binaries",
+	}
+	args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename}
+	args = append(args, dirs...)
+	cmd := exec.Command("docker", args...)
+	cmd.Stdout = os.Stdout
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "tarball cmd: %s", cmd.Args)
+	}
+	return nil
+}
+
+func copyTarballToHost(tarballFilename string) error {
+	dest := filepath.Join("out/", tarballFilename)
+	cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:/%s", profile, tarballFilename), dest)
+	cmd.Stdout = os.Stdout
+	if err := cmd.Run(); err != nil {
+		return errors.Wrapf(err, "cp cmd: %s", cmd.Args)
+	}
+	return nil
+}
+
+func deleteMinikube() error {
+	cmd := exec.Command(minikubePath, "delete", "-p", profile)
+	cmd.Stdout = os.Stdout
+	return cmd.Run()
+}

hack/preload-images/preload_images.go

@@ -18,22 +18,13 @@ package main
 
 import (
 	"bytes"
-	"flag"
 	"fmt"
-	"os"
 	"os/exec"
-	"path/filepath"
 	"strings"
 
-	"github.com/pkg/errors"
-	"k8s.io/minikube/pkg/drivers/kic"
-	"k8s.io/minikube/pkg/drivers/kic/oci"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/bsutil"
-	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
-	"k8s.io/minikube/pkg/minikube/command"
-	"k8s.io/minikube/pkg/minikube/config"
-	"k8s.io/minikube/pkg/minikube/driver"
-	"k8s.io/minikube/pkg/minikube/localpath"
+	"k8s.io/minikube/pkg/minikube/constants"
+	"k8s.io/minikube/pkg/minikube/download"
+	"k8s.io/minikube/pkg/minikube/exit"
 )
 
 const (
@@ -42,123 +33,47 @@ const (
 )
 
 var (
-	kubernetesVersion       = ""
-	tarballFilename         = ""
-	dockerStorageDriver     = ""
-	preloadedTarballVersion = ""
-	containerRuntime        = ""
+	dockerStorageDriver     = "overlay2"
+	preloadedTarballVersion = "v1"
+	containerRuntimes       = []string{"docker"}
+	kubernetesVersions      = []string{
+		constants.OldestKubernetesVersion,
+		constants.DefaultKubernetesVersion,
+		constants.NewestKubernetesVersion,
+	}
 )
 
-func init() {
-	flag.StringVar(&kubernetesVersion, "kubernetes-version", "", "desired kubernetes version, for example `v1.17.2`")
-	flag.StringVar(&dockerStorageDriver, "docker-storage-driver", "overlay2", "docker storage driver backend")
-	flag.StringVar(&preloadedTarballVersion, "preloaded-tarball-version", "", "preloaded tarball version")
-	flag.StringVar(&containerRuntime, "container-runtime", "docker", "container runtime")
-
-	flag.Parse()
-	tarballFilename = fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s.tar.lz4", preloadedTarballVersion, kubernetesVersion, containerRuntime, dockerStorageDriver)
-}
-
 func main() {
 	if err := verifyDockerStorage(); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-	if err := executePreloadImages(); err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-}
-
-func executePreloadImages() error {
-	defer func() {
-		if err := deleteMinikube(); err != nil {
-			fmt.Println(err)
-		}
-	}()
-
-	driver := kic.NewDriver(kic.Config{
-		KubernetesVersion: kubernetesVersion,
-		ContainerRuntime:  driver.Docker,
-		OCIBinary:         oci.Docker,
-		MachineName:       profile,
-		ImageDigest:       kic.BaseImage,
-		StorePath:         localpath.MiniPath(),
-		CPU:               2,
-		Memory:            4000,
-		APIServerPort:     8080,
-	})
-
-	baseDir := filepath.Dir(driver.GetSSHKeyPath())
-	defer os.Remove(baseDir)
-
-	if err := os.MkdirAll(baseDir, 0755); err != nil {
-		return errors.Wrap(err, "mkdir")
-	}
-	if err := driver.Create(); err != nil {
-		return errors.Wrap(err, "creating kic driver")
-	}
-
-	// Now, get images to pull
-	imgs, err := images.Kubeadm("", kubernetesVersion)
-	if err != nil {
-		return errors.Wrap(err, "kubeadm images")
+		exit.WithError("Docker storage type is incompatible: %v\n", err)
 	}
-
-	for _, img := range append(imgs, kic.OverlayImage) {
-		cmd := exec.Command("docker", "exec", profile, "docker", "pull", img)
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		if err := cmd.Run(); err != nil {
-			return errors.Wrapf(err, "downloading %s", img)
+	for _, kubernetesVersion := range kubernetesVersions {
+		for _, cr := range containerRuntimes {
+			tf := tarballFilename(kubernetesVersion, cr)
+			if tarballExists(tf) {
+				fmt.Printf("A preloaded tarball for k8s version %s already exists, skipping generation.\n", kubernetesVersion)
+				continue
+			}
+			fmt.Printf("A preloaded tarball for k8s version %s doesn't exist, generating now...\n", kubernetesVersion)
+			if err := generateTarball(kubernetesVersion, tf); err != nil {
+				exit.WithError(fmt.Sprintf("generating tarball for k8s version %s with %s", kubernetesVersion, cr), err)
+			}
+			if err := uploadTarball(tf); err != nil {
+				exit.WithError(fmt.Sprintf("uploading tarball for k8s version %s with %s", kubernetesVersion, cr), err)
+			}
 		}
 	}
-
-	// Transfer in k8s binaries
-	kcfg := config.KubernetesConfig{
-		KubernetesVersion: kubernetesVersion,
-	}
-	runner := command.NewKICRunner(profile, driver.OCIBinary)
-	if err := bsutil.TransferBinaries(kcfg, runner); err != nil {
-		return errors.Wrap(err, "transferring k8s binaries")
-	}
-	// Create image tarball
-	if err := createImageTarball(); err != nil {
-		return errors.Wrap(err, "create tarball")
-	}
-	return copyTarballToHost()
 }
 
-func createImageTarball() error {
-	dirs := []string{
-		fmt.Sprintf("./lib/docker/%s", dockerStorageDriver),
-		"./lib/docker/image",
-		"./lib/minikube/binaries",
-	}
-	args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename}
-	args = append(args, dirs...)
-	cmd := exec.Command("docker", args...)
-	cmd.Stdout = os.Stdout
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "tarball cmd: %s", cmd.Args)
-	}
-	return nil
-}
-
-func copyTarballToHost() error {
-	dest := filepath.Join("out/", tarballFilename)
-	cmd := exec.Command("docker", "cp", fmt.Sprintf("%s:/%s", profile, tarballFilename), dest)
-	cmd.Stdout = os.Stdout
-	if err := cmd.Run(); err != nil {
-		return errors.Wrapf(err, "cp cmd: %s", cmd.Args)
-	}
-	return nil
+func tarballFilename(kubernetesVersion string, containerRuntime string) string {
+	return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s.tar.lz4", preloadedTarballVersion, kubernetesVersion, containerRuntime, dockerStorageDriver)
 }
 
-func deleteMinikube() error {
-	cmd := exec.Command(minikubePath, "delete", "-p", profile)
-	cmd.Stdout = os.Stdout
-	return cmd.Run()
+func tarballExists(tarballFilename string) bool {
+	fmt.Println("Checking if tarball already exists...")
+	gcsPath := fmt.Sprintf("gs://%s/%s", download.PreloadBucket, tarballFilename)
+	cmd := exec.Command("gsutil", "stat", gcsPath)
+	return cmd.Run() == nil
 }
 
 func verifyDockerStorage() error {

hack/preload-images/upload.go

+/*
+Copyright 2020 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"path"
+
+	"github.com/pkg/errors"
+	"k8s.io/minikube/pkg/minikube/download"
+)
+
+func uploadTarball(tarballFilename string) error {
+	// Upload tarball to GCS
+	hostPath := path.Join("out/", tarballFilename)
+	gcsDest := fmt.Sprintf("gs://%s", download.PreloadBucket)
+	cmd := exec.Command("gsutil", "cp", hostPath, gcsDest)
+	if output, err := cmd.Output(); err != nil {
+		return errors.Wrapf(err, "uploading %s to GCS bucket: %v\n%s", hostPath, err, string(output))
+	}
+	// Make tarball public to all users
+	gcsPath := fmt.Sprintf("%s/%s", gcsDest, tarballFilename)
+	cmd = exec.Command("gsutil", "acl", "ch", "-u", "AllUsers:R", gcsPath)
+	if output, err := cmd.Output(); err != nil {
+		return errors.Wrapf(err, "uploading %s to GCS bucket: %v\n%s", hostPath, err, string(output))
+	}
+	return nil
+}