diff --git a/hack/preload-images/generate.go b/hack/preload-images/generate.go index 572352cbe81b05a21768934f869edb72d182ae02..fe2517e293642a2a4c927277c3cecd3efe72865f 100644 --- a/hack/preload-images/generate.go +++ b/hack/preload-images/generate.go @@ -60,6 +60,10 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string return errors.Wrap(err, "creating kic driver") } + if err := verifyStorage(containerRuntime); err != nil { + return errors.Wrap(err, "verifying storage") + } + // Now, get images to pull imgs, err := images.Kubeadm("", kubernetesVersion) if err != nil { @@ -128,6 +132,20 @@ func generateTarball(kubernetesVersion, containerRuntime, tarballFilename string return copyTarballToHost(tarballFilename) } +func verifyStorage(containerRuntime string) error { + if containerRuntime == "docker" || containerRuntime == "containerd" { + if err := verifyDockerStorage(); err != nil { + return errors.Wrap(err, "Docker storage type is incompatible") + } + } + if containerRuntime == "cri-o" { + if err := verifyPodmanStorage(); err != nil { + return errors.Wrap(err, "Podman storage type is incompatible") + } + } + return nil +} + // returns the right command to pull image for a specific runtime func imagePullCommand(containerRuntime, img string) *exec.Cmd { if containerRuntime == "docker" { @@ -137,6 +155,10 @@ func imagePullCommand(containerRuntime, img string) *exec.Cmd { if containerRuntime == "containerd" { return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img) } + + if containerRuntime == "cri-o" { + return exec.Command("docker", "exec", profile, "sudo", "crictl", "pull", img) + } return nil } @@ -154,6 +176,10 @@ func createImageTarball(tarballFilename, containerRuntime string) error { dirs = append(dirs, fmt.Sprintf("./lib/containerd")) } + if containerRuntime == "cri-o" { + dirs = append(dirs, fmt.Sprintf("./lib/containers")) + } + args := []string{"exec", profile, "sudo", "tar", "-I", "lz4", "-C", "/var", "-cvf", tarballFilename} args = append(args, dirs...) cmd := exec.Command("docker", args...) diff --git a/hack/preload-images/preload_images.go b/hack/preload-images/preload_images.go index 09d62e421b4dd6aa6e204b1cb065e690c8b34e76..2a34e0f7f4e2f56b47ff883c6e707823e6b7a065 100644 --- a/hack/preload-images/preload_images.go +++ b/hack/preload-images/preload_images.go @@ -18,6 +18,7 @@ package main import ( "bytes" + "encoding/json" "flag" "fmt" "os" @@ -37,7 +38,8 @@ const ( var ( dockerStorageDriver = "overlay2" - containerRuntimes = []string{"docker", "containerd"} + podmanStorageDriver = "overlay" + containerRuntimes = []string{"docker", "containerd", "cri-o"} k8sVersion string k8sVersions []string ) @@ -62,9 +64,6 @@ func main() { fmt.Printf("error cleaning up minikube at start up: %v \n", err) } - if err := verifyDockerStorage(); err != nil { - exit("Docker storage type is incompatible: %v \n", err) - } if k8sVersions == nil { var err error k8sVersions, err = RecentK8sVersions() @@ -99,7 +98,7 @@ func main() { } func verifyDockerStorage() error { - cmd := exec.Command("docker", "info", "-f", "{{.Info.Driver}}") + cmd := exec.Command("docker", "exec", profile, "docker", "info", "-f", "{{.Info.Driver}}") var stderr bytes.Buffer cmd.Stderr = &stderr output, err := cmd.Output() @@ -113,6 +112,26 @@ func verifyDockerStorage() error { return nil } +func verifyPodmanStorage() error { + cmd := exec.Command("docker", "exec", profile, "sudo", "podman", "info", "-f", "json") + var stderr bytes.Buffer + cmd.Stderr = &stderr + output, err := cmd.Output() + if err != nil { + return fmt.Errorf("%v: %v:\n%s", cmd.Args, err, stderr.String()) + } + var info map[string]map[string]interface{} + err = json.Unmarshal(output, &info) + if err != nil { + return err + } + driver := info["store"]["graphDriverName"] + if driver != podmanStorageDriver { + return fmt.Errorf("podman storage driver %s does not match requested %s", driver, podmanStorageDriver) + } + return nil +} + // exit will exit and clean up minikube func exit(msg string, err error) { fmt.Printf("WithError(%s)=%v called from:\n%s", msg, err, debug.Stack()) diff --git a/pkg/minikube/cruntime/containerd.go b/pkg/minikube/cruntime/containerd.go index 94bdc426af3b2b327c829beda2a361f7ea9f92dd..50d59bbd57daa93f3ea6cf0fdaec8c6f3e003edb 100644 --- a/pkg/minikube/cruntime/containerd.go +++ b/pkg/minikube/cruntime/containerd.go @@ -370,7 +370,7 @@ func containerdImagesPreloaded(runner command.Runner, images []string) bool { if err != nil { return false } - type containerdImages struct { + type criImages struct { Images []struct { ID string `json:"id"` RepoTags []string `json:"repoTags"` @@ -381,7 +381,7 @@ func containerdImagesPreloaded(runner command.Runner, images []string) bool { } `json:"images"` } - var jsonImages containerdImages + var jsonImages criImages err = json.Unmarshal(rr.Stdout.Bytes(), &jsonImages) if err != nil { glog.Errorf("failed to unmarshal images, will assume images are not preloaded") @@ -412,15 +412,3 @@ func containerdImagesPreloaded(runner command.Runner, images []string) bool { glog.Infof("all images are preloaded for containerd runtime.") return true } - -// addRepoTagToImageName makes sure the image name has a repo tag in it. -// in crictl images list have the repo tag prepended to them -// for example "kubernetesui/dashboard:v2.0.0 will show up as "docker.io/kubernetesui/dashboard:v2.0.0" -// warning this is only meant for kuberentes images where we know the GCR addreses have .io in them -// not mean to be used for public images -func addRepoTagToImageName(imgName string) string { - if !strings.Contains(imgName, ".io/") { - return "docker.io/" + imgName - } // else it already has repo name dont add anything - return imgName -} diff --git a/pkg/minikube/cruntime/cri.go b/pkg/minikube/cruntime/cri.go index 6f05551235c4a9a4a839766cf22135836528afdd..32bbb0f566340bda179088db1f97d3ac07f24d94 100644 --- a/pkg/minikube/cruntime/cri.go +++ b/pkg/minikube/cruntime/cri.go @@ -259,3 +259,15 @@ func criContainerLogCmd(cr CommandRunner, id string, len int, follow bool) strin cmd.WriteString(id) return cmd.String() } + +// addRepoTagToImageName makes sure the image name has a repo tag in it. +// in crictl images list have the repo tag prepended to them +// for example "kubernetesui/dashboard:v2.0.0 will show up as "docker.io/kubernetesui/dashboard:v2.0.0" +// warning this is only meant for kuberentes images where we know the GCR addreses have .io in them +// not mean to be used for public images +func addRepoTagToImageName(imgName string) string { + if !strings.Contains(imgName, ".io/") { + return "docker.io/" + imgName + } // else it already has repo name dont add anything + return imgName +} diff --git a/pkg/minikube/cruntime/crio.go b/pkg/minikube/cruntime/crio.go index 65c9442111626e0e7a90058a2b3986ba0b609c5c..f3a2022824378847bb05d47e057f5b48760ad516 100644 --- a/pkg/minikube/cruntime/crio.go +++ b/pkg/minikube/cruntime/crio.go @@ -17,15 +17,20 @@ limitations under the License. package cruntime import ( + "encoding/json" "fmt" "net" "os/exec" + "path" "strings" + "time" "github.com/blang/semver" "github.com/golang/glog" "github.com/pkg/errors" + "k8s.io/minikube/pkg/minikube/assets" "k8s.io/minikube/pkg/minikube/bootstrapper/images" + "k8s.io/minikube/pkg/minikube/command" "k8s.io/minikube/pkg/minikube/config" "k8s.io/minikube/pkg/minikube/download" "k8s.io/minikube/pkg/minikube/out" @@ -222,7 +227,103 @@ func (r *CRIO) Preload(cfg config.KubernetesConfig) error { if !download.PreloadExists(cfg.KubernetesVersion, cfg.ContainerRuntime) { return nil } - return fmt.Errorf("not yet implemented for %s", r.Name()) + + k8sVersion := cfg.KubernetesVersion + cRuntime := cfg.ContainerRuntime + + // If images already exist, return + images, err := images.Kubeadm(cfg.ImageRepository, k8sVersion) + if err != nil { + return errors.Wrap(err, "getting images") + } + if crioImagesPreloaded(r.Runner, images) { + glog.Info("Images already preloaded, skipping extraction") + return nil + } + + tarballPath := download.TarballPath(k8sVersion, cRuntime) + targetDir := "/" + targetName := "preloaded.tar.lz4" + dest := path.Join(targetDir, targetName) + + c := exec.Command("which", "lz4") + if _, err := r.Runner.RunCmd(c); err != nil { + return NewErrISOFeature("lz4") + } + + // Copy over tarball into host + fa, err := assets.NewFileAsset(tarballPath, targetDir, targetName, "0644") + if err != nil { + return errors.Wrap(err, "getting file asset") + } + t := time.Now() + if err := r.Runner.Copy(fa); err != nil { + return errors.Wrap(err, "copying file") + } + glog.Infof("Took %f seconds to copy over tarball", time.Since(t).Seconds()) + + t = time.Now() + // extract the tarball to /var in the VM + if rr, err := r.Runner.RunCmd(exec.Command("sudo", "tar", "-I", "lz4", "-C", "/var", "-xvf", dest)); err != nil { + return errors.Wrapf(err, "extracting tarball: %s", rr.Output()) + } + glog.Infof("Took %f seconds t extract the tarball", time.Since(t).Seconds()) + + // remove the tarball in the VM + if err := r.Runner.Remove(fa); err != nil { + glog.Infof("error removing tarball: %v", err) + } + + return nil +} + +// crioImagesPreloaded returns true if all images have been preloaded +func crioImagesPreloaded(runner command.Runner, images []string) bool { + rr, err := runner.RunCmd(exec.Command("sudo", "crictl", "images", "--output", "json")) + if err != nil { + return false + } + type criImages struct { + Images []struct { + ID string `json:"id"` + RepoTags []string `json:"repoTags"` + RepoDigests []string `json:"repoDigests"` + Size string `json:"size"` + UID interface{} `json:"uid"` + Username string `json:"username"` + } `json:"images"` + } + + var jsonImages criImages + err = json.Unmarshal(rr.Stdout.Bytes(), &jsonImages) + if err != nil { + glog.Errorf("failed to unmarshal images, will assume images are not preloaded") + return false + } + + // Make sure images == imgs + for _, i := range images { + found := false + for _, ji := range jsonImages.Images { + for _, rt := range ji.RepoTags { + i = addRepoTagToImageName(i) + if i == rt { + found = true + break + } + } + if found { + break + } + + } + if !found { + glog.Infof("couldn't find preloaded image for %q. assuming images are not preloaded.", i) + return false + } + } + glog.Infof("all images are preloaded for crio runtime.") + return true } // UpdateCRIONet updates CRIO CNI network configuration and restarts it diff --git a/pkg/minikube/download/preload.go b/pkg/minikube/download/preload.go index cf8b97d94b72b400ae316b1cd111f7cb985b18a0..570a7d732b10891d64e0afa52be9bb727e909824 100644 --- a/pkg/minikube/download/preload.go +++ b/pkg/minikube/download/preload.go @@ -47,7 +47,16 @@ const ( // TarballName returns name of the tarball func TarballName(k8sVersion, containerRuntime string) string { - return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-overlay2-%s.tar.lz4", PreloadVersion, k8sVersion, containerRuntime, runtime.GOARCH) + if containerRuntime == "crio" { + containerRuntime = "cri-o" + } + var storageDriver string + if containerRuntime == "cri-o" { + storageDriver = "overlay" + } else { + storageDriver = "overlay2" + } + return fmt.Sprintf("preloaded-images-k8s-%s-%s-%s-%s-%s.tar.lz4", PreloadVersion, k8sVersion, containerRuntime, storageDriver, runtime.GOARCH) } // returns the name of the checksum file @@ -78,13 +87,6 @@ func remoteTarballURL(k8sVersion, containerRuntime string) string { // PreloadExists returns true if there is a preloaded tarball that can be used func PreloadExists(k8sVersion, containerRuntime string, forcePreload ...bool) bool { - // and https://github.com/kubernetes/minikube/issues/6934 - // to track status of adding crio - if containerRuntime == "crio" { - glog.Info("crio is not supported yet, skipping preload") - return false - } - // TODO (#8166): Get rid of the need for this and viper at all force := false if len(forcePreload) > 0 {