From ad97983d9ef9a0c9fa94dbbc047b3e6aa59617cc Mon Sep 17 00:00:00 2001 From: Jose Donizetti Date: Fri, 28 Jun 2019 16:15:05 -0300 Subject: [PATCH] # This is a combination of 8 commits. # This is the 1st commit message: Fix doc comment version.gitCommitID doc # This is the commit message #2: Add hyperkit doc # This is the commit message #3: Add commit id to docker-machine-driver-kvm2 version # This is the commit message #4: removed label selector for registry-proxy daemonset # This is the commit message #5: Add support to custom qemu uri on kvm2 driver # This is the commit message #6: Improve hyperkit vm stop # This is the commit message #7: Make virtualbox DNS settings configurable # This is the commit message #8: added integration tests for registry addon --- Makefile | 11 ++-- cmd/drivers/kvm/main.go | 5 +- cmd/minikube/cmd/start.go | 11 +++- .../addons/registry/registry-proxy.yaml.tmpl | 4 +- docs/drivers.md | 2 +- pkg/drivers/hyperkit/driver.go | 25 +++++++++- pkg/drivers/kvm/domain.go | 8 +-- pkg/drivers/kvm/kvm.go | 8 ++- pkg/drivers/kvm/network.go | 9 ++-- pkg/drivers/kvm/version.go | 8 +++ pkg/minikube/config/types.go | 5 +- pkg/minikube/drivers/kvm/driver.go | 2 +- pkg/minikube/drivers/kvm2/driver.go | 4 +- pkg/minikube/drivers/virtualbox/driver.go | 4 +- pkg/version/version.go | 2 +- test/integration/addons_test.go | 50 +++++++++++++++++++ test/integration/functional_test.go | 1 + test/integration/start_stop_delete_test.go | 1 + test/integration/util/util.go | 22 ++++++++ 19 files changed, 154 insertions(+), 28 deletions(-) diff --git a/Makefile b/Makefile index 720c51376..ccc6d1eb8 100755 --- a/Makefile +++ b/Makefile @@ -83,6 +83,9 @@ CMD_SOURCE_DIRS = cmd pkg SOURCE_DIRS = $(CMD_SOURCE_DIRS) test SOURCE_PACKAGES = ./cmd/... ./pkg/... ./test/... +# kvm2 ldflags +KVM2_LDFLAGS := -X k8s.io/minikube/pkg/drivers/kvm.version=$(VERSION) -X k8s.io/minikube/pkg/drivers/kvm.gitCommitID=$(COMMIT) + # $(call DOCKER, image, command) define DOCKER docker run --rm -e GOCACHE=/app/.cache -e IN_DOCKER=1 --user $(shell id -u):$(shell id -g) -w /app -v $(PWD):/app -v $(GOPATH):/go --entrypoint /bin/bash $(1) -c '$(2)' @@ -387,11 +390,11 @@ release-minikube: out/minikube checksum gsutil cp out/minikube-$(GOOS)-$(GOARCH).sha256 $(MINIKUBE_UPLOAD_LOCATION)/$(MINIKUBE_VERSION)/minikube-$(GOOS)-$(GOARCH).sha256 out/docker-machine-driver-kvm2: - go build \ - -installsuffix "static" \ - -ldflags "-X k8s.io/minikube/pkg/drivers/kvm.version=$(VERSION)" \ + go build \ + -installsuffix "static" \ + -ldflags="$(KVM2_LDFLAGS)" \ -tags libvirt.1.3.1 \ - -o $(BUILD_DIR)/docker-machine-driver-kvm2 \ + -o $(BUILD_DIR)/docker-machine-driver-kvm2 \ k8s.io/minikube/cmd/drivers/kvm chmod +X $@ diff --git a/cmd/drivers/kvm/main.go b/cmd/drivers/kvm/main.go index 2d9134bce..455404b7c 100644 --- a/cmd/drivers/kvm/main.go +++ b/cmd/drivers/kvm/main.go @@ -27,8 +27,9 @@ import ( ) func main() { - if len(os.Args) > 1 && os.Args[1] == "--version" { - fmt.Println(kvm.GetVersion()) + if len(os.Args) > 1 && os.Args[1] == "version" { + fmt.Println("version:", kvm.GetVersion()) + fmt.Println("commit:", kvm.GetGitCommitID()) return } diff --git a/cmd/minikube/cmd/start.go b/cmd/minikube/cmd/start.go index 0999d6916..309162120 100644 --- a/cmd/minikube/cmd/start.go +++ b/cmd/minikube/cmd/start.go @@ -77,6 +77,7 @@ const ( enableDefaultCNI = "enable-default-cni" hypervVirtualSwitch = "hyperv-virtual-switch" kvmNetwork = "kvm-network" + kvmQemuURI = "kvm-qemu-uri" keepContext = "keep-context" createMount = "mount" featureGates = "feature-gates" @@ -97,6 +98,8 @@ const ( embedCerts = "embed-certs" noVTXCheck = "no-vtx-check" downloadOnly = "download-only" + dnsProxy = "dns-proxy" + hostDNSResolver = "host-dns-resolver" ) var ( @@ -122,6 +125,7 @@ func init() { startCmd.Flags().String(hostOnlyCIDR, "192.168.99.1/24", "The CIDR to be used for the minikube VM (only supported with Virtualbox driver)") startCmd.Flags().String(hypervVirtualSwitch, "", "The hyperv virtual switch name. Defaults to first found. (only supported with HyperV driver)") startCmd.Flags().String(kvmNetwork, "default", "The KVM network name. (only supported with KVM driver)") + startCmd.Flags().String(kvmQemuURI, "qemu:///system", "The KVM QEMU connection URI. (works only with kvm2 driver on linux)") startCmd.Flags().String(xhyveDiskDriver, "ahci-hd", "The disk driver to use [ahci-hd|virtio-blk] (only supported with xhyve driver)") startCmd.Flags().StringSlice(nfsShare, []string{}, "Local folders to share with Guest via NFS mounts (Only supported on with hyperkit now)") startCmd.Flags().String(nfsSharesRoot, "/nfsshares", "Where to root the NFS Shares (defaults to /nfsshares, only supported with hyperkit now)") @@ -156,6 +160,8 @@ func init() { startCmd.Flags().Bool(gpu, false, "Enable experimental NVIDIA GPU support in minikube (works only with kvm2 driver on Linux)") startCmd.Flags().Bool(hidden, false, "Hide the hypervisor signature from the guest in minikube (works only with kvm2 driver on Linux)") startCmd.Flags().Bool(noVTXCheck, false, "Disable checking for the availability of hardware virtualization before the vm is started (virtualbox)") + startCmd.Flags().Bool(dnsProxy, false, "Enable proxy for NAT DNS requests (virtualbox)") + startCmd.Flags().Bool(hostDNSResolver, true, "Enable host resolver for NAT DNS requests (virtualbox)") if err := viper.BindPFlags(startCmd.Flags()); err != nil { exit.WithError("unable to bind flags", err) } @@ -520,13 +526,16 @@ func generateConfig(cmd *cobra.Command, k8sVersion string) (cfg.Config, error) { RegistryMirror: registryMirror, HostOnlyCIDR: viper.GetString(hostOnlyCIDR), HypervVirtualSwitch: viper.GetString(hypervVirtualSwitch), - KvmNetwork: viper.GetString(kvmNetwork), + KVMNetwork: viper.GetString(kvmNetwork), + KVMQemuURI: viper.GetString(kvmQemuURI), Downloader: pkgutil.DefaultDownloader{}, DisableDriverMounts: viper.GetBool(disableDriverMounts), UUID: viper.GetString(uuid), GPU: viper.GetBool(gpu), Hidden: viper.GetBool(hidden), NoVTXCheck: viper.GetBool(noVTXCheck), + DNSProxy: viper.GetBool(dnsProxy), + HostDNSResolver: viper.GetBool(hostDNSResolver), }, KubernetesConfig: cfg.KubernetesConfig{ KubernetesVersion: k8sVersion, diff --git a/deploy/addons/registry/registry-proxy.yaml.tmpl b/deploy/addons/registry/registry-proxy.yaml.tmpl index 0a818fd9c..2131be382 100644 --- a/deploy/addons/registry/registry-proxy.yaml.tmpl +++ b/deploy/addons/registry/registry-proxy.yaml.tmpl @@ -2,7 +2,7 @@ apiVersion: extensions/v1beta1 kind: DaemonSet metadata: labels: - kubernetes.io/minikube-addons: registry + kubernetes.io/minikube-addons: registry-proxy addonmanager.kubernetes.io/mode: Reconcile name: registry-proxy namespace: kube-system @@ -10,7 +10,7 @@ spec: template: metadata: labels: - kubernetes.io/minikube-addons: registry + kubernetes.io/minikube-addons: registry-proxy addonmanager.kubernetes.io/mode: Reconcile spec: containers: diff --git a/docs/drivers.md b/docs/drivers.md index dfdd3e2b1..9c2b3fccb 100644 --- a/docs/drivers.md +++ b/docs/drivers.md @@ -99,7 +99,7 @@ virsh net-start default Make sure you are running the lastest version of your driver. ```shell -docker-machine-driver-kvm2 --version +docker-machine-driver-kvm2 version ``` ## Hyperkit driver diff --git a/pkg/drivers/hyperkit/driver.go b/pkg/drivers/hyperkit/driver.go index 94a729d1f..884fe0fb9 100644 --- a/pkg/drivers/hyperkit/driver.go +++ b/pkg/drivers/hyperkit/driver.go @@ -54,6 +54,7 @@ const ( "sudo chown root:wheel %s && sudo chmod u+s %s" ) +// Driver is the machine driver for Hyperkit type Driver struct { *drivers.BaseDriver *pkgdrivers.CommonDriver @@ -69,6 +70,7 @@ type Driver struct { VSockPorts []string } +// NewDriver creates a new driver for a host func NewDriver(hostName, storePath string) *Driver { return &Driver{ BaseDriver: &drivers.BaseDriver{ @@ -98,6 +100,7 @@ func (d *Driver) verifyRootPermissions() error { return nil } +// Create a host using the driver's config func (d *Driver) Create() error { if err := d.verifyRootPermissions(); err != nil { return err @@ -194,6 +197,7 @@ func (d *Driver) Remove() error { return nil } +// Restart a host func (d *Driver) Restart() error { return pkgdrivers.Restart(d) } @@ -342,7 +346,26 @@ func (d *Driver) Stop() error { return err } d.cleanupNfsExports() - return d.sendSignal(syscall.SIGTERM) + err := d.sendSignal(syscall.SIGTERM) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("hyperkit sigterm failed")) + } + + // wait 5s for graceful shutdown + for i := 0; i < 5; i++ { + log.Debug("waiting for graceful shutdown") + time.Sleep(time.Second * 1) + s, err := d.GetState() + if err != nil { + return errors.Wrap(err, fmt.Sprintf("hyperkit waiting graceful shutdown failed")) + } + if s == state.Stopped { + return nil + } + } + + log.Debug("sending sigkill") + return d.Kill() } func (d *Driver) extractKernel(isoPath string) error { diff --git a/pkg/drivers/kvm/domain.go b/pkg/drivers/kvm/domain.go index 50b354a29..88890fa7b 100644 --- a/pkg/drivers/kvm/domain.go +++ b/pkg/drivers/kvm/domain.go @@ -128,7 +128,7 @@ func randomMAC() (net.HardwareAddr, error) { } func (d *Driver) getDomain() (*libvirt.Domain, *libvirt.Connect, error) { - conn, err := getConnection() + conn, err := getConnection(d.ConnectionURI) if err != nil { return nil, nil, errors.Wrap(err, "getting domain") } @@ -141,8 +141,8 @@ func (d *Driver) getDomain() (*libvirt.Domain, *libvirt.Connect, error) { return dom, conn, nil } -func getConnection() (*libvirt.Connect, error) { - conn, err := libvirt.NewConnect(qemusystem) +func getConnection(connectionURI string) (*libvirt.Connect, error) { + conn, err := libvirt.NewConnect(connectionURI) if err != nil { return nil, errors.Wrap(err, connectionErrorText) } @@ -186,7 +186,7 @@ func (d *Driver) createDomain() (*libvirt.Domain, error) { return nil, errors.Wrap(err, "executing domain xml") } - conn, err := getConnection() + conn, err := getConnection(d.ConnectionURI) if err != nil { return nil, errors.Wrap(err, "Error getting libvirt connection") } diff --git a/pkg/drivers/kvm/kvm.go b/pkg/drivers/kvm/kvm.go index 24cc914a3..87ce4a0be 100644 --- a/pkg/drivers/kvm/kvm.go +++ b/pkg/drivers/kvm/kvm.go @@ -82,6 +82,9 @@ type Driver struct { // XML that needs to be added to passthrough GPU devices. DevicesXML string + + // QEMU Connection URI + ConnectionURI string } const ( @@ -107,12 +110,13 @@ func NewDriver(hostName, storePath string) *Driver { Network: defaultNetworkName, DiskPath: filepath.Join(constants.GetMinipath(), "machines", config.GetMachineName(), fmt.Sprintf("%s.rawdisk", config.GetMachineName())), ISO: filepath.Join(constants.GetMinipath(), "machines", config.GetMachineName(), "boot2docker.iso"), + ConnectionURI: qemusystem, } } // PreCommandCheck checks the connection before issuing a command func (d *Driver) PreCommandCheck() error { - conn, err := getConnection() + conn, err := getConnection(d.ConnectionURI) if err != nil { return errors.Wrap(err, "Error connecting to libvirt socket. Have you added yourself to the libvirtd group?") } @@ -424,7 +428,7 @@ func (d *Driver) Stop() (err error) { // Remove a host func (d *Driver) Remove() error { log.Debug("Removing machine...") - conn, err := getConnection() + conn, err := getConnection(d.ConnectionURI) if err != nil { return errors.Wrap(err, "getting connection") } diff --git a/pkg/drivers/kvm/network.go b/pkg/drivers/kvm/network.go index 8a7e4d98d..7a46f0ae1 100644 --- a/pkg/drivers/kvm/network.go +++ b/pkg/drivers/kvm/network.go @@ -81,7 +81,7 @@ func setupNetwork(conn *libvirt.Connect, name string) error { // ensureNetwork is called on start of the VM func (d *Driver) ensureNetwork() error { - conn, err := getConnection() + conn, err := getConnection(d.ConnectionURI) if err != nil { return errors.Wrap(err, "getting libvirt connection") } @@ -108,12 +108,11 @@ func (d *Driver) ensureNetwork() error { // createNetwork is called during creation of the VM only (and not on start) func (d *Driver) createNetwork() error { - if d.Network == defaultPrivateNetworkName { return fmt.Errorf("KVM network can't be named %s. This is the name of the private network created by minikube", defaultPrivateNetworkName) } - conn, err := getConnection() + conn, err := getConnection(d.ConnectionURI) if err != nil { return errors.Wrap(err, "getting libvirt connection") } @@ -151,7 +150,7 @@ func (d *Driver) createNetwork() error { } func (d *Driver) deleteNetwork() error { - conn, err := getConnection() + conn, err := getConnection(d.ConnectionURI) if err != nil { return errors.Wrap(err, "getting libvirt connection") } @@ -269,7 +268,7 @@ func (d *Driver) checkDomains(conn *libvirt.Connect) error { } func (d *Driver) lookupIP() (string, error) { - conn, err := getConnection() + conn, err := getConnection(d.ConnectionURI) if err != nil { return "", errors.Wrap(err, "getting connection and domain") } diff --git a/pkg/drivers/kvm/version.go b/pkg/drivers/kvm/version.go index 7b0a3a85c..edd87dde9 100644 --- a/pkg/drivers/kvm/version.go +++ b/pkg/drivers/kvm/version.go @@ -21,7 +21,15 @@ package kvm // version is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/drivers/kvm.version=vX.Y.Z" var version = "v0.0.0-unset" +// gitCommitID is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/drivers/kvm.gitCommitID=" +var gitCommitID = "" + // GetVersion returns the current docker-machine-driver-kvm2 version func GetVersion() string { return version } + +// GetGitCommitID returns the git commit id from which it is being built +func GetGitCommitID() string { + return gitCommitID +} diff --git a/pkg/minikube/config/types.go b/pkg/minikube/config/types.go index 0aa61d402..c7c0cb005 100644 --- a/pkg/minikube/config/types.go +++ b/pkg/minikube/config/types.go @@ -45,7 +45,8 @@ type MachineConfig struct { RegistryMirror []string HostOnlyCIDR string // Only used by the virtualbox driver HypervVirtualSwitch string - KvmNetwork string // Only used by the KVM driver + KVMNetwork string // Only used by the KVM driver + KVMQemuURI string // Only used by kvm2 Downloader util.ISODownloader `json:"-"` DockerOpt []string // Each entry is formatted as KEY=VALUE. DisableDriverMounts bool // Only used by virtualbox and xhyve @@ -55,6 +56,8 @@ type MachineConfig struct { GPU bool // Only used by kvm2 Hidden bool // Only used by kvm2 NoVTXCheck bool // Only used by virtualbox + DNSProxy bool // Only used by virtualbox + HostDNSResolver bool // Only used by virtualbox } // KubernetesConfig contains the parameters used to configure the VM Kubernetes. diff --git a/pkg/minikube/drivers/kvm/driver.go b/pkg/minikube/drivers/kvm/driver.go index 989e74276..451712b4c 100644 --- a/pkg/minikube/drivers/kvm/driver.go +++ b/pkg/minikube/drivers/kvm/driver.go @@ -64,7 +64,7 @@ func createKVMHost(config cfg.MachineConfig) interface{} { }, Memory: config.Memory, CPU: config.CPUs, - Network: config.KvmNetwork, + Network: config.KVMNetwork, PrivateNetwork: "docker-machines", Boot2DockerURL: config.Downloader.GetISOFileURI(config.MinikubeISO), DiskSize: config.DiskSize, diff --git a/pkg/minikube/drivers/kvm2/driver.go b/pkg/minikube/drivers/kvm2/driver.go index 93395611a..511f277a8 100644 --- a/pkg/minikube/drivers/kvm2/driver.go +++ b/pkg/minikube/drivers/kvm2/driver.go @@ -53,6 +53,7 @@ type kvmDriver struct { DiskPath string GPU bool Hidden bool + ConnectionURI string } func createKVM2Host(config cfg.MachineConfig) interface{} { @@ -64,7 +65,7 @@ func createKVM2Host(config cfg.MachineConfig) interface{} { }, Memory: config.Memory, CPU: config.CPUs, - Network: config.KvmNetwork, + Network: config.KVMNetwork, PrivateNetwork: "minikube-net", Boot2DockerURL: config.Downloader.GetISOFileURI(config.MinikubeISO), DiskSize: config.DiskSize, @@ -72,5 +73,6 @@ func createKVM2Host(config cfg.MachineConfig) interface{} { ISO: filepath.Join(constants.GetMinipath(), "machines", cfg.GetMachineName(), "boot2docker.iso"), GPU: config.GPU, Hidden: config.Hidden, + ConnectionURI: config.KVMQemuURI, } } diff --git a/pkg/minikube/drivers/virtualbox/driver.go b/pkg/minikube/drivers/virtualbox/driver.go index 1a1c50d6c..78441699d 100644 --- a/pkg/minikube/drivers/virtualbox/driver.go +++ b/pkg/minikube/drivers/virtualbox/driver.go @@ -54,8 +54,8 @@ func createVirtualboxHost(config cfg.MachineConfig) interface{} { d.NoVTXCheck = config.NoVTXCheck d.NatNicType = defaultVirtualboxNicType d.HostOnlyNicType = defaultVirtualboxNicType - d.DNSProxy = false - d.HostDNSResolver = true + d.DNSProxy = config.DNSProxy + d.HostDNSResolver = config.HostDNSResolver return d } diff --git a/pkg/version/version.go b/pkg/version/version.go index 62727d923..727035115 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -30,7 +30,7 @@ const VersionPrefix = "v" // version is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.version=vX.Y.Z" var version = "v0.0.0-unset" -// version is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.gitCommitID=" +// gitCommitID is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.gitCommitID=" var gitCommitID = "" // isoVersion is a private field and should be set when compiling with --ldflags="-X k8s.io/minikube/pkg/version.isoVersion=vX.Y.Z" diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go index 85b56a1c7..629401cf8 100644 --- a/test/integration/addons_test.go +++ b/test/integration/addons_test.go @@ -190,7 +190,57 @@ func testServicesList(t *testing.T) { t.Fatalf(err.Error()) } } +func testRegistry(t *testing.T) { + t.Parallel() + minikubeRunner := NewMinikubeRunner(t) + kubectlRunner := util.NewKubectlRunner(t) + minikubeRunner.RunCommand("addons enable registry", true) + t.Log("wait for registry to come up") + + if err := util.WaitForDockerRegistryRunning(t); err != nil { + t.Fatalf("waiting for registry to be up: %v", err) + } + + // Check access from outside the cluster on port 5000, validing connectivity via registry-proxy + checkExternalAccess := func() error { + t.Log("checking registry access from outside cluster") + expectedStr := "200" + runCmd := fmt.Sprintf("curl -sS -o /dev/null -w '%%{http_code}' http://127.0.0.1:5000") + externalCheckOutput, _ := minikubeRunner.SSH(runCmd) + if !strings.Contains(externalCheckOutput, expectedStr) { + return fmt.Errorf("ExpectedStr externalCheckOutput to be: %s. Output was: %s", expectedStr, externalCheckOutput) + } + return nil + } + if err := util.Retry(t, checkExternalAccess, 2*time.Second, 5); err != nil { + t.Fatalf(err.Error()) + } + // check access from inside the cluster via a busybox container running inside cluster + t.Log("checking registry access from inside cluster") + expectedStr := "200" + out, _ := kubectlRunner.RunCommand([]string{ + "run", + "registry-test", + "--restart=Never", + "--image=busybox", + "-it", + "--", + "sh", + "-c", + "wget --spider -S 'http://registry.kube-system.svc.cluster.local' 2>&1 | grep 'HTTP/' | awk '{print $2}'"}) + internalCheckOutput := string(out) + if !strings.Contains(internalCheckOutput, expectedStr) { + t.Fatalf("ExpectedStr internalCheckOutput to be: %s. Output was: %s", expectedStr, internalCheckOutput) + } + + defer func() { + if _, err := kubectlRunner.RunCommand([]string{"delete", "pod", "registry-test"}); err != nil { + t.Fatalf("failed to delete pod registry-test") + } + }() + minikubeRunner.RunCommand("addons disable registry", true) +} func testGvisor(t *testing.T) { minikubeRunner := NewMinikubeRunner(t) minikubeRunner.RunCommand("addons enable gvisor", true) diff --git a/test/integration/functional_test.go b/test/integration/functional_test.go index a8ced7a51..5186eae17 100644 --- a/test/integration/functional_test.go +++ b/test/integration/functional_test.go @@ -36,6 +36,7 @@ func TestFunctional(t *testing.T) { t.Run("DNS", testClusterDNS) t.Run("Logs", testClusterLogs) t.Run("Addons", testAddons) + t.Run("Registry", testRegistry) t.Run("Dashboard", testDashboard) t.Run("ServicesList", testServicesList) t.Run("Provisioning", testProvisioning) diff --git a/test/integration/start_stop_delete_test.go b/test/integration/start_stop_delete_test.go index 7a5b9392e..581091935 100644 --- a/test/integration/start_stop_delete_test.go +++ b/test/integration/start_stop_delete_test.go @@ -41,6 +41,7 @@ func TestStartStop(t *testing.T) { // default is the network created by libvirt, if we change the name minikube won't boot // because the given network doesn't exist "--kvm-network=default", + "--kvm-qemu-uri=qemu:///system", }}, {"feature_gates_newest_cni", []string{ "--feature-gates", diff --git a/test/integration/util/util.go b/test/integration/util/util.go index 08e1cebb2..056928b6c 100644 --- a/test/integration/util/util.go +++ b/test/integration/util/util.go @@ -359,6 +359,28 @@ func WaitForIngressControllerRunning(t *testing.T) error { return nil } +// WaitForDockerRegistryRunning waits until docker registry pod to be running +func WaitForDockerRegistryRunning(t *testing.T) error { + client, err := commonutil.GetClient() + if err != nil { + return errors.Wrap(err, "getting kubernetes client") + } + + if err := commonutil.WaitForRCToStabilize(client, "kube-system", "registry", time.Minute*10); err != nil { + return errors.Wrap(err, "waiting for registry replicacontroller to stabilize") + } + + registrySelector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "registry"})) + if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", registrySelector); err != nil { + return errors.Wrap(err, "waiting for registry pods") + } + proxySelector := labels.SelectorFromSet(labels.Set(map[string]string{"kubernetes.io/minikube-addons": "registry-proxy"})) + if err := commonutil.WaitForPodsWithLabelRunning(client, "kube-system", proxySelector); err != nil { + return errors.Wrap(err, "waiting for registry-proxy pods") + } + return nil +} + // WaitForIngressDefaultBackendRunning waits until ingress default backend pod to be running func WaitForIngressDefaultBackendRunning(t *testing.T) error { client, err := commonutil.GetClient() -- GitLab