Add renamed and refactored integration tests

test/integration/README.md

+# Integration tests
+
+## The basics
+
+To run all tests from the minikube root directory:
+
+`make integration`
+
+## Quickly iterating on a single test
+
+Run a single test on an active cluster:
+
+`make integration -e TEST_ARGS="-test.v -test.run TestFunctional/parallel/MountCmd --profile=minikube --cleanup=false"`
+
+WARNING: For this to work repeatedly, the test must be written so that it cleans up after itself.
+
+See `main.go` for details.
+
+## Disabling parallelism
+
+`make integration -e TEST_ARGS="-test.parallel=1"`
+
+## Testing philosophy
+

test/integration/addons_test.go

+// +build integration
+
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/go-retryablehttp"
+	"k8s.io/minikube/pkg/kapi"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+// TestAddons tests addons that require no special environment -- in parallel
+func TestAddons(t *testing.T) {
+	MaybeParallel(t)
+
+	profile := UniqueProfileName("addons")
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	defer CleanupWithLogs(t, profile, cancel)
+
+	args := append([]string{"start", "-p", profile, "--wait=false", "--memory=2600"}, StartArgs()...)
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	// Parallelized tests
+	t.Run("parallel", func(t *testing.T) {
+		tests := []struct {
+			name      string
+			validator validateFunc
+		}{
+			{"Registry", validateRegistryAddon},
+			{"Ingress", validateIngressAddon},
+		}
+		for _, tc := range tests {
+			tc := tc
+			t.Run(tc.name, func(t *testing.T) {
+				MaybeParallel(t)
+				tc.validator(ctx, t, profile)
+			})
+		}
+	})
+}
+
+func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
+	if NoneDriver() {
+		t.Skipf("skipping: ssh unsupported by none")
+	}
+
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "ingress"))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	client, err := kapi.Client(profile)
+	if err != nil {
+		t.Fatalf("kubernetes client: %v", client)
+	}
+
+	if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*5); err != nil {
+		t.Errorf("waiting for ingress-controller deployment to stabilize: %v", err)
+	}
+	if _, err := PodWait(ctx, t, profile, "kube-system", "app.kubernetes.io/name=nginx-ingress-controller", 3*time.Minute); err != nil {
+		t.Fatalf("wait: %v", err)
+	}
+
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-ing.yaml")))
+	if err != nil {
+		t.Errorf("%s failed: %v", rr.Args, err)
+	}
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-pod-svc.yaml")))
+	if err != nil {
+		t.Errorf("%s failed: %v", rr.Args, err)
+	}
+
+	if _, err := PodWait(ctx, t, profile, "default", "run=nginx", 2*time.Minute); err != nil {
+		t.Fatalf("wait: %v", err)
+	}
+	if err := kapi.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil {
+		t.Errorf("Error waiting for nginx service to be up")
+	}
+
+	want := "Welcome to nginx!"
+	checkIngress := func() error {
+		rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("curl http://127.0.0.1:80 -H 'Host: nginx.example.com'")))
+		if err != nil {
+			return err
+		}
+		if rr.Stderr.String() != "" {
+			t.Logf("%v: unexpected stderr: %s", rr.Args, rr.Stderr)
+		}
+		if !strings.Contains(rr.Stdout.String(), want) {
+			return fmt.Errorf("%v stdout = %q, want %q", rr.Args, rr.Stdout, want)
+		}
+		return nil
+	}
+
+	if err := retry.Expo(checkIngress, 500*time.Millisecond, time.Minute); err != nil {
+		t.Errorf("ingress never responded as expected on 127.0.0.1:80: %v", err)
+	}
+
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "ingress", "--alsologtostderr", "-v=1"))
+	if err != nil {
+		t.Errorf("%s failed: %v", rr.Args, err)
+	}
+}
+
+func validateRegistryAddon(ctx context.Context, t *testing.T, profile string) {
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "registry"))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	client, err := kapi.Client(profile)
+	if err != nil {
+		t.Fatalf("kubernetes client: %v", client)
+	}
+
+	start := time.Now()
+	if err := kapi.WaitForRCToStabilize(client, "kube-system", "registry", 3*time.Minute); err != nil {
+		t.Errorf("waiting for registry replicacontroller to stabilize: %v", err)
+	}
+	t.Logf("registry stabilized in %s", time.Since(start))
+
+	if _, err := PodWait(ctx, t, profile, "kube-system", "actual-registry=true", 3*time.Minute); err != nil {
+		t.Fatalf("wait: %v", err)
+	}
+	if _, err := PodWait(ctx, t, profile, "kube-system", "registry-proxy=true", 3*time.Minute); err != nil {
+		t.Fatalf("wait: %v", err)
+	}
+
+	// Test from inside the cluster (no curl available on busybox)
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "po", "-l", "run=registry-test", "--now"))
+	if err != nil {
+		t.Logf("pre-cleanup %s failed: %v (not a problem)", rr.Args, err)
+	}
+
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "run", "--rm", "registry-test", "--restart=Never", "--image=busybox", "-it", "--", "sh", "-c", "wget --spider -S http://registry.kube-system.svc.cluster.local"))
+	if err != nil {
+		t.Errorf("%s failed: %v", rr.Args, err)
+	}
+	want := "HTTP/1.1 200"
+	if !strings.Contains(rr.Stdout.String(), want) {
+		t.Errorf("curl = %q, want *%s*", rr.Stdout.String(), want)
+	}
+
+	// Test from outside the cluster
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ip"))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+	if rr.Stderr.String() != "" {
+		t.Errorf("%s: unexpected stderr: %s", rr.Args, rr.Stderr)
+	}
+
+	endpoint := fmt.Sprintf("http://%s:%d", strings.TrimSpace(rr.Stdout.String()), 5000)
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		t.Fatalf("failed to parse %q: %v", endpoint, err)
+	}
+
+	checkExternalAccess := func() error {
+		resp, err := retryablehttp.Get(u.String())
+		if err != nil {
+			return err
+		}
+		if resp.StatusCode != http.StatusOK {
+			return fmt.Errorf("%s = status code %d, want %d", u, resp.StatusCode, http.StatusOK)
+		}
+		return nil
+	}
+
+	if err := retry.Expo(checkExternalAccess, 500*time.Millisecond, 2*time.Minute); err != nil {
+		t.Errorf(err.Error())
+	}
+
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "registry", "--alsologtostderr", "-v=1"))
+	if err != nil {
+		t.Errorf("%s failed: %v", rr.Args, err)
+	}
+}

test/integration/fn_mount_cmd.go

+// +build integration
+
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+const (
+	guestMount                = "/mount-9p"
+	createdByPod              = "created-by-pod"
+	createdByTest             = "created-by-test"
+	createdByTestRemovedByPod = "created-by-test-removed-by-pod"
+	createdByPodRemovedByTest = "created-by-pod-removed-by-test"
+)
+
+func validateMountCmd(ctx context.Context, t *testing.T, profile string) {
+	if NoneDriver() {
+		t.Skip("skipping: none driver does not support mount")
+	}
+
+	tempDir, err := ioutil.TempDir("", "mounttest")
+	if err != nil {
+		t.Fatalf("Unexpected error while creating tempDir: %v", err)
+	}
+
+	ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
+
+	args := []string{"mount", "-p", profile, fmt.Sprintf("%s:%s", tempDir, guestMount), "--alsologtostderr", "-v=1"}
+	ss, err := Start(t, exec.CommandContext(ctx, Target(), args...))
+	if err != nil {
+		t.Fatalf("%v failed: %v", args, err)
+	}
+
+	defer func() {
+		if t.Failed() {
+			t.Logf("%s failed, getting debug info...", t.Name())
+			rr, err := Run(t, exec.Command(Target(), "-p", profile, "ssh", "mount | grep 9p; ls -la /mount-9p; cat /mount-9p/pod-dates"))
+			if err != nil {
+				t.Logf("%s: %v", rr.Command(), err)
+			} else {
+				t.Logf("(debug) %s:\n%s", rr.Command(), rr.Stdout)
+			}
+		}
+
+		// Cleanup in advance of future tests
+		rr, err := Run(t, exec.Command(Target(), "-p", profile, "ssh", "sudo umount -f /mount-9p"))
+		if err != nil {
+			t.Logf("%s: %v", rr.Command(), err)
+		}
+		ss.Stop(t)
+		cancel()
+		if *cleanup {
+			os.RemoveAll(tempDir)
+		}
+	}()
+
+	// Write local files
+	testMarker := fmt.Sprintf("test-%d", time.Now().UnixNano())
+	wantFromTest := []byte(testMarker)
+	for _, name := range []string{createdByTest, createdByTestRemovedByPod, testMarker} {
+		p := filepath.Join(tempDir, name)
+		err := ioutil.WriteFile(p, wantFromTest, 0644)
+		t.Logf("wrote %q to %s", wantFromTest, p)
+		if err != nil {
+			t.Errorf("WriteFile %s: %v", p, err)
+		}
+	}
+
+	// Block until the mount succeeds to avoid file race
+	checkMount := func() error {
+		_, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "findmnt -T /mount-9p | grep 9p"))
+		return err
+	}
+
+	start := time.Now()
+	if err := retry.Expo(checkMount, time.Second, 15*time.Second); err != nil {
+		// For local testing, allow macOS users to click prompt. If they don't, skip the test.
+		if runtime.GOOS == "darwin" {
+			t.Skip("skipping: mount did not appear, likely because macOS requires prompt to allow non-codesigned binaries to listen on non-localhost port")
+		}
+		t.Fatalf("/mount-9p did not appear within %s: %v", time.Since(start), err)
+	}
+
+	// Assert that we can access the mount without an error. Display for debugging.
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "--", "ls", "-la", guestMount))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+	t.Logf("guest mount directory contents\n%s", rr.Stdout)
+
+	// Assert that the mount contains our unique test marker, as opposed to a stale mount
+	tp := filepath.Join("/mount-9p", testMarker)
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "cat", tp))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	if !bytes.Equal(rr.Stdout.Bytes(), wantFromTest) {
+		// The mount is hosed, exit fast before wasting time launching pods.
+		t.Fatalf("%s = %q, want %q", tp, rr.Stdout.Bytes(), wantFromTest)
+	}
+
+	// Start the "busybox-mount" pod.
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "busybox-mount-test.yaml")))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox-mount", 2*time.Minute); err != nil {
+		t.Fatalf("wait: %v", err)
+	}
+
+	// Read the file written by pod startup
+	p := filepath.Join(tempDir, createdByPod)
+	got, err := ioutil.ReadFile(p)
+	if err != nil {
+		t.Errorf("readfile %s: %v", p, err)
+	}
+	wantFromPod := []byte("test\n")
+	if !bytes.Equal(got, wantFromPod) {
+		t.Errorf("%s = %q, want %q", p, got, wantFromPod)
+	}
+
+	// test that file written from host was read in by the pod via cat /mount-9p/written-by-host;
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", "busybox-mount"))
+	if err != nil {
+		t.Errorf("%s failed: %v", rr.Args, err)
+	}
+	if !bytes.Equal(rr.Stdout.Bytes(), wantFromTest) {
+		t.Errorf("busybox-mount logs = %q, want %q", rr.Stdout.Bytes(), wantFromTest)
+	}
+
+	// test file timestamps are correct
+	for _, name := range []string{createdByTest, createdByPod} {
+		gp := path.Join(guestMount, name)
+		// test that file written from host was read in by the pod via cat /mount-9p/fromhost;
+		rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "stat", gp))
+		if err != nil {
+			t.Errorf("%s failed: %v", rr.Args, err)
+		}
+
+		if runtime.GOOS == "windows" {
+			if strings.Contains(rr.Stdout.String(), "Access: 1970-01-01") {
+				t.Errorf("invalid access time: %v", rr.Stdout)
+			}
+		}
+
+		if strings.Contains(rr.Stdout.String(), "Modify: 1970-01-01") {
+			t.Errorf("invalid modify time: %v", rr.Stdout)
+		}
+	}
+
+	p = filepath.Join(tempDir, createdByTestRemovedByPod)
+	if _, err := os.Stat(p); err == nil {
+		t.Errorf("expected file %s to be removed", p)
+	}
+
+	p = filepath.Join(tempDir, createdByPodRemovedByTest)
+	if err := os.Remove(p); err != nil {
+		t.Errorf("unexpected error removing file %s: %v", p, err)
+	}
+}

test/integration/fn_pvc.go

+// +build integration
+
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+
+	core "k8s.io/api/core/v1"
+	storage "k8s.io/api/storage/v1"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+func validatePersistentVolumeClaim(ctx context.Context, t *testing.T, profile string) {
+	ctx, cancel := context.WithTimeout(ctx, 5*time.Minute)
+	defer cancel()
+
+	if _, err := PodWait(ctx, t, profile, "kube-system", "integration-test=storage-provisioner", 2*time.Minute); err != nil {
+		t.Fatalf("wait: %v", err)
+	}
+
+	checkStorageClass := func() error {
+		rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "storageclass", "-o=json"))
+		if err != nil {
+			return err
+		}
+		scl := storage.StorageClassList{}
+		if err := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes())).Decode(&scl); err != nil {
+			return err
+		}
+		if len(scl.Items) == 0 {
+			return fmt.Errorf("no storageclass yet")
+		}
+		return nil
+	}
+
+	// Ensure the addon-manager has created the StorageClass before creating a claim, otherwise it won't be bound
+	if err := retry.Expo(checkStorageClass, time.Second, 90*time.Second); err != nil {
+		t.Errorf("no default storage class after retry: %v", err)
+	}
+
+	// Now create a testpvc
+	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", filepath.Join(*testdataDir, "pvc.yaml")))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	checkStoragePhase := func() error {
+		rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "pvc", "testpvc", "-o=json"))
+		if err != nil {
+			return err
+		}
+		pvc := core.PersistentVolumeClaim{}
+		if err := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes())).Decode(&pvc); err != nil {
+			return err
+		}
+		// The test passes if the volume claim gets bound.
+		if pvc.Status.Phase == "Bound" {
+			return nil
+		}
+		return fmt.Errorf("testpvc phase = %q, want %q (msg=%+v)", pvc.Status.Phase, "Bound", pvc)
+	}
+
+	if err := retry.Expo(checkStoragePhase, 2*time.Second, 2*time.Minute); err != nil {
+		t.Fatalf("PV Creation failed with error: %v", err)
+	}
+}

test/integration/fn_tunnel_cmd.go

+/*
+Copyright 2018 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/wait"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/minikube/pkg/kapi"
+	"k8s.io/minikube/pkg/minikube/tunnel"
+	"k8s.io/minikube/pkg/util/retry"
+)
+
+func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) {
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Minute)
+	defer cancel()
+
+	if runtime.GOOS != "windows" {
+		// Otherwise minikube fails waiting for a password.
+		if err := exec.Command("sudo", "-n", "route").Run(); err != nil {
+			t.Skipf("password required to execute 'route', skipping testTunnel: %v", err)
+		}
+	}
+
+	client, err := kapi.Client(profile)
+	if err != nil {
+		t.Fatalf("client: %v", err)
+	}
+
+	// Pre-Cleanup
+	if err := tunnel.NewManager().CleanupNotRunningTunnels(); err != nil {
+		t.Errorf("CleanupNotRunningTunnels: %v", err)
+	}
+
+	// Start the tunnel
+	args := []string{"-p", profile, "tunnel", "--alsologtostderr", "-v=1"}
+	ss, err := Start(t, exec.CommandContext(ctx, Target(), args...))
+	defer ss.Stop(t)
+
+	// Start the "nginx" pod.
+	rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", filepath.Join(*testdataDir, "testsvc.yaml")))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+	if _, err := PodWait(ctx, t, profile, "default", "run=nginx-svc", 2*time.Minute); err != nil {
+		t.Fatalf("wait: %v", err)
+	}
+
+	if err := kapi.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil {
+		t.Fatal(errors.Wrap(err, "Error waiting for nginx service to be up"))
+	}
+
+	// Wait until the nginx-svc has a loadbalancer ingress IP
+	nginxIP := ""
+	err = wait.PollImmediate(1*time.Second, 3*time.Minute, func() (bool, error) {
+		rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "svc", "nginx-svc", "-o", "-f", "jsonpath={.status.loadBalancer.ingress[0].ip}"))
+		if err != nil {
+			return false, err
+		}
+		if len(rr.Stdout.String()) > 0 {
+			nginxIP = rr.Stdout.String()
+			return true, nil
+		}
+		return false, nil
+	})
+	if err != nil {
+		t.Errorf("nginx-svc svc.status.loadBalancer.ingress never got an IP")
+
+		rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "svc", "nginx-svc"))
+		if err != nil {
+			t.Errorf("%s failed: %v", rr.Args, err)
+		}
+		t.Logf("kubectl get svc nginx-svc:\n%s", rr.Stdout)
+	}
+
+	// Try fetching against the IP
+	var resp *http.Response
+
+	req := func() error {
+		h := &http.Client{Timeout: time.Second * 10}
+		resp, err = h.Get(fmt.Sprintf("http://%s", nginxIP))
+		if err != nil {
+			retriable := &retry.RetriableError{Err: err}
+			return retriable
+		}
+		defer resp.Body.Close()
+		return nil
+	}
+	if err = retry.Expo(req, time.Millisecond*500, 2*time.Minute, 6); err != nil {
+		t.Errorf("failed to contact nginx at %s: %v", nginxIP, err)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Errorf("ReadAll: %v", err)
+	}
+	want := "Welcome to nginx!"
+	if !strings.Contains(string(body), want) {
+		t.Errorf("body = %q, want *%s*", body, want)
+	}
+}

test/integration/guest_env_test.go

+// +build iso
+
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"fmt"
+	"os/exec"
+	"testing"
+	"time"
+)
+
+func TestGuestEnvironment(t *testing.T) {
+	MaybeParallel(t)
+	profile := UniqueProfileName("guest")
+	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
+	defer CleanupWithLogs(t, profile, cancel)
+
+	args := append([]string{"start", "-p", profile, "--wait=false"}, StartArgs()...)
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
+	if err != nil {
+		t.Errorf("%s failed: %v", rr.Args, err)
+	}
+
+	// Run as a group so that our defer doesn't happen as tests are runnings
+	t.Run("Binaries", func(t *testing.T) {
+		for _, pkg := range []string{"git", "rsync", "curl", "wget", "socat", "iptables", "VBoxControl", "VBoxService"} {
+			pkg := pkg
+			t.Run(pkg, func(t *testing.T) {
+				t.Parallel()
+				rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("which %s", pkg)))
+				if err != nil {
+					t.Errorf("%s failed: %v", rr.Args, err)
+				}
+			})
+		}
+	})
+
+	t.Run("PersistentMounts", func(t *testing.T) {
+		for _, mount := range []string{
+			"/data",
+			"/var/lib/docker",
+			"/var/lib/cni",
+			"/var/lib/kubelet",
+			"/var/lib/minikube",
+			"/var/lib/toolbox",
+			"/var/lib/boot2docker",
+		} {
+			mount := mount
+			t.Run(mount, func(t *testing.T) {
+				t.Parallel()
+				rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("df -t ext4 %s | grep %s", mount, mount)))
+				if err != nil {
+					t.Errorf("%s failed: %v", rr.Args, err)
+				}
+			})
+		}
+	})
+}

test/integration/gvisor_addon_test.go

+// +build integration
+
+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"context"
+	"os/exec"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestGvisorAddon(t *testing.T) {
+	// TODO(tstromberg): Fix or remove addon.
+	t.Skip("SKIPPING: Currently broken (gvisor-containerd-shim.toml CrashLoopBackoff): https://github.com/kubernetes/minikube/issues/5305")
+
+	if NoneDriver() {
+		t.Skip("Can't run containerd backend with none driver")
+	}
+	MaybeParallel(t)
+
+	profile := UniqueProfileName("gvisor")
+	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	defer func() {
+		CleanupWithLogs(t, profile, cancel)
+	}()
+
+	startArgs := append([]string{"start", "-p", profile, "--container-runtime=containerd", "--docker-opt", "containerd=/var/run/containerd/containerd.sock", "--wait=false"}, StartArgs()...)
+	rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	// TODO: Re-examine if we should be pulling in an image which users don't normally invoke
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "cache", "add", "gcr.io/k8s-minikube/gvisor-addon:latest"))
+	if err != nil {
+		t.Errorf("%s failed: %v", rr.Args, err)
+	}
+
+	// NOTE: addons are global, but the addon must assert that the runtime is containerd
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "gvisor"))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	// Because addons are persistent across profiles :(
+	defer func() {
+		rr, err := Run(t, exec.Command(Target(), "-p", profile, "addons", "disable", "gvisor"))
+		if err != nil {
+			t.Logf("%s failed: %v", rr.Args, err)
+		}
+	}()
+
+	if _, err := PodWait(ctx, t, profile, "kube-system", "kubernetes.io/minikube-addons=gvisor", 2*time.Minute); err != nil {
+		t.Fatalf("waiting for gvisor controller to be up: %v", err)
+	}
+
+	// Create an untrusted workload
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-untrusted.yaml")))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+	// Create gvisor workload
+	rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-gvisor.yaml")))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	if _, err := PodWait(ctx, t, profile, "default", "run=nginx,untrusted=true", 2*time.Minute); err != nil {
+		t.Errorf("nginx: %v", err)
+	}
+	if _, err := PodWait(ctx, t, profile, "default", "run=nginx,runtime=gvisor", 2*time.Minute); err != nil {
+		t.Errorf("nginx: %v", err)
+	}
+
+	// Ensure that workloads survive a restart
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), "stop", "-p", profile))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+
+	rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...))
+	if err != nil {
+		t.Fatalf("%s failed: %v", rr.Args, err)
+	}
+	if _, err := PodWait(ctx, t, profile, "kube-system", "kubernetes.io/minikube-addons=gvisor", 2*time.Minute); err != nil {
+		t.Errorf("waiting for gvisor controller to be up: %v", err)
+	}
+	if _, err := PodWait(ctx, t, profile, "default", "run=nginx,untrusted=true", 2*time.Minute); err != nil {
+		t.Errorf("nginx: %v", err)
+	}
+	if _, err := PodWait(ctx, t, profile, "default", "run=nginx,runtime=gvisor", 2*time.Minute); err != nil {
+		t.Errorf("nginx: %v", err)
+	}
+}

test/integration/helpers.go

+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+// These are test helpers that:
+//
+// - Accept *testing.T arguments (see helpers.go)
+// - Are used in multiple tests
+// - Must not compare test values
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/shirou/gopsutil/process"
+	core "k8s.io/api/core/v1"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/minikube/pkg/kapi"
+)
+
+// RunResult stores the result of an cmd.Run call
+type RunResult struct {
+	Stdout   *bytes.Buffer
+	Stderr   *bytes.Buffer
+	ExitCode int
+	Args     []string
+}
+
+// Command returns a human readable command string that does not induce eye fatigue
+func (rr RunResult) Command() string {
+	var sb strings.Builder
+	sb.WriteString(strings.TrimPrefix(rr.Args[0], "../../"))
+	for _, a := range rr.Args[1:] {
+		if strings.Contains(a, " ") {
+			sb.WriteString(fmt.Sprintf(` "%s"`, a))
+			continue
+		}
+		sb.WriteString(fmt.Sprintf(" %s", a))
+	}
+	return sb.String()
+}
+
+func (rr RunResult) String() string {
+	var sb strings.Builder
+	sb.WriteString(fmt.Sprintf("Command: %v\n", rr.Command()))
+	if rr.Stdout.Len() > 0 {
+		sb.WriteString(fmt.Sprintf("\n-- stdout -- \n%s\n", rr.Stdout.Bytes()))
+	}
+	if rr.Stderr.Len() > 0 {
+		sb.WriteString(fmt.Sprintf("\n** stderr ** \n%s\n", rr.Stderr.Bytes()))
+	}
+	return sb.String()
+}
+
+// Run is a test helper to log a command being executed \_(ツ)_/¯
+func Run(t *testing.T, cmd *exec.Cmd) (*RunResult, error) {
+	t.Helper()
+	rr := &RunResult{Args: cmd.Args}
+	t.Logf("(dbg) Run:  %v", rr.Command())
+
+	var outb, errb bytes.Buffer
+	cmd.Stdout, rr.Stdout = &outb, &outb
+	cmd.Stderr, rr.Stderr = &errb, &errb
+	start := time.Now()
+	err := cmd.Run()
+	elapsed := time.Since(start)
+	if err == nil {
+		// Reduce log spam
+		if elapsed > (1 * time.Second) {
+			t.Logf("(dbg) Done: %v: (%s)", rr.Command(), elapsed)
+		}
+	} else {
+		if exitError, ok := err.(*exec.ExitError); ok {
+			rr.ExitCode = exitError.ExitCode()
+		}
+		t.Logf("(dbg) Non-zero exit: %v: %v (%s)", rr.Command(), err, elapsed)
+		t.Logf("(dbg) %s", rr.String())
+	}
+	return rr, err
+}
+
+// StartSession stores the result of an cmd.Start call
+type StartSession struct {
+	Stdout *bufio.Reader
+	Stderr *bufio.Reader
+	cmd    *exec.Cmd
+}
+
+// Start starts a process in the background, streaming output
+func Start(t *testing.T, cmd *exec.Cmd) (*StartSession, error) {
+	t.Helper()
+	t.Logf("Daemon: %v", cmd.Args)
+
+	stdoutPipe, err := cmd.StdoutPipe()
+	if err != nil {
+		t.Fatalf("stdout pipe failed: %v %v", cmd.Args, err)
+	}
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		t.Fatalf("stderr pipe failed: %v %v", cmd.Args, err)
+	}
+
+	sr := &StartSession{Stdout: bufio.NewReader(stdoutPipe), Stderr: bufio.NewReader(stderrPipe), cmd: cmd}
+	return sr, cmd.Start()
+}
+
+// Stop stops the started process
+func (ss *StartSession) Stop(t *testing.T) {
+	t.Helper()
+	t.Logf("Stopping %s ...", ss.cmd.Args)
+	if ss.cmd.Process == nil {
+		t.Logf("%s has a nil Process. Maybe it's dead? How weird!", ss.cmd.Args)
+		return
+	}
+	killProcessFamily(t, ss.cmd.Process.Pid)
+	if t.Failed() {
+		if ss.Stdout.Size() > 0 {
+			stdout, err := ioutil.ReadAll(ss.Stdout)
+			if err != nil {
+				t.Logf("read stdout failed: %v", err)
+			}
+			t.Logf("(dbg) %s stdout:\n%s", ss.cmd.Args, stdout)
+		}
+		if ss.Stderr.Size() > 0 {
+			stderr, err := ioutil.ReadAll(ss.Stderr)
+			if err != nil {
+				t.Logf("read stderr failed: %v", err)
+			}
+			t.Logf("(dbg) %s stderr:\n%s", ss.cmd.Args, stderr)
+		}
+	}
+}
+
+// Cleanup cleans up after a test run
+func Cleanup(t *testing.T, profile string, cancel context.CancelFunc) {
+	// No helper because it makes the call log confusing.
+	if *cleanup {
+		_, err := Run(t, exec.Command(Target(), "delete", "-p", profile))
+		if err != nil {
+			t.Logf("failed cleanup: %v", err)
+		}
+	} else {
+		t.Logf("Skipping cleanup of %s (--cleanup=false)", profile)
+	}
+	cancel()
+}
+
+// CleanupWithLogs cleans up after a test run, fetching logs and deleting the profile
+func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) {
+	t.Helper()
+	if t.Failed() && *postMortemLogs {
+		t.Logf("%s failed, collecting logs ...", t.Name())
+		rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "10"))
+		if err != nil {
+			t.Logf("failed logs error: %v", err)
+		}
+		t.Logf("%s logs: %s\n", t.Name(), rr)
+		t.Logf("Sorry that %s failed :(", t.Name())
+	}
+	Cleanup(t, profile, cancel)
+}
+
+// podStatusMsg returns a human-readable pod status, for generating debug status
+func podStatusMsg(pod core.Pod) string {
+	var sb strings.Builder
+	sb.WriteString(fmt.Sprintf("%q [%s] %s", pod.ObjectMeta.GetName(), pod.ObjectMeta.GetUID(), pod.Status.Phase))
+	for i, c := range pod.Status.Conditions {
+		if c.Reason != "" {
+			if i == 0 {
+				sb.WriteString(": ")
+			} else {
+				sb.WriteString(" / ")
+			}
+			sb.WriteString(fmt.Sprintf("%s:%s", c.Type, c.Reason))
+		}
+		if c.Message != "" {
+			sb.WriteString(fmt.Sprintf(" (%s)", c.Message))
+		}
+	}
+	return sb.String()
+}
+
+// PodWait waits for pods to achieve a running state.
+func PodWait(ctx context.Context, t *testing.T, profile string, ns string, selector string, timeout time.Duration) ([]string, error) {
+	t.Helper()
+	client, err := kapi.Client(profile)
+	if err != nil {
+		return nil, err
+	}
+
+	// For example: kubernetes.io/minikube-addons=gvisor
+	listOpts := meta.ListOptions{LabelSelector: selector}
+	minUptime := 5 * time.Second
+	podStart := time.Time{}
+	foundNames := map[string]bool{}
+	lastMsg := ""
+
+	start := time.Now()
+	t.Logf("Waiting for pods with labels %q in namespace %q ...", selector, ns)
+	f := func() (bool, error) {
+		pods, err := client.CoreV1().Pods(ns).List(listOpts)
+		if err != nil {
+			t.Logf("Pod(%s).List(%v) returned error: %v", ns, selector, err)
+			// Don't bother to retry: something is very wrong.
+			return true, err
+		}
+		if len(pods.Items) == 0 {
+			podStart = time.Time{}
+			return false, nil
+		}
+
+		for _, pod := range pods.Items {
+			foundNames[pod.ObjectMeta.Name] = true
+			msg := podStatusMsg(pod)
+			// Prevent spamming logs with identical messages
+			if msg != lastMsg {
+				t.Log(msg)
+				lastMsg = msg
+			}
+			// Successful termination of a short-lived process, will not be restarted
+			if pod.Status.Phase == core.PodSucceeded {
+				return true, nil
+			}
+			// Long-running process state
+			if pod.Status.Phase != core.PodRunning {
+				if !podStart.IsZero() {
+					t.Logf("WARNING: %s was running %s ago - may be unstable", selector, time.Since(podStart))
+				}
+				podStart = time.Time{}
+				return false, nil
+			}
+
+			if podStart.IsZero() {
+				podStart = time.Now()
+			}
+
+			if time.Since(podStart) > minUptime {
+				return true, nil
+			}
+		}
+		return false, nil
+	}
+
+	err = wait.PollImmediate(500*time.Millisecond, timeout, f)
+	names := []string{}
+	for n := range foundNames {
+		names = append(names, n)
+	}
+
+	if err == nil {
+		t.Logf("pods %s up and healthy within %s", selector, time.Since(start))
+		return names, nil
+	}
+
+	t.Logf("pods %q: %v", selector, err)
+	showPodLogs(ctx, t, profile, ns, names)
+	return names, fmt.Errorf("%s: %v", fmt.Sprintf("%s within %s", selector, timeout), err)
+}
+
+// showPodLogs logs debug info for pods
+func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, names []string) {
+	rr, rerr := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "po", "-A", "--show-labels"))
+	if rerr != nil {
+		t.Logf("%s: %v", rr.Command(), rerr)
+		// return now, because kubectl is hosed
+		return
+	}
+	t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
+
+	for _, name := range names {
+		rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "describe", "po", name, "-n", ns))
+		if err != nil {
+			t.Logf("%s: %v", rr.Command(), err)
+		} else {
+			t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
+		}
+
+		rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", name, "-n", ns))
+		if err != nil {
+			t.Logf("%s: %v", rr.Command(), err)
+		} else {
+			t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout)
+		}
+	}
+}
+
+// Status returns the minikube cluster status as a string
+func Status(ctx context.Context, t *testing.T, path string, profile string) string {
+	t.Helper()
+	rr, err := Run(t, exec.CommandContext(ctx, path, "status", "--format={{.Host}}", "-p", profile))
+	if err != nil {
+		t.Logf("status error: %v (may be ok)", err)
+	}
+	return strings.TrimSpace(rr.Stdout.String())
+}
+
+// MaybeParallel sets that the test should run in parallel
+func MaybeParallel(t *testing.T) {
+	t.Helper()
+	// TODO: Allow paralellized tests on "none" that do not require independent clusters
+	if NoneDriver() {
+		return
+	}
+	t.Parallel()
+}
+
+// killProcessFamily kills a pid and all of its children
+func killProcessFamily(t *testing.T, pid int) {
+	parent, err := process.NewProcess(int32(pid))
+	if err != nil {
+		t.Logf("unable to find parent, assuming dead: %v", err)
+		return
+	}
+	procs := []*process.Process{}
+	children, err := parent.Children()
+	if err == nil {
+		procs = append(procs, children...)
+	}
+	procs = append(procs, parent)
+
+	for _, p := range procs {
+		if err := p.Terminate(); err != nil {
+			t.Logf("unable to terminate pid %d: %v", p.Pid, err)
+			continue
+		}
+		if err := p.Kill(); err != nil {
+			t.Logf("unable to kill pid %d: %v", p.Pid, err)
+			continue
+		}
+	}
+}

test/integration/main.go

+/*
+Copyright 2016 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"flag"
+	"os"
+	"strings"
+	"testing"
+)
+
+// General configuration: used to set the VM Driver
+var startArgs = flag.String("minikube-start-args", "", "Arguments to pass to minikube start")
+
+// Flags for faster local integration testing
+var forceProfile = flag.String("profile", "", "force tests to run against a particular profile")
+var cleanup = flag.Bool("cleanup", true, "cleanup failed test run")
+var postMortemLogs = flag.Bool("postmortem-logs", true, "show logs after a failed test run")
+
+// Paths to files - normally set for CI
+var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary")
+var testdataDir = flag.String("testdata-dir", "testdata", "the directory relative to test/integration where the testdata lives")
+
+// TestMain is the test main
+func TestMain(m *testing.M) {
+	flag.Parse()
+	os.Exit(m.Run())
+}
+
+// StartArgs returns the arguments normally used for starting minikube
+func StartArgs() []string {
+	return strings.Split(*startArgs, " ")
+}
+
+// Target returns where the minikube binary can be found
+func Target() string {
+	return *binaryPath
+}
+
+// NoneDriver returns whether or not this test is using the none driver
+func NoneDriver() bool {
+	return strings.Contains(*startArgs, "--vm-driver=none")
+}

test/integration/util.go

+/*
+Copyright 2019 The Kubernetes Authors All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package integration
+
+import (
+	"bufio"
+	"fmt"
+	"os"
+	"time"
+)
+
+// ReadLineWithTimeout reads a line of text from a buffer with a timeout
+func ReadLineWithTimeout(b *bufio.Reader, timeout time.Duration) (string, error) {
+	s := make(chan string)
+	e := make(chan error)
+	go func() {
+		read, err := b.ReadString('\n')
+		if err != nil {
+			e <- err
+		} else {
+			s <- read
+		}
+		close(s)
+		close(e)
+	}()
+
+	select {
+	case line := <-s:
+		return line, nil
+	case err := <-e:
+		return "", err
+	case <-time.After(timeout):
+		return "", fmt.Errorf("timeout after %s", timeout)
+	}
+}
+
+// UniqueProfileName returns a reasonably unique profile name
+func UniqueProfileName(prefix string) string {
+	if *forceProfile != "" {
+		return *forceProfile
+	}
+	if NoneDriver() {
+		return "minikube"
+	}
+	return fmt.Sprintf("%s-%s-%d", prefix, time.Now().Format("20060102T150405.999999999"), os.Getpid())
+}