From 4b484bdbff1f37e3a535529afeffc6df2b43f59e Mon Sep 17 00:00:00 2001 From: Thomas Stromberg Date: Wed, 11 Sep 2019 10:16:16 -0700 Subject: [PATCH] Add renamed and refactored integration tests --- test/integration/README.md | 24 ++ test/integration/addons_test.go | 206 +++++++++++++++ test/integration/fn_mount_cmd.go | 192 ++++++++++++++ test/integration/fn_pvc.go | 89 +++++++ test/integration/fn_tunnel_cmd.go | 127 +++++++++ test/integration/guest_env_test.go | 75 ++++++ test/integration/gvisor_addon_test.go | 111 ++++++++ test/integration/helpers.go | 353 ++++++++++++++++++++++++++ test/integration/main.go | 57 +++++ test/integration/util.go | 60 +++++ 10 files changed, 1294 insertions(+) create mode 100644 test/integration/README.md create mode 100644 test/integration/addons_test.go create mode 100644 test/integration/fn_mount_cmd.go create mode 100644 test/integration/fn_pvc.go create mode 100644 test/integration/fn_tunnel_cmd.go create mode 100644 test/integration/guest_env_test.go create mode 100644 test/integration/gvisor_addon_test.go create mode 100644 test/integration/helpers.go create mode 100644 test/integration/main.go create mode 100644 test/integration/util.go diff --git a/test/integration/README.md b/test/integration/README.md new file mode 100644 index 000000000..498883ece --- /dev/null +++ b/test/integration/README.md @@ -0,0 +1,24 @@ +# Integration tests + +## The basics + +To run all tests from the minikube root directory: + +`make integration` + +## Quickly iterating on a single test + +Run a single test on an active cluster: + +`make integration -e TEST_ARGS="-test.v -test.run TestFunctional/parallel/MountCmd --profile=minikube --cleanup=false"` + +WARNING: For this to work repeatedly, the test must be written so that it cleans up after itself. + +See `main.go` for details. + +## Disabling parallelism + +`make integration -e TEST_ARGS="-test.parallel=1"` + +## Testing philosophy + diff --git a/test/integration/addons_test.go b/test/integration/addons_test.go new file mode 100644 index 000000000..1ca6435c8 --- /dev/null +++ b/test/integration/addons_test.go @@ -0,0 +1,206 @@ +// +build integration + +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os/exec" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hashicorp/go-retryablehttp" + "k8s.io/minikube/pkg/kapi" + "k8s.io/minikube/pkg/util/retry" +) + +// TestAddons tests addons that require no special environment -- in parallel +func TestAddons(t *testing.T) { + MaybeParallel(t) + + profile := UniqueProfileName("addons") + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer CleanupWithLogs(t, profile, cancel) + + args := append([]string{"start", "-p", profile, "--wait=false", "--memory=2600"}, StartArgs()...) + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + // Parallelized tests + t.Run("parallel", func(t *testing.T) { + tests := []struct { + name string + validator validateFunc + }{ + {"Registry", validateRegistryAddon}, + {"Ingress", validateIngressAddon}, + } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + MaybeParallel(t) + tc.validator(ctx, t, profile) + }) + } + }) +} + +func validateIngressAddon(ctx context.Context, t *testing.T, profile string) { + if NoneDriver() { + t.Skipf("skipping: ssh unsupported by none") + } + + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "ingress")) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + client, err := kapi.Client(profile) + if err != nil { + t.Fatalf("kubernetes client: %v", client) + } + + if err := kapi.WaitForDeploymentToStabilize(client, "kube-system", "nginx-ingress-controller", time.Minute*5); err != nil { + t.Errorf("waiting for ingress-controller deployment to stabilize: %v", err) + } + if _, err := PodWait(ctx, t, profile, "kube-system", "app.kubernetes.io/name=nginx-ingress-controller", 3*time.Minute); err != nil { + t.Fatalf("wait: %v", err) + } + + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-ing.yaml"))) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-pod-svc.yaml"))) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + + if _, err := PodWait(ctx, t, profile, "default", "run=nginx", 2*time.Minute); err != nil { + t.Fatalf("wait: %v", err) + } + if err := kapi.WaitForService(client, "default", "nginx", true, time.Millisecond*500, time.Minute*10); err != nil { + t.Errorf("Error waiting for nginx service to be up") + } + + want := "Welcome to nginx!" + checkIngress := func() error { + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("curl http://127.0.0.1:80 -H 'Host: nginx.example.com'"))) + if err != nil { + return err + } + if rr.Stderr.String() != "" { + t.Logf("%v: unexpected stderr: %s", rr.Args, rr.Stderr) + } + if !strings.Contains(rr.Stdout.String(), want) { + return fmt.Errorf("%v stdout = %q, want %q", rr.Args, rr.Stdout, want) + } + return nil + } + + if err := retry.Expo(checkIngress, 500*time.Millisecond, time.Minute); err != nil { + t.Errorf("ingress never responded as expected on 127.0.0.1:80: %v", err) + } + + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "ingress", "--alsologtostderr", "-v=1")) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } +} + +func validateRegistryAddon(ctx context.Context, t *testing.T, profile string) { + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "registry")) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + client, err := kapi.Client(profile) + if err != nil { + t.Fatalf("kubernetes client: %v", client) + } + + start := time.Now() + if err := kapi.WaitForRCToStabilize(client, "kube-system", "registry", 3*time.Minute); err != nil { + t.Errorf("waiting for registry replicacontroller to stabilize: %v", err) + } + t.Logf("registry stabilized in %s", time.Since(start)) + + if _, err := PodWait(ctx, t, profile, "kube-system", "actual-registry=true", 3*time.Minute); err != nil { + t.Fatalf("wait: %v", err) + } + if _, err := PodWait(ctx, t, profile, "kube-system", "registry-proxy=true", 3*time.Minute); err != nil { + t.Fatalf("wait: %v", err) + } + + // Test from inside the cluster (no curl available on busybox) + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "delete", "po", "-l", "run=registry-test", "--now")) + if err != nil { + t.Logf("pre-cleanup %s failed: %v (not a problem)", rr.Args, err) + } + + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "run", "--rm", "registry-test", "--restart=Never", "--image=busybox", "-it", "--", "sh", "-c", "wget --spider -S http://registry.kube-system.svc.cluster.local")) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + want := "HTTP/1.1 200" + if !strings.Contains(rr.Stdout.String(), want) { + t.Errorf("curl = %q, want *%s*", rr.Stdout.String(), want) + } + + // Test from outside the cluster + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ip")) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + if rr.Stderr.String() != "" { + t.Errorf("%s: unexpected stderr: %s", rr.Args, rr.Stderr) + } + + endpoint := fmt.Sprintf("http://%s:%d", strings.TrimSpace(rr.Stdout.String()), 5000) + u, err := url.Parse(endpoint) + if err != nil { + t.Fatalf("failed to parse %q: %v", endpoint, err) + } + + checkExternalAccess := func() error { + resp, err := retryablehttp.Get(u.String()) + if err != nil { + return err + } + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("%s = status code %d, want %d", u, resp.StatusCode, http.StatusOK) + } + return nil + } + + if err := retry.Expo(checkExternalAccess, 500*time.Millisecond, 2*time.Minute); err != nil { + t.Errorf(err.Error()) + } + + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "disable", "registry", "--alsologtostderr", "-v=1")) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } +} diff --git a/test/integration/fn_mount_cmd.go b/test/integration/fn_mount_cmd.go new file mode 100644 index 000000000..f08c90060 --- /dev/null +++ b/test/integration/fn_mount_cmd.go @@ -0,0 +1,192 @@ +// +build integration + +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "bytes" + "context" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "k8s.io/minikube/pkg/util/retry" +) + +const ( + guestMount = "/mount-9p" + createdByPod = "created-by-pod" + createdByTest = "created-by-test" + createdByTestRemovedByPod = "created-by-test-removed-by-pod" + createdByPodRemovedByTest = "created-by-pod-removed-by-test" +) + +func validateMountCmd(ctx context.Context, t *testing.T, profile string) { + if NoneDriver() { + t.Skip("skipping: none driver does not support mount") + } + + tempDir, err := ioutil.TempDir("", "mounttest") + if err != nil { + t.Fatalf("Unexpected error while creating tempDir: %v", err) + } + + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + + args := []string{"mount", "-p", profile, fmt.Sprintf("%s:%s", tempDir, guestMount), "--alsologtostderr", "-v=1"} + ss, err := Start(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Fatalf("%v failed: %v", args, err) + } + + defer func() { + if t.Failed() { + t.Logf("%s failed, getting debug info...", t.Name()) + rr, err := Run(t, exec.Command(Target(), "-p", profile, "ssh", "mount | grep 9p; ls -la /mount-9p; cat /mount-9p/pod-dates")) + if err != nil { + t.Logf("%s: %v", rr.Command(), err) + } else { + t.Logf("(debug) %s:\n%s", rr.Command(), rr.Stdout) + } + } + + // Cleanup in advance of future tests + rr, err := Run(t, exec.Command(Target(), "-p", profile, "ssh", "sudo umount -f /mount-9p")) + if err != nil { + t.Logf("%s: %v", rr.Command(), err) + } + ss.Stop(t) + cancel() + if *cleanup { + os.RemoveAll(tempDir) + } + }() + + // Write local files + testMarker := fmt.Sprintf("test-%d", time.Now().UnixNano()) + wantFromTest := []byte(testMarker) + for _, name := range []string{createdByTest, createdByTestRemovedByPod, testMarker} { + p := filepath.Join(tempDir, name) + err := ioutil.WriteFile(p, wantFromTest, 0644) + t.Logf("wrote %q to %s", wantFromTest, p) + if err != nil { + t.Errorf("WriteFile %s: %v", p, err) + } + } + + // Block until the mount succeeds to avoid file race + checkMount := func() error { + _, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "findmnt -T /mount-9p | grep 9p")) + return err + } + + start := time.Now() + if err := retry.Expo(checkMount, time.Second, 15*time.Second); err != nil { + // For local testing, allow macOS users to click prompt. If they don't, skip the test. + if runtime.GOOS == "darwin" { + t.Skip("skipping: mount did not appear, likely because macOS requires prompt to allow non-codesigned binaries to listen on non-localhost port") + } + t.Fatalf("/mount-9p did not appear within %s: %v", time.Since(start), err) + } + + // Assert that we can access the mount without an error. Display for debugging. + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "--", "ls", "-la", guestMount)) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + t.Logf("guest mount directory contents\n%s", rr.Stdout) + + // Assert that the mount contains our unique test marker, as opposed to a stale mount + tp := filepath.Join("/mount-9p", testMarker) + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "cat", tp)) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + if !bytes.Equal(rr.Stdout.Bytes(), wantFromTest) { + // The mount is hosed, exit fast before wasting time launching pods. + t.Fatalf("%s = %q, want %q", tp, rr.Stdout.Bytes(), wantFromTest) + } + + // Start the "busybox-mount" pod. + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "busybox-mount-test.yaml"))) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + if _, err := PodWait(ctx, t, profile, "default", "integration-test=busybox-mount", 2*time.Minute); err != nil { + t.Fatalf("wait: %v", err) + } + + // Read the file written by pod startup + p := filepath.Join(tempDir, createdByPod) + got, err := ioutil.ReadFile(p) + if err != nil { + t.Errorf("readfile %s: %v", p, err) + } + wantFromPod := []byte("test\n") + if !bytes.Equal(got, wantFromPod) { + t.Errorf("%s = %q, want %q", p, got, wantFromPod) + } + + // test that file written from host was read in by the pod via cat /mount-9p/written-by-host; + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", "busybox-mount")) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + if !bytes.Equal(rr.Stdout.Bytes(), wantFromTest) { + t.Errorf("busybox-mount logs = %q, want %q", rr.Stdout.Bytes(), wantFromTest) + } + + // test file timestamps are correct + for _, name := range []string{createdByTest, createdByPod} { + gp := path.Join(guestMount, name) + // test that file written from host was read in by the pod via cat /mount-9p/fromhost; + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", "stat", gp)) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + + if runtime.GOOS == "windows" { + if strings.Contains(rr.Stdout.String(), "Access: 1970-01-01") { + t.Errorf("invalid access time: %v", rr.Stdout) + } + } + + if strings.Contains(rr.Stdout.String(), "Modify: 1970-01-01") { + t.Errorf("invalid modify time: %v", rr.Stdout) + } + } + + p = filepath.Join(tempDir, createdByTestRemovedByPod) + if _, err := os.Stat(p); err == nil { + t.Errorf("expected file %s to be removed", p) + } + + p = filepath.Join(tempDir, createdByPodRemovedByTest) + if err := os.Remove(p); err != nil { + t.Errorf("unexpected error removing file %s: %v", p, err) + } +} diff --git a/test/integration/fn_pvc.go b/test/integration/fn_pvc.go new file mode 100644 index 000000000..c5d3d5f85 --- /dev/null +++ b/test/integration/fn_pvc.go @@ -0,0 +1,89 @@ +// +build integration + +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + "path/filepath" + "testing" + "time" + + core "k8s.io/api/core/v1" + storage "k8s.io/api/storage/v1" + "k8s.io/minikube/pkg/util/retry" +) + +func validatePersistentVolumeClaim(ctx context.Context, t *testing.T, profile string) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Minute) + defer cancel() + + if _, err := PodWait(ctx, t, profile, "kube-system", "integration-test=storage-provisioner", 2*time.Minute); err != nil { + t.Fatalf("wait: %v", err) + } + + checkStorageClass := func() error { + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "storageclass", "-o=json")) + if err != nil { + return err + } + scl := storage.StorageClassList{} + if err := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes())).Decode(&scl); err != nil { + return err + } + if len(scl.Items) == 0 { + return fmt.Errorf("no storageclass yet") + } + return nil + } + + // Ensure the addon-manager has created the StorageClass before creating a claim, otherwise it won't be bound + if err := retry.Expo(checkStorageClass, time.Second, 90*time.Second); err != nil { + t.Errorf("no default storage class after retry: %v", err) + } + + // Now create a testpvc + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", filepath.Join(*testdataDir, "pvc.yaml"))) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + checkStoragePhase := func() error { + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "pvc", "testpvc", "-o=json")) + if err != nil { + return err + } + pvc := core.PersistentVolumeClaim{} + if err := json.NewDecoder(bytes.NewReader(rr.Stdout.Bytes())).Decode(&pvc); err != nil { + return err + } + // The test passes if the volume claim gets bound. + if pvc.Status.Phase == "Bound" { + return nil + } + return fmt.Errorf("testpvc phase = %q, want %q (msg=%+v)", pvc.Status.Phase, "Bound", pvc) + } + + if err := retry.Expo(checkStoragePhase, 2*time.Second, 2*time.Minute); err != nil { + t.Fatalf("PV Creation failed with error: %v", err) + } +} diff --git a/test/integration/fn_tunnel_cmd.go b/test/integration/fn_tunnel_cmd.go new file mode 100644 index 000000000..2f52d5f3b --- /dev/null +++ b/test/integration/fn_tunnel_cmd.go @@ -0,0 +1,127 @@ +/* +Copyright 2018 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + "io/ioutil" + "net/http" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/pkg/errors" + + "k8s.io/minikube/pkg/kapi" + "k8s.io/minikube/pkg/minikube/tunnel" + "k8s.io/minikube/pkg/util/retry" +) + +func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Minute) + defer cancel() + + if runtime.GOOS != "windows" { + // Otherwise minikube fails waiting for a password. + if err := exec.Command("sudo", "-n", "route").Run(); err != nil { + t.Skipf("password required to execute 'route', skipping testTunnel: %v", err) + } + } + + client, err := kapi.Client(profile) + if err != nil { + t.Fatalf("client: %v", err) + } + + // Pre-Cleanup + if err := tunnel.NewManager().CleanupNotRunningTunnels(); err != nil { + t.Errorf("CleanupNotRunningTunnels: %v", err) + } + + // Start the tunnel + args := []string{"-p", profile, "tunnel", "--alsologtostderr", "-v=1"} + ss, err := Start(t, exec.CommandContext(ctx, Target(), args...)) + defer ss.Stop(t) + + // Start the "nginx" pod. + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "apply", "-f", filepath.Join(*testdataDir, "testsvc.yaml"))) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + if _, err := PodWait(ctx, t, profile, "default", "run=nginx-svc", 2*time.Minute); err != nil { + t.Fatalf("wait: %v", err) + } + + if err := kapi.WaitForService(client, "default", "nginx-svc", true, 1*time.Second, 2*time.Minute); err != nil { + t.Fatal(errors.Wrap(err, "Error waiting for nginx service to be up")) + } + + // Wait until the nginx-svc has a loadbalancer ingress IP + nginxIP := "" + err = wait.PollImmediate(1*time.Second, 3*time.Minute, func() (bool, error) { + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "svc", "nginx-svc", "-o", "-f", "jsonpath={.status.loadBalancer.ingress[0].ip}")) + if err != nil { + return false, err + } + if len(rr.Stdout.String()) > 0 { + nginxIP = rr.Stdout.String() + return true, nil + } + return false, nil + }) + if err != nil { + t.Errorf("nginx-svc svc.status.loadBalancer.ingress never got an IP") + + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "svc", "nginx-svc")) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + t.Logf("kubectl get svc nginx-svc:\n%s", rr.Stdout) + } + + // Try fetching against the IP + var resp *http.Response + + req := func() error { + h := &http.Client{Timeout: time.Second * 10} + resp, err = h.Get(fmt.Sprintf("http://%s", nginxIP)) + if err != nil { + retriable := &retry.RetriableError{Err: err} + return retriable + } + defer resp.Body.Close() + return nil + } + if err = retry.Expo(req, time.Millisecond*500, 2*time.Minute, 6); err != nil { + t.Errorf("failed to contact nginx at %s: %v", nginxIP, err) + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + t.Errorf("ReadAll: %v", err) + } + want := "Welcome to nginx!" + if !strings.Contains(string(body), want) { + t.Errorf("body = %q, want *%s*", body, want) + } +} diff --git a/test/integration/guest_env_test.go b/test/integration/guest_env_test.go new file mode 100644 index 000000000..360758637 --- /dev/null +++ b/test/integration/guest_env_test.go @@ -0,0 +1,75 @@ +// +build iso + +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "fmt" + "os/exec" + "testing" + "time" +) + +func TestGuestEnvironment(t *testing.T) { + MaybeParallel(t) + profile := UniqueProfileName("guest") + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer CleanupWithLogs(t, profile, cancel) + + args := append([]string{"start", "-p", profile, "--wait=false"}, StartArgs()...) + rr, err := Run(t, exec.CommandContext(ctx, Target(), args...)) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + + // Run as a group so that our defer doesn't happen as tests are runnings + t.Run("Binaries", func(t *testing.T) { + for _, pkg := range []string{"git", "rsync", "curl", "wget", "socat", "iptables", "VBoxControl", "VBoxService"} { + pkg := pkg + t.Run(pkg, func(t *testing.T) { + t.Parallel() + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("which %s", pkg))) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + }) + } + }) + + t.Run("PersistentMounts", func(t *testing.T) { + for _, mount := range []string{ + "/data", + "/var/lib/docker", + "/var/lib/cni", + "/var/lib/kubelet", + "/var/lib/minikube", + "/var/lib/toolbox", + "/var/lib/boot2docker", + } { + mount := mount + t.Run(mount, func(t *testing.T) { + t.Parallel() + rr, err := Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "ssh", fmt.Sprintf("df -t ext4 %s | grep %s", mount, mount))) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + }) + } + }) +} diff --git a/test/integration/gvisor_addon_test.go b/test/integration/gvisor_addon_test.go new file mode 100644 index 000000000..1eff0e5cd --- /dev/null +++ b/test/integration/gvisor_addon_test.go @@ -0,0 +1,111 @@ +// +build integration + +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "context" + "os/exec" + "path/filepath" + "testing" + "time" +) + +func TestGvisorAddon(t *testing.T) { + // TODO(tstromberg): Fix or remove addon. + t.Skip("SKIPPING: Currently broken (gvisor-containerd-shim.toml CrashLoopBackoff): https://github.com/kubernetes/minikube/issues/5305") + + if NoneDriver() { + t.Skip("Can't run containerd backend with none driver") + } + MaybeParallel(t) + + profile := UniqueProfileName("gvisor") + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer func() { + CleanupWithLogs(t, profile, cancel) + }() + + startArgs := append([]string{"start", "-p", profile, "--container-runtime=containerd", "--docker-opt", "containerd=/var/run/containerd/containerd.sock", "--wait=false"}, StartArgs()...) + rr, err := Run(t, exec.CommandContext(ctx, Target(), startArgs...)) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + // TODO: Re-examine if we should be pulling in an image which users don't normally invoke + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "cache", "add", "gcr.io/k8s-minikube/gvisor-addon:latest")) + if err != nil { + t.Errorf("%s failed: %v", rr.Args, err) + } + + // NOTE: addons are global, but the addon must assert that the runtime is containerd + rr, err = Run(t, exec.CommandContext(ctx, Target(), "-p", profile, "addons", "enable", "gvisor")) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + // Because addons are persistent across profiles :( + defer func() { + rr, err := Run(t, exec.Command(Target(), "-p", profile, "addons", "disable", "gvisor")) + if err != nil { + t.Logf("%s failed: %v", rr.Args, err) + } + }() + + if _, err := PodWait(ctx, t, profile, "kube-system", "kubernetes.io/minikube-addons=gvisor", 2*time.Minute); err != nil { + t.Fatalf("waiting for gvisor controller to be up: %v", err) + } + + // Create an untrusted workload + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-untrusted.yaml"))) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + // Create gvisor workload + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "replace", "--force", "-f", filepath.Join(*testdataDir, "nginx-gvisor.yaml"))) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + if _, err := PodWait(ctx, t, profile, "default", "run=nginx,untrusted=true", 2*time.Minute); err != nil { + t.Errorf("nginx: %v", err) + } + if _, err := PodWait(ctx, t, profile, "default", "run=nginx,runtime=gvisor", 2*time.Minute); err != nil { + t.Errorf("nginx: %v", err) + } + + // Ensure that workloads survive a restart + rr, err = Run(t, exec.CommandContext(ctx, Target(), "stop", "-p", profile)) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + + rr, err = Run(t, exec.CommandContext(ctx, Target(), startArgs...)) + if err != nil { + t.Fatalf("%s failed: %v", rr.Args, err) + } + if _, err := PodWait(ctx, t, profile, "kube-system", "kubernetes.io/minikube-addons=gvisor", 2*time.Minute); err != nil { + t.Errorf("waiting for gvisor controller to be up: %v", err) + } + if _, err := PodWait(ctx, t, profile, "default", "run=nginx,untrusted=true", 2*time.Minute); err != nil { + t.Errorf("nginx: %v", err) + } + if _, err := PodWait(ctx, t, profile, "default", "run=nginx,runtime=gvisor", 2*time.Minute); err != nil { + t.Errorf("nginx: %v", err) + } +} diff --git a/test/integration/helpers.go b/test/integration/helpers.go new file mode 100644 index 000000000..47af15a26 --- /dev/null +++ b/test/integration/helpers.go @@ -0,0 +1,353 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +// These are test helpers that: +// +// - Accept *testing.T arguments (see helpers.go) +// - Are used in multiple tests +// - Must not compare test values + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io/ioutil" + "os/exec" + "strings" + "testing" + "time" + + "github.com/shirou/gopsutil/process" + core "k8s.io/api/core/v1" + meta "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/minikube/pkg/kapi" +) + +// RunResult stores the result of an cmd.Run call +type RunResult struct { + Stdout *bytes.Buffer + Stderr *bytes.Buffer + ExitCode int + Args []string +} + +// Command returns a human readable command string that does not induce eye fatigue +func (rr RunResult) Command() string { + var sb strings.Builder + sb.WriteString(strings.TrimPrefix(rr.Args[0], "../../")) + for _, a := range rr.Args[1:] { + if strings.Contains(a, " ") { + sb.WriteString(fmt.Sprintf(` "%s"`, a)) + continue + } + sb.WriteString(fmt.Sprintf(" %s", a)) + } + return sb.String() +} + +func (rr RunResult) String() string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("Command: %v\n", rr.Command())) + if rr.Stdout.Len() > 0 { + sb.WriteString(fmt.Sprintf("\n-- stdout -- \n%s\n", rr.Stdout.Bytes())) + } + if rr.Stderr.Len() > 0 { + sb.WriteString(fmt.Sprintf("\n** stderr ** \n%s\n", rr.Stderr.Bytes())) + } + return sb.String() +} + +// Run is a test helper to log a command being executed \_(ツ)_/¯ +func Run(t *testing.T, cmd *exec.Cmd) (*RunResult, error) { + t.Helper() + rr := &RunResult{Args: cmd.Args} + t.Logf("(dbg) Run: %v", rr.Command()) + + var outb, errb bytes.Buffer + cmd.Stdout, rr.Stdout = &outb, &outb + cmd.Stderr, rr.Stderr = &errb, &errb + start := time.Now() + err := cmd.Run() + elapsed := time.Since(start) + if err == nil { + // Reduce log spam + if elapsed > (1 * time.Second) { + t.Logf("(dbg) Done: %v: (%s)", rr.Command(), elapsed) + } + } else { + if exitError, ok := err.(*exec.ExitError); ok { + rr.ExitCode = exitError.ExitCode() + } + t.Logf("(dbg) Non-zero exit: %v: %v (%s)", rr.Command(), err, elapsed) + t.Logf("(dbg) %s", rr.String()) + } + return rr, err +} + +// StartSession stores the result of an cmd.Start call +type StartSession struct { + Stdout *bufio.Reader + Stderr *bufio.Reader + cmd *exec.Cmd +} + +// Start starts a process in the background, streaming output +func Start(t *testing.T, cmd *exec.Cmd) (*StartSession, error) { + t.Helper() + t.Logf("Daemon: %v", cmd.Args) + + stdoutPipe, err := cmd.StdoutPipe() + if err != nil { + t.Fatalf("stdout pipe failed: %v %v", cmd.Args, err) + } + stderrPipe, err := cmd.StderrPipe() + if err != nil { + t.Fatalf("stderr pipe failed: %v %v", cmd.Args, err) + } + + sr := &StartSession{Stdout: bufio.NewReader(stdoutPipe), Stderr: bufio.NewReader(stderrPipe), cmd: cmd} + return sr, cmd.Start() +} + +// Stop stops the started process +func (ss *StartSession) Stop(t *testing.T) { + t.Helper() + t.Logf("Stopping %s ...", ss.cmd.Args) + if ss.cmd.Process == nil { + t.Logf("%s has a nil Process. Maybe it's dead? How weird!", ss.cmd.Args) + return + } + killProcessFamily(t, ss.cmd.Process.Pid) + if t.Failed() { + if ss.Stdout.Size() > 0 { + stdout, err := ioutil.ReadAll(ss.Stdout) + if err != nil { + t.Logf("read stdout failed: %v", err) + } + t.Logf("(dbg) %s stdout:\n%s", ss.cmd.Args, stdout) + } + if ss.Stderr.Size() > 0 { + stderr, err := ioutil.ReadAll(ss.Stderr) + if err != nil { + t.Logf("read stderr failed: %v", err) + } + t.Logf("(dbg) %s stderr:\n%s", ss.cmd.Args, stderr) + } + } +} + +// Cleanup cleans up after a test run +func Cleanup(t *testing.T, profile string, cancel context.CancelFunc) { + // No helper because it makes the call log confusing. + if *cleanup { + _, err := Run(t, exec.Command(Target(), "delete", "-p", profile)) + if err != nil { + t.Logf("failed cleanup: %v", err) + } + } else { + t.Logf("Skipping cleanup of %s (--cleanup=false)", profile) + } + cancel() +} + +// CleanupWithLogs cleans up after a test run, fetching logs and deleting the profile +func CleanupWithLogs(t *testing.T, profile string, cancel context.CancelFunc) { + t.Helper() + if t.Failed() && *postMortemLogs { + t.Logf("%s failed, collecting logs ...", t.Name()) + rr, err := Run(t, exec.Command(Target(), "-p", profile, "logs", "-n", "10")) + if err != nil { + t.Logf("failed logs error: %v", err) + } + t.Logf("%s logs: %s\n", t.Name(), rr) + t.Logf("Sorry that %s failed :(", t.Name()) + } + Cleanup(t, profile, cancel) +} + +// podStatusMsg returns a human-readable pod status, for generating debug status +func podStatusMsg(pod core.Pod) string { + var sb strings.Builder + sb.WriteString(fmt.Sprintf("%q [%s] %s", pod.ObjectMeta.GetName(), pod.ObjectMeta.GetUID(), pod.Status.Phase)) + for i, c := range pod.Status.Conditions { + if c.Reason != "" { + if i == 0 { + sb.WriteString(": ") + } else { + sb.WriteString(" / ") + } + sb.WriteString(fmt.Sprintf("%s:%s", c.Type, c.Reason)) + } + if c.Message != "" { + sb.WriteString(fmt.Sprintf(" (%s)", c.Message)) + } + } + return sb.String() +} + +// PodWait waits for pods to achieve a running state. +func PodWait(ctx context.Context, t *testing.T, profile string, ns string, selector string, timeout time.Duration) ([]string, error) { + t.Helper() + client, err := kapi.Client(profile) + if err != nil { + return nil, err + } + + // For example: kubernetes.io/minikube-addons=gvisor + listOpts := meta.ListOptions{LabelSelector: selector} + minUptime := 5 * time.Second + podStart := time.Time{} + foundNames := map[string]bool{} + lastMsg := "" + + start := time.Now() + t.Logf("Waiting for pods with labels %q in namespace %q ...", selector, ns) + f := func() (bool, error) { + pods, err := client.CoreV1().Pods(ns).List(listOpts) + if err != nil { + t.Logf("Pod(%s).List(%v) returned error: %v", ns, selector, err) + // Don't bother to retry: something is very wrong. + return true, err + } + if len(pods.Items) == 0 { + podStart = time.Time{} + return false, nil + } + + for _, pod := range pods.Items { + foundNames[pod.ObjectMeta.Name] = true + msg := podStatusMsg(pod) + // Prevent spamming logs with identical messages + if msg != lastMsg { + t.Log(msg) + lastMsg = msg + } + // Successful termination of a short-lived process, will not be restarted + if pod.Status.Phase == core.PodSucceeded { + return true, nil + } + // Long-running process state + if pod.Status.Phase != core.PodRunning { + if !podStart.IsZero() { + t.Logf("WARNING: %s was running %s ago - may be unstable", selector, time.Since(podStart)) + } + podStart = time.Time{} + return false, nil + } + + if podStart.IsZero() { + podStart = time.Now() + } + + if time.Since(podStart) > minUptime { + return true, nil + } + } + return false, nil + } + + err = wait.PollImmediate(500*time.Millisecond, timeout, f) + names := []string{} + for n := range foundNames { + names = append(names, n) + } + + if err == nil { + t.Logf("pods %s up and healthy within %s", selector, time.Since(start)) + return names, nil + } + + t.Logf("pods %q: %v", selector, err) + showPodLogs(ctx, t, profile, ns, names) + return names, fmt.Errorf("%s: %v", fmt.Sprintf("%s within %s", selector, timeout), err) +} + +// showPodLogs logs debug info for pods +func showPodLogs(ctx context.Context, t *testing.T, profile string, ns string, names []string) { + rr, rerr := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "get", "po", "-A", "--show-labels")) + if rerr != nil { + t.Logf("%s: %v", rr.Command(), rerr) + // return now, because kubectl is hosed + return + } + t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout) + + for _, name := range names { + rr, err := Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "describe", "po", name, "-n", ns)) + if err != nil { + t.Logf("%s: %v", rr.Command(), err) + } else { + t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout) + } + + rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "logs", name, "-n", ns)) + if err != nil { + t.Logf("%s: %v", rr.Command(), err) + } else { + t.Logf("(dbg) %s:\n%s", rr.Command(), rr.Stdout) + } + } +} + +// Status returns the minikube cluster status as a string +func Status(ctx context.Context, t *testing.T, path string, profile string) string { + t.Helper() + rr, err := Run(t, exec.CommandContext(ctx, path, "status", "--format={{.Host}}", "-p", profile)) + if err != nil { + t.Logf("status error: %v (may be ok)", err) + } + return strings.TrimSpace(rr.Stdout.String()) +} + +// MaybeParallel sets that the test should run in parallel +func MaybeParallel(t *testing.T) { + t.Helper() + // TODO: Allow paralellized tests on "none" that do not require independent clusters + if NoneDriver() { + return + } + t.Parallel() +} + +// killProcessFamily kills a pid and all of its children +func killProcessFamily(t *testing.T, pid int) { + parent, err := process.NewProcess(int32(pid)) + if err != nil { + t.Logf("unable to find parent, assuming dead: %v", err) + return + } + procs := []*process.Process{} + children, err := parent.Children() + if err == nil { + procs = append(procs, children...) + } + procs = append(procs, parent) + + for _, p := range procs { + if err := p.Terminate(); err != nil { + t.Logf("unable to terminate pid %d: %v", p.Pid, err) + continue + } + if err := p.Kill(); err != nil { + t.Logf("unable to kill pid %d: %v", p.Pid, err) + continue + } + } +} diff --git a/test/integration/main.go b/test/integration/main.go new file mode 100644 index 000000000..10861fafa --- /dev/null +++ b/test/integration/main.go @@ -0,0 +1,57 @@ +/* +Copyright 2016 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "flag" + "os" + "strings" + "testing" +) + +// General configuration: used to set the VM Driver +var startArgs = flag.String("minikube-start-args", "", "Arguments to pass to minikube start") + +// Flags for faster local integration testing +var forceProfile = flag.String("profile", "", "force tests to run against a particular profile") +var cleanup = flag.Bool("cleanup", true, "cleanup failed test run") +var postMortemLogs = flag.Bool("postmortem-logs", true, "show logs after a failed test run") + +// Paths to files - normally set for CI +var binaryPath = flag.String("binary", "../../out/minikube", "path to minikube binary") +var testdataDir = flag.String("testdata-dir", "testdata", "the directory relative to test/integration where the testdata lives") + +// TestMain is the test main +func TestMain(m *testing.M) { + flag.Parse() + os.Exit(m.Run()) +} + +// StartArgs returns the arguments normally used for starting minikube +func StartArgs() []string { + return strings.Split(*startArgs, " ") +} + +// Target returns where the minikube binary can be found +func Target() string { + return *binaryPath +} + +// NoneDriver returns whether or not this test is using the none driver +func NoneDriver() bool { + return strings.Contains(*startArgs, "--vm-driver=none") +} diff --git a/test/integration/util.go b/test/integration/util.go new file mode 100644 index 000000000..f3d7c1916 --- /dev/null +++ b/test/integration/util.go @@ -0,0 +1,60 @@ +/* +Copyright 2019 The Kubernetes Authors All rights reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package integration + +import ( + "bufio" + "fmt" + "os" + "time" +) + +// ReadLineWithTimeout reads a line of text from a buffer with a timeout +func ReadLineWithTimeout(b *bufio.Reader, timeout time.Duration) (string, error) { + s := make(chan string) + e := make(chan error) + go func() { + read, err := b.ReadString('\n') + if err != nil { + e <- err + } else { + s <- read + } + close(s) + close(e) + }() + + select { + case line := <-s: + return line, nil + case err := <-e: + return "", err + case <-time.After(timeout): + return "", fmt.Errorf("timeout after %s", timeout) + } +} + +// UniqueProfileName returns a reasonably unique profile name +func UniqueProfileName(prefix string) string { + if *forceProfile != "" { + return *forceProfile + } + if NoneDriver() { + return "minikube" + } + return fmt.Sprintf("%s-%s-%d", prefix, time.Now().Format("20060102T150405.999999999"), os.Getpid()) +} -- GitLab