Merge pull request #7202 from medyagh/gha_pass

adjust github actions and fix test helpers

.github/workflows/main.yml

@@ -113,7 +113,7 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -test.timeout=80m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker -test.timeout=80m -test.v -timeout-multiplier=1.3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
@@ -145,6 +145,8 @@ jobs:
         echo "----------------${numFail} Failures----------------------------"
         echo $STAT | jq '.FailedTests' || true
         echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   docker_ubuntu_18_04:
     runs-on: ubuntu-18.04
@@ -186,7 +188,7 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.timeout=70m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker -test.timeout=80m -test.v -timeout-multiplier=1.3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
@@ -218,6 +220,8 @@ jobs:
         echo "----------------${numFail} Failures----------------------------"
         echo $STAT | jq '.FailedTests' || true
         echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   none_ubuntu16_04:
     needs: [build_minikube]
@@ -254,7 +258,7 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=70m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=50m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
@@ -286,6 +290,8 @@ jobs:
         echo "----------------${numFail} Failures----------------------------"
         echo $STAT | jq '.FailedTests' || true
         echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   none_ubuntu18_04:
     needs: [build_minikube]
@@ -322,7 +328,7 @@ jobs:
         chmod a+x e2e-*
         chmod a+x minikube-*
         START_TIME=$(date -u +%s)
-        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=70m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+        KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=50m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
         END_TIME=$(date -u +%s)
         TIME_ELAPSED=$(($END_TIME-$START_TIME))
         min=$((${TIME_ELAPSED}/60))
@@ -354,6 +360,8 @@ jobs:
         echo "----------------${numFail} Failures----------------------------"
         echo $STAT | jq '.FailedTests' || true
         echo "-------------------------------------------------------"
+        numPass=$(echo $STAT | jq '.NumberOfPass')
+        echo "*** $numPass Passed ***"
         if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   podman_ubuntu_18_04:
       needs: [build_minikube]
@@ -400,7 +408,7 @@ jobs:
           chmod a+x e2e-*
           chmod a+x minikube-*
           START_TIME=$(date -u +%s)
-          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=podman -test.timeout=70m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
+          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=podman -test.timeout=70m -test.v -timeout-multiplier=1.3 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
           END_TIME=$(date -u +%s)
           TIME_ELAPSED=$(($END_TIME-$START_TIME))
           min=$((${TIME_ELAPSED}/60))
@@ -432,6 +440,8 @@ jobs:
           echo "----------------${numFail} Failures----------------------------"
           echo $STAT | jq '.FailedTests' || true
           echo "-------------------------------------------------------"
+          numPass=$(echo $STAT | jq '.NumberOfPass')
+          echo "*** $numPass Passed ***"
           if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
   # After all 4 integration tests finished
   # collect all the reports and upload

test/integration/aaa_download_only_test.go

@@ -29,7 +29,6 @@ import (
 	"runtime"
 	"strings"
 	"testing"
-	"time"
 
 	"k8s.io/minikube/pkg/minikube/bootstrapper/images"
 	"k8s.io/minikube/pkg/minikube/constants"
@@ -147,16 +146,16 @@ func TestDownloadOnly(t *testing.T) {
 	}
 }
 
-func TestDownloadOnlyDocker(t *testing.T) {
-	if !runningDockerDriver(StartArgs()) {
-		t.Skip("this test only runs with the docker driver")
+func TestDownloadOnlyKic(t *testing.T) {
+	if !KicDriver() {
+		t.Skip("skipping, only for docker or podman driver")
 	}
-
 	profile := UniqueProfileName("download-docker")
-	ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), Minutes(15))
 	defer Cleanup(t, profile, cancel)
 
-	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr", "--driver=docker"}
+	args := []string{"start", "--download-only", "-p", profile, "--force", "--alsologtostderr"}
+	args = append(args, StartArgs()...)
 	rr, err := Run(t, exec.CommandContext(ctx, Target(), args...))
 	if err != nil {
 		t.Errorf("%s failed: %v:\n%s", args, err, rr.Output())
@@ -178,12 +177,3 @@ func TestDownloadOnlyDocker(t *testing.T) {
 		t.Errorf("checksum of %s does not match remote checksum (%s != %s)", tarball, string(remoteChecksum), string(checksum[:]))
 	}
 }
-
-func runningDockerDriver(startArgs []string) bool {
-	for _, s := range startArgs {
-		if s == "--driver=docker" {
-			return true
-		}
-	}
-	return false
-}

test/integration/addons_test.go

@@ -129,7 +129,7 @@ func validateIngressAddon(ctx context.Context, t *testing.T, profile string) {
 		return nil
 	}
 
-	if err := retry.Expo(checkIngress, 500*time.Millisecond, Minutes(1)); err != nil {
+	if err := retry.Expo(checkIngress, 500*time.Millisecond, Seconds(90)); err != nil {
 		t.Errorf("ingress never responded as expected on 127.0.0.1:80: %v", err)
 	}
 
@@ -241,7 +241,7 @@ func validateMetricsServerAddon(ctx context.Context, t *testing.T, profile strin
 	}
 
 	// metrics-server takes some time to be able to collect metrics
-	if err := retry.Expo(checkMetricsServer, Seconds(13), Minutes(6)); err != nil {
+	if err := retry.Expo(checkMetricsServer, time.Second*3, Minutes(6)); err != nil {
 		t.Errorf(err.Error())
 	}
 

test/integration/fn_mount_cmd.go

@@ -106,7 +106,7 @@ func validateMountCmd(ctx context.Context, t *testing.T, profile string) {
 	}
 
 	start := time.Now()
-	if err := retry.Expo(checkMount, time.Second, 15*time.Second); err != nil {
+	if err := retry.Expo(checkMount, time.Millisecond*500, Seconds(15)); err != nil {
 		// For local testing, allow macOS users to click prompt. If they don't, skip the test.
 		if runtime.GOOS == "darwin" {
 			t.Skip("skipping: mount did not appear, likely because macOS requires prompt to allow non-codesigned binaries to listen on non-localhost port")

test/integration/fn_pvc.go

@@ -57,7 +57,7 @@ func validatePersistentVolumeClaim(ctx context.Context, t *testing.T, profile st
 	}
 
 	// Ensure the addon-manager has created the StorageClass before creating a claim, otherwise it won't be bound
-	if err := retry.Expo(checkStorageClass, time.Second, 90*time.Second); err != nil {
+	if err := retry.Expo(checkStorageClass, time.Millisecond*500, Seconds(100)); err != nil {
 		t.Errorf("no default storage class after retry: %v", err)
 	}
 

test/integration/fn_tunnel_cmd.go

@@ -119,7 +119,7 @@ func validateTunnelCmd(ctx context.Context, t *testing.T, profile string) {
 		}
 		return nil
 	}
-	if err = retry.Expo(fetch, time.Millisecond*500, Minutes(2), 6); err != nil {
+	if err = retry.Expo(fetch, time.Millisecond*500, Minutes(2), 13); err != nil {
 		t.Errorf("failed to contact nginx at %s: %v", nginxIP, err)
 	}
 

test/integration/functional_test.go

@@ -725,7 +725,7 @@ func validateMySQL(ctx context.Context, t *testing.T, profile string) {
 		rr, err = Run(t, exec.CommandContext(ctx, "kubectl", "--context", profile, "exec", names[0], "--", "mysql", "-ppassword", "-e", "show databases;"))
 		return err
 	}
-	if err = retry.Expo(mysql, 5*time.Second, 180*time.Second); err != nil {
+	if err = retry.Expo(mysql, 2*time.Second, Seconds(180)); err != nil {
 		t.Errorf("mysql failing: %v", err)
 	}
 }

test/integration/main.go

@@ -68,6 +68,11 @@ func HyperVDriver() bool {
 	return strings.Contains(*startArgs, "--driver=hyperv") || strings.Contains(*startArgs, "--vm-driver=hyperv")
 }
 
+// KicDriver returns whether or not this test is using the docker or podman driver
+func KicDriver() bool {
+	return strings.Contains(*startArgs, "--driver=docker") || strings.Contains(*startArgs, "--vm-driver=docker") || strings.Contains(*startArgs, "--vm-driver=podman") || strings.Contains(*startArgs, "driver=podman")
+}
+
 // CanCleanup returns if cleanup is allowed
 func CanCleanup() bool {
 	return *cleanup

test/integration/version_upgrade_test.go

@@ -75,8 +75,8 @@ func TestVersionUpgrade(t *testing.T) {
 		return err
 	}
 
-	// Retry to allow flakiness for the previous release
-	if err := retry.Expo(r, 1*time.Second, Minutes(30), 3); err != nil {
+	// Retry up to two times, to allow flakiness for the previous release
+	if err := retry.Expo(r, 1*time.Second, Minutes(30), 2); err != nil {
 		t.Fatalf("release start failed: %v", err)
 	}
 
@@ -120,14 +120,8 @@ func TestVersionUpgrade(t *testing.T) {
 	}
 
 	args = append([]string{"start", "-p", profile, fmt.Sprintf("--kubernetes-version=%s", constants.OldestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)
-	rr = &RunResult{}
-	r = func() error {
-		rr, err = Run(t, exec.CommandContext(ctx, tf.Name(), args...))
-		return err
-	}
-
-	if err := retry.Expo(r, 1*time.Second, Minutes(30), 3); err == nil {
-		t.Fatalf("downgrading kubernetes should not be allowed: %v", err)
+	if rr, err := Run(t, exec.CommandContext(ctx, tf.Name(), args...)); err == nil {
+		t.Fatalf("downgrading kubernetes should not be allowed. expected to see error but got %v for %q", err, rr.Args)
 	}
 
 	args = append([]string{"start", "-p", profile, fmt.Sprintf("--kubernetes-version=%s", constants.NewestKubernetesVersion), "--alsologtostderr", "-v=1"}, StartArgs()...)