master.yml

name: Master
on:
  push:
    branches:
      - master
    paths:
      - "**.go"
      - "**.yml"
      - "Makefile"
env:
  GOPROXY: https://proxy.golang.org
jobs:
  # Runs before all other jobs
  # builds the minikube binaries
  build_minikube:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v2
      - name: Download Dependencies
        run: go mod download
      - name: Build Binaries
        run: |
          make minikube-linux-amd64
          make e2e-linux-amd64
          make windows
          make e2e-windows-amd64.exe
          cp -r test/integration/testdata ./out
          whoami
          echo github ref $GITHUB_REF
          echo workflow $GITHUB_WORKFLOW
          echo home $HOME
          echo event name $GITHUB_EVENT_NAME
          echo workspace $GITHUB_WORKSPACE
          echo "end of debug stuff"
          echo $(which jq)
      - uses: actions/upload-artifact@v1
        with:
          name: minikube_binaries
          path: out
  lint:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v2
      - name: Install libvirt
        run: |
          sudo apt-get update
          sudo apt-get install -y libvirt-dev
      - name: Download Dependencies
        run: go mod download
      - name: Lint
        env:
          TESTSUITE: lintall
        run: make test
        continue-on-error: false
  unit_test:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v2
      - name: Install libvirt
        run: |
          sudo apt-get update
          sudo apt-get install -y libvirt-dev
      - name: Download Dependencies
        run: go mod download
      - name: Unit Test
        env:
          TESTSUITE: unittest
        run: make test
        continue-on-error: false
  # Run the following integration tests after the build_minikube
  # They will run in parallel and use the binaries in previous step
  functional_docker_ubuntu:
    needs: [build_minikube]
    env:
      TIME_ELAPSED: time
      JOB_NAME: "functional_docker_ubuntu"
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
    runs-on: ubuntu-18.04
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true
      - name: Docker Info
        shell: bash
        run: |
          echo "--------------------------"
          docker version || true
          echo "--------------------------"
          docker info || true
          echo "--------------------------"
          docker system df || true
          echo "--------------------------"
          docker system info || true
          echo "--------------------------"
          docker ps || true
          echo "--------------------------"
      - name: Install gopogh
        shell: bash
        run: |
          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-linux-amd64
          sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
      - name: Download Binaries
        uses: actions/download-artifact@v1
        with:
          name: minikube_binaries
      - name: Run Integration Test
        continue-on-error: false
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--vm-driver=docker  -test.run TestFunctional -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
          echo ::set-env name=STAT::${STAT}
      - uses: actions/upload-artifact@v1
        with:
          name: functional_docker_ubuntu
          path: minikube_binaries/report
      - name: The End Result functional_docker_ubuntu
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          echo "----------------${numFail} Failures----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*** $numPass Passed ***"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 36 ];then echo "*** Failed to pass at least 36 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
  functional_docker_windows:
    needs: [build_minikube]
    env:
      TIME_ELAPSED: time
      JOB_NAME: "functional_docker_windows"
      GOPOGH_RESULT: ""
      STAT: ""
    runs-on: [self-hosted, windows-10-ent, 8CPUs]
    steps:
      - name: Clean up
        continue-on-error: true
        shell: powershell
        run: |
          echo $env:computerName
          ls 
          $ErrorActionPreference = "SilentlyContinue"
          cd minikube_binaries
          ls
          $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
          $env:MINIKUBE_HOME="${pwd}\testhome"
          .\minikube-windows-amd64.exe delete --all --purge
          Get-VM | Where-Object {$_.Name -ne "DockerDesktopVM"} | Foreach {
            minikube-windows-amd64.exe delete -p $_.Name
            Suspend-VM $_.Name
            Stop-VM $_.Name -Force
            Remove-VM $_.Name -Force
          }
          cd ..
          Remove-Item minikube_binaries -Force -Recurse
          ls
      - name: Download Binaries
        uses: actions/download-artifact@v1
        with:
          name: minikube_binaries
      - name: Info
        shell: powershell
        run: |
          echo $env:computername
          echo "------------------------"
          docker info
          echo "------------------------"
          docker volume ls
          echo "------------------------"
      - name: Install tools
        continue-on-error: true
        shell: powershell
        run: |
          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")
          choco install -y kubernetes-cli
          choco install -y jq
          if (Test-Path 'C:\Program Files\Docker\Docker\resources\bin\kubectl.exe') { Remove-Item 'C:\Program Files\Docker\Docker\resources\bin\kubectl.exe' };
      - name: Run Integration Test in powershell
        continue-on-error: true
        shell: powershell
        run: |
          cd minikube_binaries
          New-Item -Force -Path "report" -ItemType Directory
          New-Item -Force -Path "testhome" -ItemType Directory
          $START_TIME=(GET-DATE)
          $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
          $env:MINIKUBE_HOME="${pwd}\testhome"
          $ErrorActionPreference = "SilentlyContinue"
          .\e2e-windows-amd64.exe --minikube-start-args="--driver=docker" --test.timeout=10m --timeout-multiplier=1 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Tee-Object -FilePath ".\report\testout.txt"
          $END_TIME=(GET-DATE)
          echo $END_TIME
          $DURATION=(NEW-TIMESPAN -Start $START_TIME -End $END_TIME)
          echo $DURATION
          $SECS=($DURATION.TotalSeconds)
          $MINS=($DURATION.TotalMinutes)
          $T_ELAPSED="$MINS m $SECS s"
          echo "----"
          echo $T_ELAPSED
          echo "----"
          echo "::set-env name=TIME_ELAPSED::$T_ELAPSED"
      - name: Generate HTML Report
        continue-on-error: true
        shell: powershell
        run: |
          cd minikube_binaries
          Get-Content .\report\testout.txt -Encoding ASCII  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding ASCII
          $STAT=(gopogh -in .\report\testout.json -out .\report\testout.html -name "${Env:JOB_NAME} ${Env:GITHUB_REF}" -repo "${Env:GITHUB_REPOSITORY}"  -details "${Env:GITHUB_SHA}")
          echo status: ${STAT}
          $FailNum=$(echo $STAT | jq '.NumberOfFail')
          $TestsNum=$(echo $STAT | jq '.NumberOfTests')
          $GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${Env:TIME_ELAPSED}"
          echo "::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}"
          echo "::set-env name=STAT::${STAT}"
          echo ${GOPOGH_RESULT}
          $numFail=(echo $STAT | jq '.NumberOfFail')
          $failedTests=( echo $STAT | jq '.FailedTests')
          echo "----------------${numFail} Failures----------------------------"
          echo $failedTest
          echo "-------------------------------------------------------"
          $numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*** $numPass Passed ***"
      - uses: actions/upload-artifact@v1
        with:
          name: functional_docker_windows
          path: minikube_binaries/report
      - name: The End Result functional_docker_windows
        shell: powershell
        run: |
          $numFail=(echo $Env:STAT | jq '.NumberOfFail')
          $failedTests=( echo $Env:STAT | jq '.FailedTests')
          echo "----------------${numFail} Failures----------------------------"
          echo $failedTests
          echo "-------------------------------------------------------"
          $numPass=$(echo $Env:STAT | jq '.NumberOfPass')
          echo "*** $numPass Passed ***"
          If ($numFail -gt 0){ exit 2 } 
          If ($numPass -eq 0){ exit 2 }
          If ($numPass -lt 33){ exit 2 }
          If ($numFail -eq 0){ exit 0 }
  functional_hyperv_windows:
    needs: [build_minikube]
    env:
      TIME_ELAPSED: time
      JOB_NAME: "functional_hyperv_windows"
      GOPOGH_RESULT: ""
    runs-on: [self-hosted, windows-10-ent, Standard_D16s_v3, hyperv]
    steps:
      - name: Clean up
        continue-on-error: true
        shell: powershell
        run: |
          echo $env:computerName
          ls 
          $ErrorActionPreference = "SilentlyContinue"
          cd minikube_binaries
          ls
          $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
          $env:MINIKUBE_HOME="${pwd}\testhome"
          .\minikube-windows-amd64.exe delete --all --purge
          Get-VM | Where-Object {$_.Name -ne "DockerDesktopVM"} | Foreach {
            minikube-windows-amd64.exe delete -p $_.Name
            Suspend-VM $_.Name
            Stop-VM $_.Name -Force
            Remove-VM $_.Name -Force
          }
          cd ..
          Remove-Item minikube_binaries -Force -Recurse
          ls
      - name: Download Binaries
        uses: actions/download-artifact@v1
        with:
          name: minikube_binaries
      - name: Info
        shell: powershell
        run: |
          echo $env:computername
          Get-WmiObject -class Win32_ComputerSystem
      - name: Run Integration Test in powershell
        continue-on-error: true
        shell: powershell
        run: |
          cd minikube_binaries
          New-Item -Force -Path "report" -ItemType Directory
          New-Item -Force -Path "testhome" -ItemType Directory
          $START_TIME=(GET-DATE)
          $env:KUBECONFIG="${pwd}\testhome\kubeconfig"
          $env:MINIKUBE_HOME="${pwd}\testhome"
          $ErrorActionPreference = "SilentlyContinue"
          .\e2e-windows-amd64.exe --minikube-start-args="--driver=hyperv" --test.timeout=13m --timeout-multiplier=1.5 --test.v --test.run=TestFunctional --binary=./minikube-windows-amd64.exe | Tee-Object -FilePath ".\report\testout.txt"
          $END_TIME=(GET-DATE)
          echo $END_TIME
          $DURATION=(NEW-TIMESPAN -Start $START_TIME -End $END_TIME)
          echo $DURATION
          $SECS=($DURATION.TotalSeconds)
          $MINS=($DURATION.TotalMinutes)
          $T_ELAPSED="$MINS m $SECS s"
          echo "----"
          echo $T_ELAPSED
          echo "----"
          echo "::set-env name=TIME_ELAPSED::$T_ELAPSED"
      - name: Generate HTML Report
        continue-on-error: true
        shell: powershell
        run: |
          cd minikube_binaries
          Get-Content .\report\testout.txt -Encoding ASCII  | go tool test2json -t | Out-File -FilePath .\report\testout.json -Encoding ASCII
          $STAT=(gopogh -in .\report\testout.json -out .\report\testout.html -name "${Env:JOB_NAME} ${Env:GITHUB_REF}" -repo "${Env:GITHUB_REPOSITORY}"  -details "${Env:GITHUB_SHA}")
          echo status: ${STAT}
          $FailNum=$(echo $STAT | jq '.NumberOfFail')
          $TestsNum=$(echo $STAT | jq '.NumberOfTests')
          $GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${Env:TIME_ELAPSED}"
          echo "::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}"
          echo "::set-env name=STAT::${STAT}"
          echo ${GOPOGH_RESULT}
          $numFail=(echo $STAT | jq '.NumberOfFail')
          $failedTests=( echo $STAT | jq '.FailedTests')
          echo "----------------${numFail} Failures----------------------------"
          echo $failedTest
          echo "-------------------------------------------------------"
          $numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*** $numPass Passed ***"
      - uses: actions/upload-artifact@v1
        with:
          name: functional_hyperv_windows
          path: minikube_binaries/report
      - name: Install tools
        continue-on-error: true
        shell: powershell
        run: |
          $ErrorActionPreference = "SilentlyContinue"
          (New-Object Net.WebClient).DownloadFile("https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh.exe", "C:\ProgramData\chocolatey\bin\gopogh.exe")          
          choco install -y kubernetes-cli
          choco install -y jq
          if (Test-Path 'C:\Program Files\Docker\Docker\resources\bin\kubectl.exe') { Remove-Item 'C:\Program Files\Docker\Docker\resources\bin\kubectl.exe' };
      - name: The End Result functional_hyperv_windows
        shell: powershell
        run: |
          $numFail=(echo $Env:STAT | jq '.NumberOfFail')
          $failedTests=( echo $Env:STAT | jq '.FailedTests')
          echo "----------------${numFail} Failures----------------------------"
          echo $failedTests
          echo "-------------------------------------------------------"
          $numPass=$(echo $Env:STAT | jq '.NumberOfPass')
          echo "*** $numPass Passed ***"
          If ($numFail -gt 0){ exit 2 } 
          If ($numPass -eq 0){ exit 2 }
          If ($numPass -lt 33){ exit 2 }
          If ($numFail -eq 0){ exit 0 }
  addons_certs_docker_ubuntu:
    runs-on: ubuntu-18.04
    env:
      TIME_ELAPSED: time
      JOB_NAME: "addons_certs_docker_ubuntu"
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
    needs: [build_minikube]
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true
      - name: Docker Info
        shell: bash
        run: |
          echo "--------------------------"
          docker version || true
          echo "--------------------------"
          docker info || true
          echo "--------------------------"
          docker system df || true
          echo "--------------------------"
          docker system info || true
          echo "--------------------------"
          docker ps || true
          echo "--------------------------"
      - name: Install gopogh
        shell: bash
        run: |
          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-linux-amd64
          sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
      - name: Download Binaries
        uses: actions/download-artifact@v1
        with:
          name: minikube_binaries
      - name: Run Integration Test
        continue-on-error: true
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestAddons|TestCertOptions)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
          echo ::set-env name=STAT::${STAT}
      - uses: actions/upload-artifact@v1
        with:
          name: addons_certs_docker_ubuntu
          path: minikube_binaries/report
      - name: The End Result - addons_certs_docker_ubuntu
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          echo "----------------${numFail} Failures----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*** $numPass Passed ***"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 6 ];then echo "*** Failed to pass at least 6 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
  multinode_pause_docker_ubuntu:
    runs-on: ubuntu-18.04
    env:
      TIME_ELAPSED: time
      JOB_NAME: "multinode_pause_docker_ubuntu"
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
    needs: [build_minikube]
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true
      - name: Docker Info
        shell: bash
        run: |
          echo "--------------------------"
          docker version || true
          echo "--------------------------"
          docker info || true
          echo "--------------------------"
          docker system df || true
          echo "--------------------------"
          docker system info || true
          echo "--------------------------"
          docker ps || true
          echo "--------------------------"
      - name: Install gopogh
        shell: bash
        run: |
          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-linux-amd64
          sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
      - name: Download Binaries
        uses: actions/download-artifact@v1
        with:
          name: minikube_binaries
      - name: Run Integration Test
        continue-on-error: true
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPause|TestMultiNode)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
          echo ::set-env name=STAT::${STAT}
      - uses: actions/upload-artifact@v1
        with:
          name: multinode_pause_docker_ubuntu
          path: minikube_binaries/report
      - name: The End Result - multinode_pause_docker_ubuntu
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          echo "----------------${numFail} Failures----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*** $numPass Passed ***"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 6 ];then echo "*** Failed to pass at least 6 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
  preload_docker_flags_docker_ubuntu:
    runs-on: ubuntu-18.04
    env:
      TIME_ELAPSED: time
      JOB_NAME: "preload_docker_flags_docker_ubuntu"
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
    needs: [build_minikube]
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true
      - name: Docker Info
        shell: bash
        run: |
          echo "--------------------------"
          docker version || true
          echo "--------------------------"
          docker info || true
          echo "--------------------------"
          docker system df || true
          echo "--------------------------"
          docker system info || true
          echo "--------------------------"
          docker ps || true
          echo "--------------------------"
      - name: Install gopogh
        shell: bash
        run: |
          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-linux-amd64
          sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
      - name: Download Binaries
        uses: actions/download-artifact@v1
        with:
          name: minikube_binaries
      - name: Run Integration Test
        continue-on-error: true
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
          sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome ./e2e-linux-amd64 -minikube-start-args=--driver=docker  -test.run "(TestPreload|TestDockerFlags)" -test.timeout=10m -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
          echo ::set-env name=STAT::${STAT}
      - uses: actions/upload-artifact@v1
        with:
          name: preload_docker_flags_docker_ubuntu
          path: minikube_binaries/report
      - name: The End Result - preload_docker_flags_docker_ubuntu
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          echo "----------------${numFail} Failures----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*** $numPass Passed ***"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 2 ];then echo "*** Failed to pass at least 27 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi
  functional_baremetal_ubuntu18_04:
    needs: [build_minikube]
    env:
      TIME_ELAPSED: time
      JOB_NAME: "functional_baremetal_ubuntu18_04"
      GOPOGH_RESULT: ""
      SHELL: "/bin/bash" # To prevent https://github.com/kubernetes/minikube/issues/6643
    runs-on: ubuntu-18.04
    steps:
      - name: Install kubectl
        shell: bash
        run: |
          curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.18.0/bin/linux/amd64/kubectl
          sudo install kubectl /usr/local/bin/kubectl
          kubectl version --client=true
      # conntrack is required for kubernetes 1.18 and higher
      # socat is required for kubectl port forward which is used in some tests such as validateHelmTillerAddon
      - name: Install tools for none
        shell: bash
        run: |
          sudo apt-get update -qq
          sudo apt-get -qq -y install conntrack
          sudo apt-get -qq -y install socat
          VERSION="v1.17.0"
          curl -L https://github.com/kubernetes-sigs/cri-tools/releases/download/$VERSION/crictl-${VERSION}-linux-amd64.tar.gz --output crictl-${VERSION}-linux-amd64.tar.gz
          sudo tar zxvf crictl-$VERSION-linux-amd64.tar.gz -C /usr/local/bin
      - name: Install gopogh
        shell: bash
        run: |
          curl -LO https://github.com/medyagh/gopogh/releases/download/v0.1.23/gopogh-linux-amd64
          sudo install gopogh-linux-amd64 /usr/local/bin/gopogh
      - name: Download Binaries
        uses: actions/download-artifact@v1
        with:
          name: minikube_binaries
      - name: Run Integration Test
        continue-on-error: true
        # bash {0} to allow test to continue to next step. in case of
        shell: bash {0}
        run: |
          cd minikube_binaries
          mkdir -p report
          mkdir -p testhome
          chmod a+x e2e-*
          chmod a+x minikube-*
          START_TIME=$(date -u +%s)
          KUBECONFIG=$(pwd)/testhome/kubeconfig MINIKUBE_HOME=$(pwd)/testhome sudo -E ./e2e-linux-amd64 -minikube-start-args=--driver=none -test.timeout=35m -test.run TestFunctional -test.v -timeout-multiplier=1.5 -binary=./minikube-linux-amd64 2>&1 | tee ./report/testout.txt
          END_TIME=$(date -u +%s)
          TIME_ELAPSED=$(($END_TIME-$START_TIME))
          min=$((${TIME_ELAPSED}/60))
          sec=$((${TIME_ELAPSED}%60))
          TIME_ELAPSED="${min} min $sec seconds "
          echo ::set-env name=TIME_ELAPSED::${TIME_ELAPSED}
      - name: Generate HTML Report
        shell: bash
        run: |
          cd minikube_binaries
          export PATH=${PATH}:`go env GOPATH`/bin
          go tool test2json -t < ./report/testout.txt > ./report/testout.json || true
          STAT=$(gopogh -in ./report/testout.json -out ./report/testout.html -name "${JOB_NAME} ${GITHUB_REF}" -repo "${GITHUB_REPOSITORY}"  -details "${GITHUB_SHA}")  || true
          echo status: ${STAT}
          FailNum=$(echo $STAT | jq '.NumberOfFail')
          TestsNum=$(echo $STAT | jq '.NumberOfTests')
          GOPOGH_RESULT="${JOB_NAME} : completed with ${FailNum} / ${TestsNum} failures in ${TIME_ELAPSED}"
          echo ::set-env name=GOPOGH_RESULT::${GOPOGH_RESULT}
          echo ::set-env name=STAT::${STAT}
      - uses: actions/upload-artifact@v1
        with:
          name: none_ubuntu18_04
          path: minikube_binaries/report
      - name: The End Result - None on Ubuntu 18:04
        shell: bash
        run: |
          echo ${GOPOGH_RESULT}
          numFail=$(echo $STAT | jq '.NumberOfFail')
          echo "----------------${numFail} Failures----------------------------"
          echo $STAT | jq '.FailedTests' || true
          echo "-------------------------------------------------------"
          numPass=$(echo $STAT | jq '.NumberOfPass')
          echo "*** $numPass Passed ***"
          if [ "$numFail" -gt 0 ];then echo "*** $numFail Failed ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** 0 Passed! ***";exit 2;fi
          if [ "$numPass" -lt 27 ];then echo "*** Failed to pass at least 27 ! ***";exit 2;fi
          if [ "$numPass" -eq 0 ];then echo "*** Passed! ***";exit 0;fi

  # After all integration tests finished
  # collect all the reports and upload them
  upload_all_reports:
    if: always()
    needs:
      [
        functional_docker_ubuntu,
        addons_certs_docker_ubuntu,
        multinode_pause_docker_ubuntu,
        preload_docker_flags_docker_ubuntu,
        functional_baremetal_ubuntu18_04,
        functional_docker_windows,
        functional_hyperv_windows,
      ]
    runs-on: ubuntu-18.04
    steps:
      - name: download all reports
        uses: actions/download-artifact@v2-preview
      - name: upload all reports
        shell: bash {0}
        continue-on-error: true
        run: |
          mkdir -p all_reports
          ls -lah
          cp -r ./functional_docker_ubuntu ./all_reports/
          cp -r ./addons_certs_docker_ubuntu ./all_reports/
          cp -r ./multinode_pause_docker_ubuntu ./all_reports/
          cp -r ./preload_docker_flags_docker_ubuntu ./all_reports/
          cp -r ./functional_baremetal_ubuntu18_04 ./all_reports/
          cp -r ./functional_docker_windows ./all_reports/
          cp -r ./functional_hyperv_windows ./all_reports/
      - uses: actions/upload-artifact@v1
        with:
          name: all_reports
          path: all_reports