[CI] Make e2e more friendly for local test (#3606)

And group the test cases according to their consuming time

Jenkinsfile-E2E

@@ -50,12 +50,6 @@ pipeline {
                 // thus save unnecessary E2E builds(which is expensive)
                 sh './mvnw checkstyle:check apache-rat:check'
                 sh './mvnw -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -Dmaven.compiler.maxmem=3072 -DskipTests clean install'
-                // Some of the tests will modify files in the distribution folder, e.g. cluster test will modify the application.yml
-                // so we give each test a separate distribution folder here
-                sh 'mkdir -p dist-for-single-node-service && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-single-node-service'
-                sh 'mkdir -p dist-for-cluster && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-cluster'
-                sh 'mkdir -p dist-for-agent-reboot && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-agent-reboot'
-                sh 'mkdir -p dist-for-ttl-es && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-ttl-es'
             }
         }
 
@@ -67,27 +61,35 @@ pipeline {
 
         stage('Run End-to-End Tests') {
             parallel {
-                stage('Run Single Node Tests') {
-                    steps {
-                        sh './mvnw -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml -pl e2e-single-service -am verify'
-                    }
-                }
+                stage('Group 1') {
+                    stages {
+                        stage('Single Node Tests') {
+                            steps {
+                                sh 'bash -x test/e2e/run.sh e2e-single-service'
+                            }
+                        }
 
-                stage('Run Cluster Tests (ES/ZK)') {
-                    steps {
-                        sh './mvnw -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml -pl e2e-cluster/test-runner -am verify'
+                        stage('Agent Reboot Tests') {
+                            steps {
+                                sh 'bash -x test/e2e/run.sh e2e-agent-reboot'
+                            }
+                        }
                     }
                 }
 
-                stage('Run Agent Reboot Tests') {
-                    steps {
-                        sh './mvnw -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml -pl e2e-agent-reboot -am verify'
-                    }
-                }
+                stage('Group 2') {
+                    stages {
+                        stage('Cluster Tests (ES6/ZK)') {
+                            steps {
+                                sh 'bash -x test/e2e/run.sh e2e-cluster/test-runner'
+                            }
+                        }
 
-                stage('Run TTL Tests') {
-                    steps {
-                        sh './mvnw -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml -pl e2e-ttl/e2e-ttl-es -am verify'
+                        stage('TTL ES Tests') {
+                            steps {
+                                sh 'bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es'
+                            }
+                        }
                     }
                 }
             }

test/e2e/e2e-agent-reboot/pom.xml

@@ -93,15 +93,9 @@
                                 </ports>
                                 <volumes>
                                     <bind>
-                                        <volume>
-                                            ${project.basedir}/../../../dist-for-agent-reboot/apache-skywalking-apm-bin:/sw
-                                        </volume>
-                                        <volume>
-                                            ${project.build.directory}:/home
-                                        </volume>
-                                        <volume>
-                                            ${project.basedir}/src/docker/rc.d:/rc.d:ro
-                                        </volume>
+                                        <volume>${sw.home}:/sw</volume>
+                                        <volume>${project.build.directory}:/home</volume>
+                                        <volume>${project.basedir}/src/docker/rc.d:/rc.d:ro</volume>
                                     </bind>
                                 </volumes>
                                 <wait>

test/e2e/e2e-cluster/test-runner/pom.xml

@@ -159,9 +159,7 @@
                                 </links>
                                 <volumes>
                                     <bind>
-                                        <volume>
-                                            ../../../../dist-for-cluster/apache-skywalking-apm-bin:/sw
-                                        </volume>
+                                        <volume>${sw.home}:/sw</volume>
                                         <volume>
                                             ../${gateway.name}/target/${gateway.name}-${project.version}.jar:/home/${gateway.name}-${project.version}.jar
                                         </volume>

test/e2e/e2e-single-service/pom.xml

@@ -91,12 +91,8 @@
                                 </ports>
                                 <volumes>
                                     <bind>
-                                        <volume>
-                                            ${project.basedir}/../../../dist-for-single-node-service/apache-skywalking-apm-bin:/skywalking
-                                        </volume>
-                                        <volume>
-                                            ${project.build.directory}:/home
-                                        </volume>
+                                        <volume>${sw.home}:/skywalking</volume>
+                                        <volume>${project.build.directory}:/home</volume>
                                     </bind>
                                 </volumes>
                                 <wait>

test/e2e/e2e-ttl/e2e-ttl-es/pom.xml

@@ -114,15 +114,9 @@
                                 </links>
                                 <volumes>
                                     <bind>
-                                        <volume>
-                                            ../../../../${e2e.dist.directory}/apache-skywalking-apm-bin:/sw
-                                        </volume>
-                                        <volume>
-                                            ${project.basedir}/src/docker/rc.d:/rc.d:ro
-                                        </volume>
-                                        <volume>
-                                            ${project.basedir}/src/docker/es_storage.awk:/es_storage.awk
-                                        </volume>
+                                        <volume>${sw.home}:/sw</volume>
+                                        <volume>${project.basedir}/src/docker/rc.d:/rc.d:ro</volume>
+                                        <volume>${project.basedir}/src/docker/es_storage.awk:/es_storage.awk</volume>
                                     </bind>
                                 </volumes>
                                 <wait>

test/e2e/run.sh

+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:TieredStopAtLevel=1 -XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC -XX:-UseGCOverheadLimit -Xmx3g'
+
+base_dir=$(pwd)
+build=0
+fast_fail=0
+cases=()
+
+# Parse the arguments
+# --build-dist: build the distribution package ignoring the existance of `dist` folder, useful when running e2e locally
+# --fast-fail: when testing multiple cases, skip following cases when a previous one failed
+
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --build)
+      build=1
+      ;;
+    --fast-fail)
+      fast_fail=1
+      ;;
+    *)
+      cases+=($1)
+  esac
+  shift
+done
+
+[ ! -f "$base_dir/mvnw" ] \
+  && echo 'Please run run.sh in the root directory of SkyWalking' \
+  && exit 1
+
+[ ${#cases[@]} -le 0 ] \
+  && echo 'Usage: sh test/e2e/run.sh [--build-dist] [--fast-fail] <case1 maven module>[<case2 maven module>...<caseN maven module>]' \
+  && exit 1
+
+[ $build -eq 1 ] \
+  && echo 'Building distribution package...' \
+  && ./mvnw -q -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -DskipTests clean install
+
+echo "Running cases: $(IFS=$' '; echo "${cases[*]}")"
+
+for test_case in "${cases[@]}"
+do
+  echo "Running case: $test_case"
+
+  [ -d "$base_dir/$test_case" ] && rm -rf "$base_dir/$test_case"
+
+  # Some of the tests will modify files in the distribution folder, e.g. cluster test will modify the application.yml
+  # so we give each test a separate distribution folder here
+  mkdir -p "$test_case" && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C "$test_case"
+
+  ./mvnw -Dbuild.id="${BUILD_ID:-local}" -Dsw.home="${base_dir}/$test_case/apache-skywalking-apm-bin" -f test/e2e/pom.xml -pl "$test_case" -am verify
+
+  status_code=$?
+
+  [ $status_code -ne 0 ] \
+    && [ $fast_fail -eq 1 ] \
+    && echo "Fast failing due to previous failure: $test_case, exit status code: $status_code" \
+    && exit $status_code
+done
+
+exit 0