[CI] Make E2E tests run in multiple JDK versions (#3657)

* [CI] Make E2E tests run in multiple JDK versions

* Polish

* Use new e2e-container version number

* Install socat in reboot case

* Fix reboot case

* Polish

* Reorganize jobs

Jenkinsfile-E2E

@@ -63,15 +63,33 @@ pipeline {
             parallel {
                 stage('Group 1') {
                     stages {
-                        stage('Single Node Tests') {
+                        stage('Single Node Tests(JDK8)') {
                             steps {
-                                sh 'bash -x test/e2e/run.sh e2e-single-service'
+                                sh 'E2E_VERSION=jdk8-1.3 bash -x test/e2e/run.sh e2e-single-service'
                             }
                         }
 
-                        stage('Agent Reboot Tests') {
+                        stage('Single Node Tests(JDK9)') {
                             steps {
-                                sh 'bash -x test/e2e/run.sh e2e-agent-reboot'
+                                sh 'E2E_VERSION=jdk9-1.3 bash -x test/e2e/run.sh e2e-single-service'
+                            }
+                        }
+
+                        stage('Single Node Tests(JDK11)') {
+                            steps {
+                                sh 'E2E_VERSION=jdk11-1.3 bash -x test/e2e/run.sh e2e-single-service'
+                            }
+                        }
+
+                        stage('Single Node Tests(JDK12)') {
+                            steps {
+                                sh 'E2E_VERSION=jdk12-1.3 bash -x test/e2e/run.sh e2e-single-service'
+                            }
+                        }
+
+                        stage('Agent Reboot Tests(JDK8)') {
+                            steps {
+                                sh 'E2E_VERSION=jdk8-1.3 bash -x test/e2e/run.sh e2e-agent-reboot'
                             }
                         }
                     }
@@ -79,15 +97,15 @@ pipeline {
 
                 stage('Group 2') {
                     stages {
-                        stage('Cluster Tests (ES6/ZK)') {
+                        stage('Cluster Tests (ES6/ZK/JDK8)') {
                             steps {
-                                sh 'bash -x test/e2e/run.sh e2e-cluster/test-runner'
+                                sh 'E2E_VERSION=jdk8-1.3 bash -x test/e2e/run.sh e2e-cluster/test-runner'
                             }
                         }
 
-                        stage('TTL ES Tests') {
+                        stage('TTL ES Tests(JDK8)') {
                             steps {
-                                sh 'bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es'
+                                sh 'E2E_VERSION=jdk8-1.3 bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es'
                             }
                         }
                     }

apm-webapp/pom.xml

@@ -70,11 +70,6 @@
             <artifactId>spring-boot-configuration-processor</artifactId>
             <version>${spring.boot.version}</version>
         </dependency>
-        <dependency>
-            <groupId>org.springframework.boot</groupId>
-            <artifactId>spring-boot-devtools</artifactId>
-            <version>${spring.boot.version}</version>
-        </dependency>
         <dependency>
             <groupId>com.google.code.gson</groupId>
             <artifactId>gson</artifactId>
@@ -100,7 +95,6 @@
             <groupId>ch.qos.logback</groupId>
             <artifactId>logback-classic</artifactId>
             <version>${logback-classic.version}</version>
-            <scope>test</scope>
         </dependency>
     </dependencies>
 

test/e2e/e2e-agent-reboot/src/docker/rc.d/rc0-prepare.sh

+#!/usr/bin/env bash
 # Licensed to the SkyAPM under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,19 +15,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/usr/bin/env bash
-
 # in order to make it easier to restart the OAP (by executing the restart script) from outside (container),
 # we'll expose a tcp port and whenever we receive a message on that port, we'll restart the OAP server,
 # socat will help on this to execute the script when receiving a message on that port
 
-apk update && apk add socat
+apt-get update && apt-get -y install socat
 
 # socat will execute the command in a new shell, thus won't catch the original functions' declarations
 # so we'll put the restart command in a script file
 
 echo '
-    ps -ef | grep -v grep | grep oap.logDir | awk '"'"'{print $1}'"'"' | xargs --no-run-if-empty kill -9
+    ps -ef | grep -v grep | grep oap.logDir | awk '"'"'{print $2}'"'"' | xargs --no-run-if-empty kill -9
     rm -rf /tmp/oap/trace_buffer1
     rm -rf /tmp/oap/mesh_buffer1
     echo "restarting OAP server..." \

test/e2e/e2e-agent-reboot/src/docker/rc.d/rc1-startup.sh

+#!/usr/bin/env bash
 # Licensed to the SkyAPM under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,8 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/usr/bin/env bash
-
 # set up the tcp server to listen for the restart command
 
 socat -u tcp-l:9091,fork system:'bash restart_oap' &

test/e2e/e2e-cluster/test-runner/src/docker/rc.d/rc0-prepare.sh

+#!/usr/bin/env bash
 # Licensed to the SkyAPM under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,14 +15,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/usr/bin/env bash
+apt-get update && apt-get install -y gawk
 
 if test "${MODE}" = "cluster"; then
     original_wd=$(pwd)
 
     # substitute application.yml to be capable of cluster mode
     cd ${SW_HOME}/config \
-        && awk -f /clusterize.awk application.yml > clusterized_app.yml \
+        && gawk -f /clusterize.awk application.yml > clusterized_app.yml \
         && mv clusterized_app.yml application.yml \
         && echo '
 gateways:
@@ -35,7 +36,7 @@ gateways:
         && mv log4j2debuggable.xml log4j2.xml
 
     cd ${SW_HOME}/webapp \
-        && awk '/^\s+listOfServers:/ {gsub("listOfServers:.*", "listOfServers: 127.0.0.1:12800,127.0.0.1:12801", $0)} {print}' webapp.yml > clusterized_webapp.yml \
+        && gawk '/^\s+listOfServers:/ {gsub("listOfServers:.*", "listOfServers: 127.0.0.1:12800,127.0.0.1:12801", $0)} {print}' webapp.yml > clusterized_webapp.yml \
         && mv clusterized_webapp.yml webapp.yml
 
     cd ${original_wd}

test/e2e/e2e-cluster/test-runner/src/docker/rc.d/rc1-startup.sh

+#!/usr/bin/env bash
 # Licensed to the SkyAPM under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,8 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/usr/bin/env bash
-
 echo 'starting gateway service...' \
     && java -jar /home/gateway-1.0.0.jar 2>&1 > /tmp/gateway.log &
 

test/e2e/e2e-single-service/pom.xml

@@ -86,13 +86,14 @@
                                     <INSTRUMENTED_SERVICE>${project.build.finalName}.jar</INSTRUMENTED_SERVICE>
                                 </env>
                                 <ports>
-                                    <port>+webapp.host:webapp.port:8080</port>
+                                    <port>+webapp.host:webapp.port:8081</port>
                                     <port>+client.host:client.port:9090</port>
                                 </ports>
                                 <volumes>
                                     <bind>
-                                        <volume>${sw.home}:/skywalking</volume>
+                                        <volume>${sw.home}:/sw</volume>
                                         <volume>${project.build.directory}:/home</volume>
+                                        <volume>${project.basedir}/src/docker/rc.d:/rc.d:ro</volume>
                                     </bind>
                                 </volumes>
                                 <wait>

test/e2e/e2e-single-service/src/docker/rc.d/rc1-startup.sh

+#!/usr/bin/env bash
+# Licensed to the SkyAPM under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+echo 'starting OAP server...' && start_oap 'init'
+
+echo 'starting Web app...' && start_webapp '0.0.0.0' 8081
+
+echo 'starting instrumented services...' && start_instrumented_services
+
+check_tcp 127.0.0.1 \
+          9090 \
+          60 \
+          10 \
+          "waiting for the instrumented service to be ready"
+
+if [[ $? -ne 0 ]]; then
+    echo "instrumented service 0 failed to start in 30 * 10 seconds: "
+    cat ${SERVICE_LOG}/*
+    exit 1
+fi
+
+echo "SkyWalking e2e container is ready for tests"
+
+tail -f ${OAP_LOG_DIR}/* \
+        ${WEBAPP_LOG_DIR}/* \
+        ${SERVICE_LOG}/*

test/e2e/e2e-single-service/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java

@@ -101,6 +101,7 @@ public class SampleVerificationITCase {
                     user,
                     String.class
                 );
+                LOGGER.info("responseEntity: {}", responseEntity);
                 assertThat(responseEntity.getStatusCode()).isEqualTo(HttpStatus.OK);
                 final List<Trace> traces = queryClient.traces(
                     new TracesQuery()
@@ -150,6 +151,7 @@ public class SampleVerificationITCase {
                 .start(minutesAgo.minusDays(1))
                 .end(now)
         );
+        LOGGER.info("topoData: {}", topoData);
 
         InputStream expectedInputStream =
             new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.topo.yml").getInputStream();
@@ -166,6 +168,7 @@ public class SampleVerificationITCase {
                 .start(minutesAgo)
                 .end(now)
         );
+        LOGGER.info("services: {}", services);
 
         InputStream expectedInputStream =
             new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.services.yml").getInputStream();
@@ -197,6 +200,7 @@ public class SampleVerificationITCase {
                 .start(minutesAgo)
                 .end(now)
         );
+        LOGGER.info("instances: {}", instances);
         expectedInputStream =
             new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.instances.yml").getInputStream();
         final InstancesMatcher instancesMatcher = new Yaml().loadAs(expectedInputStream, InstancesMatcher.class);
@@ -209,6 +213,7 @@ public class SampleVerificationITCase {
         Endpoints instances = queryClient.endpoints(
             new EndpointQuery().serviceId(service.getKey())
         );
+        LOGGER.info("instances: {}", instances);
         InputStream expectedInputStream =
             new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.endpoints.yml").getInputStream();
         final EndpointsMatcher endpointsMatcher = new Yaml().loadAs(expectedInputStream, EndpointsMatcher.class);
@@ -226,6 +231,7 @@ public class SampleVerificationITCase {
                         .metricsName(metricsName)
                         .id(instance.getKey())
                 );
+                LOGGER.info("instanceMetrics: {}", instanceMetrics);
                 AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
                 MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
                 greaterThanZero.setValue("gt 0");
@@ -249,6 +255,7 @@ public class SampleVerificationITCase {
                         .metricsName(metricName)
                         .id(endpoint.getKey())
                 );
+                LOGGER.info("metrics: {}", metrics);
                 AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
                 MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
                 greaterThanZero.setValue("gt 0");
@@ -268,6 +275,7 @@ public class SampleVerificationITCase {
                     .metricsName(metricName)
                     .id(service.getKey())
             );
+            LOGGER.info("serviceMetrics: {}", serviceMetrics);
             AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
             MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
             greaterThanZero.setValue("gt 0");
@@ -286,6 +294,7 @@ public class SampleVerificationITCase {
                 .end(now)
                 .orderByDuration()
         );
+        LOGGER.info("traces: {}", traces);
 
         InputStream expectedInputStream =
             new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.traces.yml").getInputStream();

test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc0-prepare.sh

+#!/usr/bin/env bash
 # Licensed to the SkyAPM under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,13 +15,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/usr/bin/env bash
+apt-get update && apt-get install -y gawk
 
 original_wd=$(pwd)
 
 # substitute application.yml to be capable of es mode
 cd ${SW_HOME}/config \
-    && awk -f /es_storage.awk application.yml > es_storage_app.yml \
+    && gawk -f /es_storage.awk application.yml > es_storage_app.yml \
     && mv es_storage_app.yml application.yml \
     && sed '/<Loggers>/a<logger name="org.apache.skywalking.oap.server.storage" level="DEBUG"/>' log4j2.xml > log4j2debuggable.xml \
     && mv log4j2debuggable.xml log4j2.xml

test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc1-startup.sh

+#!/usr/bin/env bash
 # Licensed to the SkyAPM under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -14,8 +15,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-#!/usr/bin/env bash
-
 echo 'starting OAP server...' \
     && SW_STORAGE_ES_BULK_ACTIONS=1 \
     SW_CORE_DATA_KEEPER_EXECUTE_PERIOD=1 \

test/e2e/pom.xml

@@ -66,7 +66,7 @@
          -->
         <build.id>local</build.id>
 
-        <e2e.container.version>1.0.0</e2e.container.version>
+        <e2e.container.version>jdk8-1.0</e2e.container.version>
 
         <maven-failsafe-plugin.version>2.22.0</maven-failsafe-plugin.version>
         <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version>

test/e2e/run.sh

@@ -20,36 +20,31 @@ export MAVEN_OPTS='-Dmaven.repo.local=.m2/repository -XX:+TieredCompilation -XX:
 
 base_dir=$(pwd)
 build=0
-fast_fail=0
 cases=()
 
 # Parse the arguments
 # --build-dist: build the distribution package ignoring the existance of `dist` folder, useful when running e2e locally
-# --fast-fail: when testing multiple cases, skip following cases when a previous one failed
 
-while [ $# -gt 0 ]; do
+while [[ $# -gt 0 ]]; do
   case "$1" in
     --build)
       build=1
       ;;
-    --fast-fail)
-      fast_fail=1
-      ;;
     *)
       cases+=($1)
   esac
   shift
 done
 
-[ ! -f "$base_dir/mvnw" ] \
+[[ ! -f "${base_dir}/mvnw" ]] \
   && echo 'Please run run.sh in the root directory of SkyWalking' \
   && exit 1
 
-[ ${#cases[@]} -le 0 ] \
+[[ ${#cases[@]} -le 0 ]] \
   && echo 'Usage: sh test/e2e/run.sh [--build-dist] [--fast-fail] <case1 maven module>[<case2 maven module>...<caseN maven module>]' \
   && exit 1
 
-[ $build -eq 1 ] \
+[[ ${build} -eq 1 ]] \
   && echo 'Building distribution package...' \
   && ./mvnw -q -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -DskipTests clean install
 
@@ -59,20 +54,19 @@ for test_case in "${cases[@]}"
 do
   echo "Running case: $test_case"
 
-  [ -d "$base_dir/$test_case" ] && rm -rf "$base_dir/$test_case"
+  [[ -d "$base_dir/$test_case" ]] && rm -rf "$base_dir/$test_case"
 
   # Some of the tests will modify files in the distribution folder, e.g. cluster test will modify the application.yml
   # so we give each test a separate distribution folder here
   mkdir -p "$test_case" && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C "$test_case"
 
-  ./mvnw -Dbuild.id="${BUILD_ID:-local}" -Dsw.home="${base_dir}/$test_case/apache-skywalking-apm-bin" -f test/e2e/pom.xml -pl "$test_case" -am verify
+  ./mvnw -Dbuild.id="${BUILD_ID:-local}" -De2e.container.version="${E2E_VERSION}" -Dsw.home="${base_dir}/$test_case/apache-skywalking-apm-bin" -f test/e2e/pom.xml -pl "$test_case" -am verify
 
   status_code=$?
 
-  [ $status_code -ne 0 ] \
-    && [ $fast_fail -eq 1 ] \
-    && echo "Fast failing due to previous failure: $test_case, exit status code: $status_code" \
-    && exit $status_code
+  if [[ ${status_code} -ne 0 ]]; then
+    echo "Fast failing due to previous failure: ${test_case}, exit status code: ${status_code}" && exit ${status_code}
+  fi
 done
 
 exit 0