Add e2e cluster tests (#3016)

* Add e2e cluster tests

Jenkinsfile-E2E

@@ -42,8 +42,18 @@ pipeline {
         }
 
         stage('Run End-to-End Tests') {
-            steps {
-                sh './mvnw -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml clean verify'
+            parallel {
+                stage('Run Single Node Tests') {
+                    steps {
+                        sh './mvnw -DskipSurefire=false -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml -pl e2e-single-service -am verify'
+                    }
+                }
+
+                stage('Run Cluster Tests (ES/ZK)') {
+                    steps {
+                        sh './mvnw -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml -pl e2e-cluster/test-runner -am verify'
+                    }
+                }
             }
         }
     }
@@ -54,6 +64,7 @@ pipeline {
             // we need to clean up when there are containers started by the e2e tests
             sh 'docker ps'
             sh 'docker ps | grep -e "skywalking-e2e-container-${BUILD_ID}" | awk \'{print $1}\' | xargs --no-run-if-empty docker stop'
+            sh 'docker ps | grep -e "skywalking-e2e-container-${BUILD_ID}" | awk \'{print $1}\' | xargs --no-run-if-empty docker rm'
             deleteDir()
         }
     }

apm-dist/bin/oapServiceNoInit.sh

@@ -20,7 +20,7 @@ PRG="$0"
 PRGDIR=`dirname "$PRG"`
 [ -z "$OAP_HOME" ] && OAP_HOME=`cd "$PRGDIR/.." >/dev/null; pwd`
 
-OAP_LOG_DIR="${OAP_HOME}/logs"
+OAP_LOG_DIR=${OAP_LOG_DIR:-"${OAP_HOME}/logs"}
 JAVA_OPTS=" -Xms256M -Xmx512M"
 
 if [ ! -d "${OAP_HOME}/logs" ]; then

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/AbstractQuery.java

@@ -38,8 +38,8 @@ public abstract class AbstractQuery<T extends AbstractQuery<?>> {
             return start;
         }
         return "SECOND".equals(step())
-            ? LocalDateTime.now(ZoneOffset.UTC).minusMinutes(5).format(TIME_FORMATTER)
-            : LocalDateTime.now(ZoneOffset.UTC).minusMinutes(5).format(MINUTE_TIME_FORMATTER);
+            ? LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(TIME_FORMATTER)
+            : LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(MINUTE_TIME_FORMATTER);
     }
 
     public T start(String start) {
@@ -87,4 +87,14 @@ public abstract class AbstractQuery<T extends AbstractQuery<?>> {
         this.step = step;
         return (T) this;
     }
+
+    public T stepByMinute() {
+        this.step = "MINUTE";
+        return (T) this;
+    }
+
+    public T stepBySecond() {
+        this.step = "SECOND";
+        return (T) this;
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/SimpleQueryClient.java

@@ -57,6 +57,10 @@ public class SimpleQueryClient {
 
     private final String endpointUrl;
 
+    public SimpleQueryClient(String host, String port) {
+        this("http://" + host + ":" + port + "/graphql");
+    }
+
     public SimpleQueryClient(String endpointUrl) {
         this.endpointUrl = endpointUrl;
     }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/MetricsQuery.java

@@ -29,16 +29,35 @@ public class MetricsQuery extends AbstractQuery<MetricsQuery> {
     public static String SERVICE_P90 = "service_p90";
     public static String SERVICE_P75 = "service_p75";
     public static String SERVICE_P50 = "service_p50";
+    public static String[] ALL_SERVICE_METRICS = {
+        SERVICE_P99,
+        SERVICE_P95,
+        SERVICE_P90,
+        SERVICE_P75,
+        SERVICE_P50
+    };
 
     public static String ENDPOINT_P99 = "endpoint_p99";
     public static String ENDPOINT_P95 = "endpoint_p95";
     public static String ENDPOINT_P90 = "endpoint_p90";
     public static String ENDPOINT_P75 = "endpoint_p75";
     public static String ENDPOINT_P50 = "endpoint_p50";
+    public static String[] ALL_ENDPOINT_METRICS = {
+        ENDPOINT_P99,
+        ENDPOINT_P95,
+        ENDPOINT_P90,
+        ENDPOINT_P75,
+        ENDPOINT_P50
+    };
 
     public static String SERVICE_INSTANCE_RESP_TIME = "service_instance_resp_time";
     public static String SERVICE_INSTANCE_CPM = "service_instance_cpm";
     public static String SERVICE_INSTANCE_SLA = "service_instance_sla";
+    public static String[] ALL_INSTANCE_METRICS = {
+        SERVICE_INSTANCE_RESP_TIME,
+        SERVICE_INSTANCE_CPM,
+        SERVICE_INSTANCE_SLA
+    };
 
     private String id;
     private String metricsName;

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/service/ServiceMatcher.java

@@ -44,15 +44,15 @@ public class ServiceMatcher extends AbstractMatcher<Service> {
     }
 
     private void verifyKey(Service service) {
-        final String expected = this.getKey();
+        final String expected = getKey();
         final String actual = service.getKey();
 
         doVerify(expected, actual);
     }
 
     private void verifyLabel(Service service) {
-        final String expected = this.getLabel();
-        final String actual = String.valueOf(service.getLabel());
+        final String expected = getLabel();
+        final String actual = service.getLabel();
 
         doVerify(expected, actual);
     }
@@ -72,4 +72,12 @@ public class ServiceMatcher extends AbstractMatcher<Service> {
     public void setLabel(String label) {
         this.label = label;
     }
+
+    @Override
+    public String toString() {
+        return "ServiceMatcher{" +
+            "key='" + key + '\'' +
+            ", label='" + label + '\'' +
+            '}';
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/service/ServicesMatcher.java

@@ -22,6 +22,7 @@ import java.util.LinkedList;
 import java.util.List;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
 
 /**
  * @author kezhenxu94
@@ -44,10 +45,25 @@ public class ServicesMatcher {
     public void verify(final List<Service> services) {
         assertThat(services).hasSameSizeAs(this.getServices());
 
-        int size = this.getServices().size();
-
-        for (int i = 0; i < size; i++) {
-            this.getServices().get(i).verify(services.get(i));
+        for (int i = 0; i < getServices().size(); i++) {
+            boolean matched = false;
+            for (Service service : services) {
+                try {
+                    this.getServices().get(i).verify(service);
+                    matched = true;
+                } catch (Throwable ignored) {
+                }
+            }
+            if (!matched) {
+                fail("Expected: %s\nActual: %s", getServices(), services);
+            }
         }
     }
+
+    @Override
+    public String toString() {
+        return "ServicesMatcher{" +
+            "services=" + services +
+            '}';
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/Call.java

@@ -64,4 +64,14 @@ public class Call {
         this.target = target;
         return this;
     }
+
+    @Override
+    public String toString() {
+        return "Call{" +
+            "id='" + id + '\'' +
+            ", source='" + source + '\'' +
+            ", detectPoints=" + detectPoints +
+            ", target='" + target + '\'' +
+            '}';
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/CallMatcher.java

@@ -101,4 +101,14 @@ public class CallMatcher extends AbstractMatcher<Call> {
     public void setTarget(String target) {
         this.target = target;
     }
+
+    @Override
+    public String toString() {
+        return "CallMatcher{" +
+            "id='" + id + '\'' +
+            ", source='" + source + '\'' +
+            ", detectPoints=" + detectPoints +
+            ", target='" + target + '\'' +
+            '}';
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/Node.java

@@ -62,4 +62,14 @@ public class Node {
         isReal = real;
         return this;
     }
+
+    @Override
+    public String toString() {
+        return "Node{" +
+            "id='" + id + '\'' +
+            ", name='" + name + '\'' +
+            ", type='" + type + '\'' +
+            ", isReal='" + isReal + '\'' +
+            '}';
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/NodeMatcher.java

@@ -93,4 +93,14 @@ public class NodeMatcher extends AbstractMatcher<Node> {
     public void setIsReal(String isReal) {
         this.isReal = isReal;
     }
+
+    @Override
+    public String toString() {
+        return "NodeMatcher{" +
+            "id='" + id + '\'' +
+            ", name='" + name + '\'' +
+            ", type='" + type + '\'' +
+            ", isReal='" + isReal + '\'' +
+            '}';
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/topo/TopoMatcher.java

@@ -23,7 +23,7 @@ import org.apache.skywalking.e2e.verification.AbstractMatcher;
 import java.util.List;
 import java.util.Objects;
 
-import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
 
 /**
  * A simple matcher to verify the given {@code Service} is expected
@@ -47,22 +47,34 @@ public class TopoMatcher extends AbstractMatcher<TopoData> {
     }
 
     private void verifyNodes(TopoData topoData) {
-        assertThat(topoData.getNodes()).hasSameSizeAs(getNodes());
-
-        int size = getNodes().size();
-
-        for (int i = 0; i < size; i++) {
-            getNodes().get(i).verify(topoData.getNodes().get(i));
+        for (int i = 0; i < getNodes().size(); i++) {
+            boolean matched = false;
+            for (int j = 0; j < topoData.getNodes().size(); j++) {
+                try {
+                    getNodes().get(i).verify(topoData.getNodes().get(j));
+                    matched = true;
+                } catch (Throwable ignored) {
+                }
+            }
+            if (!matched) {
+                fail("Expected: %s\nActual: %s", getNodes(), topoData.getNodes());
+            }
         }
     }
 
     private void verifyCalls(TopoData topoData) {
-        assertThat(topoData.getCalls()).hasSameSizeAs(getCalls());
-
-        int size = getCalls().size();
-
-        for (int i = 0; i < size; i++) {
-            getCalls().get(i).verify(topoData.getCalls().get(i));
+        for (int i = 0; i < getCalls().size(); i++) {
+            boolean matched = false;
+            for (int j = 0; j < topoData.getCalls().size(); j++) {
+                try {
+                    getCalls().get(i).verify(topoData.getCalls().get(j));
+                    matched = true;
+                } catch (Throwable ignored) {
+                }
+            }
+            if (!matched) {
+                fail("Expected: %s\nActual: %s", getCalls(), topoData.getCalls());
+            }
         }
     }
 
@@ -81,4 +93,12 @@ public class TopoMatcher extends AbstractMatcher<TopoData> {
     public void setCalls(List<CallMatcher> calls) {
         this.calls = calls;
     }
+
+    @Override
+    public String toString() {
+        return "TopoMatcher{" +
+            "nodes=" + nodes +
+            ", calls=" + calls +
+            '}';
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/trace/Span.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.trace;
+
+import lombok.Data;
+
+import java.util.List;
+
+/**
+ * @author kezhenxu94
+ */
+@Data
+public class Span {
+    private String traceId;
+    private String segmentId;
+    private int spanId;
+    private int parentSpanId;
+    private String serviceCode;
+    private long startTime;
+    private long endTime;
+    private String endpointName;
+    private String type;
+    private String peer;
+    private String component;
+    private boolean isError;
+    private String layer;
+    private List<String> tags;
+}

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/trace/SpanMatcher.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.trace;
+
+import com.google.common.base.Strings;
+import lombok.Data;
+import lombok.ToString;
+import org.apache.skywalking.e2e.verification.AbstractMatcher;
+
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * @author kezhenxu94
+ */
+@Data
+@ToString(callSuper = true)
+public class SpanMatcher extends AbstractMatcher<Span> {
+    private String traceId;
+    private String segmentId;
+    private String spanId;
+    private String parentSpanId;
+    private String serviceCode;
+    private String startTime;
+    private String endTime;
+    private String endpointName;
+    private String type;
+    private String peer;
+    private String component;
+    private String isError;
+    private String layer;
+    private List<String> tags;
+
+    @Override
+    public void verify(final Span span) {
+        if (Objects.nonNull(traceId)) {
+            String expected = this.getTraceId();
+            String actual = span.getTraceId();
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(segmentId)) {
+            String expected = this.getSegmentId();
+            String actual = span.getSegmentId();
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(spanId)) {
+            String expected = String.valueOf(this.getSpanId());
+            String actual = String.valueOf(span.getSpanId());
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(parentSpanId)) {
+            String expected = String.valueOf(this.getParentSpanId());
+            String actual = String.valueOf(span.getParentSpanId());
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(serviceCode)) {
+            String expected = this.getServiceCode();
+            String actual = span.getServiceCode();
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(startTime)) {
+            String expected = String.valueOf(this.getStartTime());
+            String actual = String.valueOf(span.getStartTime());
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(endTime)) {
+            String expected = String.valueOf(this.getEndTime());
+            String actual = String.valueOf(span.getEndTime());
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(endpointName)) {
+            String expected = this.getEndpointName();
+            String actual = span.getEndpointName();
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(type)) {
+            String expected = this.getType();
+            String actual = span.getType();
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(peer)) {
+            String expected = this.getPeer();
+            String actual = span.getPeer();
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(component)) {
+            String expected = this.getComponent();
+            String actual = span.getComponent();
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(isError)) {
+            String expected = Strings.nullToEmpty(String.valueOf(this.getIsError()));
+            String actual = Strings.nullToEmpty(String.valueOf(span.isError()));
+            doVerify(expected, actual);
+        }
+        if (Objects.nonNull(layer)) {
+            String expected = this.getLayer();
+            String actual = span.getLayer();
+            doVerify(expected, actual);
+        }
+
+    }
+
+}

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/trace/Tag.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.trace;
+
+/**
+ * @author kezhenxu94
+ */
+public class Tag {
+    private String key;
+    private String value;
+
+    public String getKey() {
+        return key;
+    }
+
+    public void setKey(final String key) {
+        this.key = key;
+    }
+
+    public String getValue() {
+        return value;
+    }
+
+    public void setValue(final String value) {
+        this.value = value;
+    }
+}

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/trace/Trace.java

@@ -31,10 +31,12 @@ public class Trace {
     private String start;
     private boolean isError;
     private final List<String> traceIds;
+    private final List<Span> spans;
 
     public Trace() {
         this.endpointNames = new ArrayList<>();
         this.traceIds = new ArrayList<>();
+        this.spans = new ArrayList<>();
     }
 
     public String getKey() {
@@ -81,6 +83,10 @@ public class Trace {
         return traceIds;
     }
 
+    public List<Span> getSpans() {
+        return spans;
+    }
+
     @Override
     public String toString() {
         return "Trace{" +

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/trace/TraceMatcher.java

@@ -39,6 +39,7 @@ public class TraceMatcher extends AbstractMatcher<Trace> {
     private String start;
     private String isError;
     private List<String> traceIds;
+    private List<SpanMatcher> spans;
 
     @Override
     public void verify(final Trace trace) {
@@ -65,6 +66,10 @@ public class TraceMatcher extends AbstractMatcher<Trace> {
         if (Objects.nonNull(getTraceIds())) {
             verifyTraceIds(trace);
         }
+
+        if (Objects.nonNull(getSpans())) {
+            verifySpans(trace);
+        }
     }
 
     private void verifyKey(Trace trace) {
@@ -121,6 +126,16 @@ public class TraceMatcher extends AbstractMatcher<Trace> {
         }
     }
 
+    private void verifySpans(Trace trace) {
+        assertThat(trace.getSpans()).hasSameSizeAs(getSpans());
+
+        int size = getSpans().size();
+
+        for (int i = 0; i < size; i++) {
+            getSpans().get(i).verify(trace.getSpans().get(i));
+        }
+    }
+
     public String getKey() {
         return key;
     }
@@ -168,4 +183,25 @@ public class TraceMatcher extends AbstractMatcher<Trace> {
     public List<String> getTraceIds() {
         return traceIds != null ? traceIds : new ArrayList<>();
     }
+
+    public List<SpanMatcher> getSpans() {
+        return spans;
+    }
+
+    public void setSpans(final List<SpanMatcher> spans) {
+        this.spans = spans;
+    }
+
+    @Override
+    public String toString() {
+        return "TraceMatcher{" +
+            "key='" + key + '\'' +
+            ", endpointNames=" + endpointNames +
+            ", duration='" + duration + '\'' +
+            ", start='" + start + '\'' +
+            ", isError='" + isError + '\'' +
+            ", traceIds=" + traceIds +
+            ", spans=" + spans +
+            '}';
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/trace/TracesData.java

@@ -24,26 +24,26 @@ import java.util.List;
  * @author kezhenxu94
  */
 public class TracesData {
-  public static class Traces {
-    private List<Trace> data;
+    public static class Traces {
+        private List<Trace> data;
 
-    public List<Trace> getData() {
-      return data;
-    }
+        public List<Trace> getData() {
+            return data;
+        }
 
-    public Traces setData(List<Trace> data) {
-      this.data = data;
-      return this;
+        public Traces setData(List<Trace> data) {
+            this.data = data;
+            return this;
+        }
     }
-  }
 
-  private Traces traces;
+    private Traces traces;
 
-  public Traces getTraces() {
-    return traces;
-  }
+    public Traces getTraces() {
+        return traces;
+    }
 
-  public void setTraces(final Traces traces) {
-    this.traces = traces;
-  }
+    public void setTraces(final Traces traces) {
+        this.traces = traces;
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/trace/TracesMatcher.java

@@ -22,6 +22,7 @@ import java.util.LinkedList;
 import java.util.List;
 
 import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.fail;
 
 /**
  * @author kezhenxu94
@@ -50,4 +51,32 @@ public class TracesMatcher {
             this.traces.get(i).verify(traces.get(i));
         }
     }
+
+    /**
+     * Verify the traces in a loose manner
+     *
+     * @param traces
+     */
+    public void verifyLoosely(final List<Trace> traces) {
+        for (int i = 0; i < getTraces().size(); i++) {
+            boolean matched = false;
+            for (int j = 0; j < traces.size(); j++) {
+                try {
+                    getTraces().get(i).verify(traces.get(j));
+                    matched = true;
+                } catch (Throwable ignored) {
+                }
+            }
+            if (!matched) {
+                fail("Expected: %s\n Actual: %s", getTraces(), traces);
+            }
+        }
+    }
+
+    @Override
+    public String toString() {
+        return "TracesMatcher{" +
+            "traces=" + traces +
+            '}';
+    }
 }

test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/trace/TracesQuery.java

@@ -85,6 +85,11 @@ public class TracesQuery extends AbstractQuery<TracesQuery> {
         return this;
     }
 
+    public TracesQuery orderByStartTime() {
+        this.queryOrder = "BY_START_TIME";
+        return this;
+    }
+
     public TracesQuery pageSize(int pageSize) {
         this.pageSize = String.valueOf(pageSize);
         return this;

test/e2e/e2e-cluster/consumer/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>e2e-cluster</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>1.0.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <packaging>jar</packaging>
+
+    <artifactId>consumer</artifactId>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <version>${spring.boot.version}</version>
+                <configuration>
+                    <executable>true</executable>
+                    <addResources>true</addResources>
+                    <excludeDevtools>true</excludeDevtools>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

test/e2e/e2e-cluster/consumer/src/main/java/org/apache/skywalking/e2e/cluster/Service1Application.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+
+/**
+ * @author kezhenxu94
+ */
+@SpringBootApplication
+public class Service1Application {
+    public static void main(String[] args) {
+        SpringApplication.run(Service1Application.class, args);
+    }
+}

test/e2e/e2e-cluster/consumer/src/main/java/org/apache/skywalking/e2e/cluster/TestController.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import org.springframework.http.ResponseEntity;
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.client.RestTemplate;
+
+/**
+ * @author kezhenxu94
+ */
+@RestController
+@RequestMapping("/e2e")
+public class TestController {
+    private final RestTemplate restTemplate = new RestTemplate();
+
+    @GetMapping("/health-check")
+    public String hello() {
+        return "healthy";
+    }
+
+    @PostMapping("/users")
+    public User createAuthor(@RequestBody final User user) throws InterruptedException {
+        Thread.sleep(1000L);
+        final ResponseEntity<User> response = restTemplate.postForEntity(
+            "http://localhost:9090/e2e/users", user, User.class
+        );
+        return response.getBody();
+    }
+}

test/e2e/e2e-cluster/consumer/src/main/java/org/apache/skywalking/e2e/cluster/User.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+/**
+ * @author kezhenxu94
+ */
+public class User {
+    public User() {
+    }
+
+    private Long id;
+
+    private String name;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(final Long id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(final String name) {
+        this.name = name;
+    }
+}

test/e2e/e2e-cluster/consumer/src/main/resources/application.yml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+server:
+  port: 9091
+
+spring:
+  main:
+    banner-mode: 'off'

test/e2e/e2e-cluster/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>apache-skywalking-e2e</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>1.0.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>e2e-cluster</artifactId>
+    <packaging>pom</packaging>
+
+    <modules>
+        <module>provider</module>
+        <module>consumer</module>
+        <module>test-runner</module>
+    </modules>
+</project>

test/e2e/e2e-cluster/provider/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>e2e-cluster</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>1.0.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+    <packaging>jar</packaging>
+
+    <artifactId>provider</artifactId>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-jpa</artifactId>
+            <version>${spring.boot.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>com.h2database</groupId>
+            <artifactId>h2</artifactId>
+            <version>${h2.version}</version>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.springframework.boot</groupId>
+                <artifactId>spring-boot-maven-plugin</artifactId>
+                <version>${spring.boot.version}</version>
+                <configuration>
+                    <executable>true</executable>
+                    <addResources>true</addResources>
+                    <excludeDevtools>true</excludeDevtools>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>repackage</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

test/e2e/e2e-cluster/provider/src/main/java/org/apache/skywalking/e2e/cluster/Service0Application.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
+
+/**
+ * @author kezhenxu94
+ */
+@EnableJpaRepositories
+@SpringBootApplication
+public class Service0Application {
+    public static void main(String[] args) {
+        SpringApplication.run(Service0Application.class, args);
+    }
+}

test/e2e/e2e-cluster/provider/src/main/java/org/apache/skywalking/e2e/cluster/TestController.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import org.springframework.web.bind.annotation.GetMapping;
+import org.springframework.web.bind.annotation.PostMapping;
+import org.springframework.web.bind.annotation.RequestBody;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RestController;
+
+/**
+ * @author kezhenxu94
+ */
+@RestController
+@RequestMapping("/e2e")
+public class TestController {
+    private final UserRepo userRepo;
+
+    public TestController(final UserRepo userRepo) {
+        this.userRepo = userRepo;
+    }
+
+    @GetMapping("/health-check")
+    public String hello() {
+        return "healthy";
+    }
+
+    @PostMapping("/users")
+    public User createAuthor(@RequestBody final User user) {
+        return userRepo.save(user);
+    }
+}

test/e2e/e2e-cluster/provider/src/main/java/org/apache/skywalking/e2e/cluster/User.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.GeneratedValue;
+import javax.persistence.Id;
+
+/**
+ * @author kezhenxu94
+ */
+@Entity
+public class User {
+    public User() {
+    }
+
+    @Id
+    @GeneratedValue
+    private Long id;
+
+    @Column
+    private String name;
+
+    public Long getId() {
+        return id;
+    }
+
+    public void setId(final Long id) {
+        this.id = id;
+    }
+
+    public String getName() {
+        return name;
+    }
+
+    public void setName(final String name) {
+        this.name = name;
+    }
+}

test/e2e/e2e-cluster/provider/src/main/java/org/apache/skywalking/e2e/cluster/UserRepo.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e.cluster;
+
+import org.springframework.data.jpa.repository.JpaRepository;
+
+/**
+ * @author kezhenxu94
+ */
+public interface UserRepo extends JpaRepository<User, Long> {
+}

test/e2e/e2e-cluster/provider/src/main/resources/application.yml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+server:
+  port: 9090
+
+spring:
+  main:
+    banner-mode: 'off'
+  datasource:
+    url: jdbc:h2:mem:testdb
+    driver-class-name: org.h2.Driver
+    data-username: sa
+    password: sa
+    platform: org.hibernate.dialect.H2Dialect
+  jpa:
+    generate-ddl: true
+    hibernate:
+      ddl-auto: create-drop
+    properties:
+      hibernate.format_sql: true
+    show-sql: true

test/e2e/e2e-cluster/test-runner/pom.xml

+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Licensed to the Apache Software Foundation (ASF) under one or more
+  ~ contributor license agreements.  See the NOTICE file distributed with
+  ~ this work for additional information regarding copyright ownership.
+  ~ The ASF licenses this file to You under the Apache License, Version 2.0
+  ~ (the "License"); you may not use this file except in compliance with
+  ~ the License.  You may obtain a copy of the License at
+  ~
+  ~     http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  ~
+  -->
+
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>e2e-cluster</artifactId>
+        <groupId>org.apache.skywalking</groupId>
+        <version>1.0.0</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <artifactId>test-runner</artifactId>
+
+    <dependencies>
+
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>e2e-base</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>provider</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+        <dependency>
+            <groupId>org.apache.skywalking</groupId>
+            <artifactId>consumer</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+
+    </dependencies>
+
+    <properties>
+        <service0.name>provider</service0.name>
+        <service1.name>consumer</service1.name>
+        <e2e.container.version>1.1</e2e.container.version>
+    </properties>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>io.fabric8</groupId>
+                <artifactId>docker-maven-plugin</artifactId>
+                <configuration>
+                    <containerNamePattern>%a-%t-%i</containerNamePattern>
+                    <imagePullPolicy>Always</imagePullPolicy>
+                    <images>
+                        <image>
+                            <name>elastic/elasticsearch:${elasticsearch.version}</name>
+                            <alias>skywalking-e2e-container-${build.id}-elasticsearch</alias>
+                            <run>
+                                <ports>
+                                    <port>es.port:9200</port>
+                                </ports>
+                                <wait>
+                                    <http>
+                                        <url>http://localhost:${es.port}</url>
+                                        <method>GET</method>
+                                        <status>200</status>
+                                    </http>
+                                    <time>120000</time>
+                                </wait>
+                                <env>
+                                    <discovery.type>single-node</discovery.type>
+                                </env>
+                            </run>
+                        </image>
+                        <image>
+                            <name>zookeeper:${zookeeper.image.version}</name>
+                            <alias>skywalking-e2e-container-${build.id}-zookeeper</alias>
+                            <run>
+                                <ports>
+                                    <port>zk.port:2181</port>
+                                </ports>
+                                <wait>
+                                    <log>binding to port</log>
+                                    <time>30000</time>
+                                </wait>
+                            </run>
+                        </image>
+                        <image>
+                            <name>skyapm/e2e-container:${e2e.container.version}</name>
+                            <alias>skywalking-e2e-container-${build.id}</alias>
+                            <run>
+                                <env>
+                                    <MODE>cluster</MODE>
+                                    <SW_STORAGE_ES_CLUSTER_NODES>
+                                        skywalking-e2e-container-${build.id}-elasticsearch:9200
+                                    </SW_STORAGE_ES_CLUSTER_NODES>
+                                    <SW_CLUSTER_ZK_HOST_PORT>
+                                        skywalking-e2e-container-${build.id}-zookeeper:2181
+                                    </SW_CLUSTER_ZK_HOST_PORT>
+
+                                    <INSTRUMENTED_SERVICE_1>
+                                        ${service0.name}-${project.version}.jar
+                                    </INSTRUMENTED_SERVICE_1>
+                                    <INSTRUMENTED_SERVICE_1_OPTS>
+                                        -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11800
+                                        -DSW_AGENT_NAME=${service0.name}
+                                    </INSTRUMENTED_SERVICE_1_OPTS>
+                                    <INSTRUMENTED_SERVICE_1_ARGS>
+                                        --server.port=9090
+                                    </INSTRUMENTED_SERVICE_1_ARGS>
+
+                                    <INSTRUMENTED_SERVICE_2>
+                                        ${service1.name}-${project.version}.jar
+                                    </INSTRUMENTED_SERVICE_2>
+                                    <INSTRUMENTED_SERVICE_2_OPTS>
+                                        -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801
+                                        -DSW_AGENT_NAME=${service1.name}
+                                    </INSTRUMENTED_SERVICE_2_OPTS>
+                                    <INSTRUMENTED_SERVICE_2_ARGS>
+                                        --server.port=9091
+                                    </INSTRUMENTED_SERVICE_2_ARGS>
+                                </env>
+                                <dependsOn>
+                                    <container>skywalking-e2e-container-${build.id}-elasticsearch</container>
+                                    <container>skywalking-e2e-container-${build.id}-zookeeper</container>
+                                </dependsOn>
+                                <ports>
+                                    <port>+webapp.host:webapp.port:8081</port>
+                                    <port>+service.host:service.port:9091</port>
+                                </ports>
+                                <links>
+                                    <link>skywalking-e2e-container-${build.id}-elasticsearch</link>
+                                    <link>skywalking-e2e-container-${build.id}-zookeeper</link>
+                                </links>
+                                <volumes>
+                                    <bind>
+                                        <volume>
+                                            ../../../../dist/apache-skywalking-apm-bin:/sw
+                                        </volume>
+                                        <volume>
+                                            ../${service0.name}/target/${service0.name}-${project.version}.jar:/home/${service0.name}-${project.version}.jar
+                                        </volume>
+                                        <volume>
+                                            ../${service1.name}/target/${service1.name}-${project.version}.jar:/home/${service1.name}-${project.version}.jar
+                                        </volume>
+                                        <volume>
+                                            ${project.basedir}/src/docker/rc.d:/rc.d:ro
+                                        </volume>
+                                        <volume>
+                                            ${project.basedir}/src/docker/clusterize.awk:/clusterize.awk
+                                        </volume>
+                                    </bind>
+                                </volumes>
+                                <wait>
+                                    <log>SkyWalking e2e container is ready for tests</log>
+                                    <time>2400000</time>
+                                </wait>
+                            </run>
+                        </image>
+                    </images>
+                </configuration>
+            </plugin>
+
+            <!-- set the system properties that can be used in test codes -->
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-failsafe-plugin</artifactId>
+                <configuration>
+                    <systemPropertyVariables>
+                        <sw.webapp.host>${webapp.host}</sw.webapp.host>
+                        <sw.webapp.port>${webapp.port}</sw.webapp.port>
+                        <service.host>${service.host}</service.host>
+                        <service.port>${service.port}</service.port>
+                    </systemPropertyVariables>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>verify</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+        </plugins>
+    </build>
+
+</project>

test/e2e/e2e-cluster/test-runner/src/docker/clusterize.awk

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/awk -f
+
+BEGIN {
+    in_cluster_section=0;
+    in_cluster_zk_section=0;
+
+    in_storage_section=0;
+    in_storage_es_section=0;
+    in_storage_h2_section=0;
+}
+
+{
+    if (in_cluster_section == 0) {
+        in_cluster_section=$0 ~ /^cluster:$/
+    } else {
+        in_cluster_section=$0 ~ /^(#|\s{2})/
+    }
+    if (in_storage_section == 0) {
+        in_storage_section=$0 ~ /^storage:$/
+    } else {
+        in_storage_section=$0 ~ /^(#|\s{2})/
+    }
+
+    if (in_cluster_section == 1) {
+        # in the cluster: section now
+        # disable standalone module
+        if ($0 ~ /^  standalone:$/) {
+            print "#" $0
+        } else {
+            if (in_cluster_zk_section == 0) {
+                in_cluster_zk_section=$0 ~ /^#?\s+zookeeper:$/
+            } else {
+                in_cluster_zk_section=$0 ~ /^(#\s{4}|\s{2})/
+            }
+            if (in_cluster_zk_section == 1) {
+                # in the cluster.zookeeper section now
+                # uncomment zk config
+                gsub("^#", "", $0)
+                print
+            } else {
+                print
+            }
+        }
+    } else if (in_storage_section == 1) {
+        # in the storage: section now
+        # disable h2 module
+        if (in_storage_es_section == 0) {
+            in_storage_es_section=$0 ~ /^#?\s+elasticsearch:$/
+        } else {
+            in_storage_es_section=$0 ~ /^#?\s{4}/
+        }
+        if (in_storage_h2_section == 0) {
+            in_storage_h2_section=$0 ~ /^#?\s+h2:$/
+        } else {
+            in_storage_h2_section=$0 ~ /^#?\s{4}/
+        }
+        if (in_storage_es_section == 1) {
+            # in the storage.elasticsearch section now
+            # uncomment es config
+            gsub("^#", "", $0)
+            print
+        } else if (in_storage_h2_section == 1) {
+            # comment out h2 config
+            if ($0 !~ /^#/) {
+                print "#" $0
+            } else {
+                print
+            }
+        } else {
+            print
+        }
+    } else {
+        print
+    }
+}
+

test/e2e/e2e-cluster/test-runner/src/docker/rc.d/rc0-prepare.sh

+# Licensed to the SkyAPM under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+if test "${MODE}" = "cluster"; then
+    original_wd=$(pwd)
+
+    # substitute application.yml to be capable of cluster mode
+    cd ${SW_HOME}/config \
+        && awk -f /clusterize.awk application.yml > clusterized_app.yml \
+        && mv clusterized_app.yml application.yml
+
+    cd ${SW_HOME}/webapp \
+        && awk '/^\s+listOfServers/ {gsub("127.0.0.1:12800", "127.0.0.1:12800,127.0.0.1:12801", $0)} {print}' webapp.yml > clusterized_webapp.yml \
+        && mv clusterized_webapp.yml webapp.yml
+
+    cd ${original_wd}
+fi

test/e2e/e2e-cluster/test-runner/src/docker/rc.d/rc1-startup.sh

+# Licensed to the SkyAPM under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+#!/usr/bin/env bash
+
+echo 'starting OAP server...' \
+    && SW_STORAGE_ES_BULK_ACTIONS=1 \
+    && SW_STORAGE_ES_FLUSH_INTERVAL=1 \
+    && start_oap 'init'
+
+echo 'starting Web app...' \
+    && start_webapp '0.0.0.0' 8081
+
+if test "${MODE}" = "cluster"; then
+    # start another OAP server in a different port
+    echo 'starting OAP server...' \
+        && SW_CORE_GRPC_PORT=11801 \
+        && SW_CORE_REST_PORT=12801 \
+        && SW_STORAGE_ES_BULK_ACTIONS=1 \
+        && SW_STORAGE_ES_FLUSH_INTERVAL=1 \
+        && start_oap 'no-init'
+
+    # start another WebApp server in a different port
+    echo 'starting Web app...' \
+        && start_webapp '0.0.0.0' 8082
+fi
+
+echo 'starting instrumented services...' && start_instrumented_services
+
+check_tcp 127.0.0.1 \
+          9090 \
+          60 \
+          10 \
+          "waiting for the instrumented service 0 to be ready"
+
+if [[ $? -ne 0 ]]; then
+    echo "instrumented service 0 failed to start in 30 * 10 seconds: "
+    cat ${SERVICE_LOG}/*
+    exit 1
+fi
+
+check_tcp 127.0.0.1 \
+          9091 \
+          60 \
+          10 \
+          "waiting for the instrumented service 1 to be ready"
+
+if [[ $? -ne 0 ]]; then
+    echo "instrumented service 1 failed to start in 24 * 10 seconds: "
+    cat ${SERVICE_LOG}/*
+    exit 1
+fi
+
+echo "SkyWalking e2e container is ready for tests"
+
+tail -f ${OAP_LOG_DIR}/* \
+        ${WEBAPP_LOG_DIR}/* \
+        ${SERVICE_LOG}/* \
+        ${ES_HOME}/logs/elasticsearch.log \
+        ${ES_HOME}/logs/stdout.log

test/e2e/e2e-cluster/test-runner/src/test/java/org/apache/skywalking/e2e/ClusterVerificationITCase.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.e2e;
+
+import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher;
+import org.apache.skywalking.e2e.metrics.Metrics;
+import org.apache.skywalking.e2e.metrics.MetricsQuery;
+import org.apache.skywalking.e2e.metrics.MetricsValueMatcher;
+import org.apache.skywalking.e2e.service.Service;
+import org.apache.skywalking.e2e.service.ServicesMatcher;
+import org.apache.skywalking.e2e.service.ServicesQuery;
+import org.apache.skywalking.e2e.service.endpoint.Endpoint;
+import org.apache.skywalking.e2e.service.endpoint.EndpointQuery;
+import org.apache.skywalking.e2e.service.endpoint.Endpoints;
+import org.apache.skywalking.e2e.service.endpoint.EndpointsMatcher;
+import org.apache.skywalking.e2e.service.instance.Instance;
+import org.apache.skywalking.e2e.service.instance.Instances;
+import org.apache.skywalking.e2e.service.instance.InstancesMatcher;
+import org.apache.skywalking.e2e.service.instance.InstancesQuery;
+import org.apache.skywalking.e2e.topo.TopoData;
+import org.apache.skywalking.e2e.topo.TopoMatcher;
+import org.apache.skywalking.e2e.topo.TopoQuery;
+import org.apache.skywalking.e2e.trace.Trace;
+import org.apache.skywalking.e2e.trace.TracesMatcher;
+import org.apache.skywalking.e2e.trace.TracesQuery;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.core.io.ClassPathResource;
+import org.springframework.http.HttpStatus;
+import org.springframework.http.ResponseEntity;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.web.client.RestTemplate;
+import org.yaml.snakeyaml.Yaml;
+
+import java.io.InputStream;
+import java.time.LocalDateTime;
+import java.time.ZoneOffset;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS;
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS;
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_METRICS;
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author kezhenxu94
+ */
+@RunWith(SpringJUnit4ClassRunner.class)
+public class ClusterVerificationITCase {
+    private static final Logger LOGGER = LoggerFactory.getLogger(ClusterVerificationITCase.class);
+
+    private final RestTemplate restTemplate = new RestTemplate();
+    private final Yaml yaml = new Yaml();
+
+    private SimpleQueryClient queryClient;
+    private String instrumentedServiceUrl;
+    private long retryInterval = TimeUnit.SECONDS.toMillis(30);
+
+    @Before
+    public void setUp() {
+        final String swWebappHost = System.getProperty("sw.webapp.host", "127.0.0.1");
+        final String swWebappPort = System.getProperty("sw.webapp.port", "32791");
+        final String instrumentedServiceHost = System.getProperty("service.host", "127.0.0.1");
+        final String instrumentedServicePort = System.getProperty("service.port", "32790");
+        queryClient = new SimpleQueryClient(swWebappHost, swWebappPort);
+        instrumentedServiceUrl = "http://" + instrumentedServiceHost + ":" + instrumentedServicePort;
+    }
+
+    @Test(timeout = 1200000)
+    @DirtiesContext
+    public void verify() throws Exception {
+        LocalDateTime startTime = LocalDateTime.now(ZoneOffset.UTC);
+
+        final Map<String, String> user = new HashMap<>();
+        user.put("name", "SkyWalking");
+        List<Service> services = Collections.emptyList();
+        while (services.size() < 2) {
+            try {
+                restTemplate.postForEntity(
+                    instrumentedServiceUrl + "/e2e/users",
+                    user,
+                    String.class
+                );
+                services = queryClient.services(
+                    new ServicesQuery()
+                        .start(startTime)
+                        .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                );
+            } catch (Throwable ignored) {
+            }
+        }
+
+        final ResponseEntity<String> responseEntity = restTemplate.postForEntity(
+            instrumentedServiceUrl + "/e2e/users",
+            user,
+            String.class
+        );
+        LOGGER.info("responseEntity: {}, {}", responseEntity.getStatusCode(), responseEntity.getBody());
+        assertThat(responseEntity.getStatusCode()).isEqualTo(HttpStatus.OK);
+
+        verifyTraces(startTime);
+
+        verifyServices(startTime);
+
+        verifyTopo(startTime);
+    }
+
+    private void verifyTopo(LocalDateTime minutesAgo) throws Exception {
+        final TopoData topoData = queryClient.topo(
+            new TopoQuery()
+                .stepByMinute()
+                .start(minutesAgo)
+                .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+        );
+
+        InputStream expectedInputStream =
+            new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.topo.yml").getInputStream();
+
+        final TopoMatcher topoMatcher = yaml.loadAs(expectedInputStream, TopoMatcher.class);
+        topoMatcher.verify(topoData);
+    }
+
+    private void verifyServices(LocalDateTime minutesAgo) throws Exception {
+        List<Service> services = queryClient.services(
+            new ServicesQuery()
+                .start(minutesAgo)
+                .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+        );
+        while (services.isEmpty()) {
+            LOGGER.warn("services is null, will retry to query");
+            services = queryClient.services(
+                new ServicesQuery()
+                    .start(minutesAgo)
+                    .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+            );
+            Thread.sleep(retryInterval);
+        }
+
+        InputStream expectedInputStream =
+            new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.services.yml").getInputStream();
+
+        final ServicesMatcher servicesMatcher = yaml.loadAs(expectedInputStream, ServicesMatcher.class);
+        servicesMatcher.verify(services);
+
+        for (Service service : services) {
+            LOGGER.info("verifying service instances: {}", service);
+
+            verifyServiceMetrics(service, minutesAgo);
+
+            Instances instances = verifyServiceInstances(minutesAgo, service);
+
+            verifyInstancesMetrics(instances, minutesAgo);
+
+            Endpoints endpoints = verifyServiceEndpoints(minutesAgo, service);
+
+            verifyEndpointsMetrics(endpoints, minutesAgo);
+        }
+    }
+
+    private Instances verifyServiceInstances(LocalDateTime minutesAgo, Service service) throws Exception {
+        Instances instances = queryClient.instances(
+            new InstancesQuery()
+                .serviceId(service.getKey())
+                .start(minutesAgo)
+                .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+        );
+        while (instances == null) {
+            LOGGER.warn("instances is null, will retry to query");
+            instances = queryClient.instances(
+                new InstancesQuery()
+                    .serviceId(service.getKey())
+                    .start(minutesAgo)
+                    .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+            );
+            Thread.sleep(retryInterval);
+        }
+        InputStream expectedInputStream =
+            new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.instances.yml").getInputStream();
+        final InstancesMatcher instancesMatcher = yaml.loadAs(expectedInputStream, InstancesMatcher.class);
+        instancesMatcher.verify(instances);
+        return instances;
+    }
+
+    private Endpoints verifyServiceEndpoints(LocalDateTime minutesAgo, Service service) throws Exception {
+        Endpoints endpoints = queryClient.endpoints(
+            new EndpointQuery().serviceId(service.getKey())
+        );
+        while (endpoints == null) {
+            LOGGER.warn("endpoints is null, will retry to query");
+            endpoints = queryClient.endpoints(
+                new EndpointQuery().serviceId(service.getKey())
+            );
+            Thread.sleep(retryInterval);
+        }
+        InputStream expectedInputStream =
+            new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.endpoints.yml").getInputStream();
+        final EndpointsMatcher endpointsMatcher = yaml.loadAs(expectedInputStream, EndpointsMatcher.class);
+        endpointsMatcher.verify(endpoints);
+        return endpoints;
+    }
+
+    private void verifyInstancesMetrics(Instances instances, final LocalDateTime minutesAgo) throws Exception {
+        for (Instance instance : instances.getInstances()) {
+            for (String metricsName : ALL_INSTANCE_METRICS) {
+                LOGGER.info("verifying service instance response time: {}", instance);
+
+                boolean matched = false;
+                while (!matched) {
+                    LOGGER.warn("instanceRespTime is null, will retry to query");
+                    Metrics instanceRespTime = queryClient.metrics(
+                            new MetricsQuery()
+                                .stepByMinute()
+                                .metricsName(metricsName)
+                                .start(minutesAgo)
+                                .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                                .id(instance.getKey())
+                        );
+                    AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
+                    MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
+                    greaterThanZero.setValue("gt 0");
+                    instanceRespTimeMatcher.setValue(greaterThanZero);
+                    try {
+                        instanceRespTimeMatcher.verify(instanceRespTime);
+                        matched = true;
+                    } catch (Throwable ignored) {
+                        Thread.sleep(retryInterval);
+                    }
+                    LOGGER.info("{}: {}", metricsName, instanceRespTime);
+                }
+            }
+        }
+    }
+
+    private void verifyEndpointsMetrics(Endpoints endpoints, final LocalDateTime minutesAgo) throws Exception {
+        for (Endpoint endpoint : endpoints.getEndpoints()) {
+            if (!endpoint.getLabel().equals("/e2e/users")) {
+                continue;
+            }
+            for (String metricName : ALL_ENDPOINT_METRICS) {
+                LOGGER.info("verifying endpoint {}, metrics: {}", endpoint, metricName);
+
+                boolean matched = false;
+                while (!matched) {
+                    LOGGER.warn("serviceMetrics is null, will retry to query");
+                    Metrics metrics = queryClient.metrics(
+                        new MetricsQuery()
+                            .stepByMinute()
+                            .metricsName(metricName)
+                            .start(minutesAgo)
+                            .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                            .id(endpoint.getKey())
+                    );
+                    AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
+                    MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
+                    greaterThanZero.setValue("gt 0");
+                    instanceRespTimeMatcher.setValue(greaterThanZero);
+                    try {
+                        instanceRespTimeMatcher.verify(metrics);
+                        matched = true;
+                    } catch (Throwable ignored) {
+                        Thread.sleep(retryInterval);
+                    }
+                    LOGGER.info("metrics: {}", metrics);
+                }
+            }
+        }
+    }
+
+    private void verifyServiceMetrics(Service service, final LocalDateTime minutesAgo) throws Exception {
+        for (String metricName : ALL_SERVICE_METRICS) {
+            LOGGER.info("verifying service {}, metrics: {}", service, metricName);
+
+            boolean matched = false;
+            while (!matched) {
+                Metrics serviceMetrics = queryClient.metrics(
+                    new MetricsQuery()
+                        .stepByMinute()
+                        .metricsName(metricName)
+                        .start(minutesAgo)
+                        .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                        .id(service.getKey())
+                );
+                AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher();
+                MetricsValueMatcher greaterThanZero = new MetricsValueMatcher();
+                greaterThanZero.setValue("gt 0");
+                instanceRespTimeMatcher.setValue(greaterThanZero);
+                try {
+                    instanceRespTimeMatcher.verify(serviceMetrics);
+                    matched = true;
+                } catch (Throwable ignored) {
+                    Thread.sleep(retryInterval);
+                }
+                LOGGER.info("serviceMetrics: {}", serviceMetrics);
+            }
+        }
+    }
+
+    private void verifyTraces(LocalDateTime minutesAgo) throws Exception {
+        List<Trace> traces = queryClient.traces(
+            new TracesQuery()
+                .stepBySecond()
+                .start(minutesAgo)
+                .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                .orderByStartTime()
+        );
+        while (traces.isEmpty()) {
+            LOGGER.warn("traces is empty, will retry to query");
+            traces = queryClient.traces(
+                new TracesQuery()
+                    .stepBySecond()
+                    .start(minutesAgo)
+                    .end(LocalDateTime.now(ZoneOffset.UTC).plusMinutes(1))
+                    .orderByStartTime()
+            );
+            Thread.sleep(retryInterval);
+        }
+
+        InputStream expectedInputStream =
+            new ClassPathResource("expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.traces.yml").getInputStream();
+
+        final TracesMatcher tracesMatcher = yaml.loadAs(expectedInputStream, TracesMatcher.class);
+        tracesMatcher.verifyLoosely(traces);
+    }
+}

test/e2e/e2e-cluster/test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.endpoints.yml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+endpoints:
+  - key: not null
+    label: /e2e/users

test/e2e/e2e-cluster/test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.instances.yml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+instances:
+  - key: gt 0
+    label: not null
+    attributes:
+      - name: os_name
+        value: not null
+      - name: host_name
+        value: not null
+      - name: process_no
+        value: gt 0
+      - name: ipv4s
+        value: not null

test/e2e/e2e-cluster/test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.services.yml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# 1 health-check by docker-maven-plugin
+# 1 drop table if exists, because we have `ddl-auto: create-drop`
+# 1 drop sequence
+# 1 create sequence
+# 1 create table statement
+
+services:
+  - key: not null
+    label: provider
+
+  - key: not null
+    label: consumer

test/e2e/e2e-cluster/test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.topo.yml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+nodes:
+  - id: not null
+    name: User
+    type: USER
+  - id: not null
+    name: provider
+    type: Tomcat
+  - id: not null
+    name: consumer
+    type: Tomcat
+  - id: not null
+    name: "localhost:-1"
+    type: H2
+calls:
+  - id: not null
+    source: not null
+    target: not null
+  - id: not null
+    source: not null
+    target: not null
+  - id: not null
+    source: not null
+    target: not null

test/e2e/e2e-cluster/test-runner/src/test/resources/expected-data/org.apache.skywalking.e2e.ClusterVerificationITCase.traces.yml

+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+traces:
+  - key: not null
+    endpointNames:
+      - /e2e/users
+    duration: ge 0
+    start: gt 0
+    isError: false
+    traceIds:
+      - not null

test/e2e/e2e-single-service/pom.xml

@@ -72,7 +72,7 @@
                 <groupId>io.fabric8</groupId>
                 <artifactId>docker-maven-plugin</artifactId>
                 <configuration>
-                    <containerNamePattern>%a-%t</containerNamePattern>
+                    <containerNamePattern>%a-%t-%i</containerNamePattern>
                     <images>
                         <image>
                             <name>skyapm/e2e-container:${e2e.container.version}</name>

test/e2e/e2e-single-service/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java

@@ -59,19 +59,9 @@ import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ENDPOINT_P50;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ENDPOINT_P75;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ENDPOINT_P90;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ENDPOINT_P95;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.ENDPOINT_P99;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.SERVICE_INSTANCE_CPM;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.SERVICE_INSTANCE_RESP_TIME;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.SERVICE_INSTANCE_SLA;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.SERVICE_P50;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.SERVICE_P75;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.SERVICE_P90;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.SERVICE_P95;
-import static org.apache.skywalking.e2e.metrics.MetricsQuery.SERVICE_P99;
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_ENDPOINT_METRICS;
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_INSTANCE_METRICS;
+import static org.apache.skywalking.e2e.metrics.MetricsQuery.ALL_SERVICE_METRICS;
 import static org.assertj.core.api.Assertions.assertThat;
 
 /**
@@ -82,6 +72,8 @@ public class SampleVerificationITCase {
     private static final Logger LOGGER = LoggerFactory.getLogger(SampleVerificationITCase.class);
 
     private final RestTemplate restTemplate = new RestTemplate();
+    private final int retryTimes = 5;
+    private final int retryInterval = 30;
 
     private SimpleQueryClient queryClient;
     private String instrumentedServiceUrl;
@@ -90,11 +82,10 @@ public class SampleVerificationITCase {
     public void setUp() {
         final String swWebappHost = System.getProperty("sw.webapp.host", "127.0.0.1");
         final String swWebappPort = System.getProperty("sw.webapp.port", "32783");
-        final String instrumentedServiceHost0 = System.getProperty("client.host", "127.0.0.1");
-        final String instrumentedServicePort0 = System.getProperty("client.port", "32782");
-        final String queryClientUrl = "http://" + swWebappHost + ":" + swWebappPort + "/graphql";
-        queryClient = new SimpleQueryClient(queryClientUrl);
-        instrumentedServiceUrl = "http://" + instrumentedServiceHost0 + ":" + instrumentedServicePort0;
+        final String instrumentedServiceHost = System.getProperty("client.host", "127.0.0.1");
+        final String instrumentedServicePort = System.getProperty("client.port", "32782");
+        queryClient = new SimpleQueryClient(swWebappHost, swWebappPort);
+        instrumentedServiceUrl = "http://" + instrumentedServiceHost + ":" + instrumentedServicePort;
     }
 
     @Test
@@ -111,13 +102,29 @@ public class SampleVerificationITCase {
         );
         assertThat(responseEntity.getStatusCode()).isEqualTo(HttpStatus.OK);
 
-        Thread.sleep(5000);
-
-        verifyTraces(minutesAgo);
+        doRetryableVerification(() -> {
+            try {
+                verifyTraces(minutesAgo);
+            } catch (Exception e) {
+                LOGGER.warn(e.getMessage(), e);
+            }
+        });
 
-        verifyServices(minutesAgo);
+        doRetryableVerification(() -> {
+            try {
+                verifyServices(minutesAgo);
+            } catch (Exception e) {
+                LOGGER.warn(e.getMessage(), e);
+            }
+        });
 
-        verifyTopo(minutesAgo);
+        doRetryableVerification(() -> {
+            try {
+                verifyTopo(minutesAgo);
+            } catch (Exception e) {
+                LOGGER.warn(e.getMessage(), e);
+            }
+        });
     }
 
     private void verifyTopo(LocalDateTime minutesAgo) throws Exception {
@@ -125,7 +132,7 @@ public class SampleVerificationITCase {
 
         final TopoData topoData = queryClient.topo(
             new TopoQuery()
-                .step("MINUTE")
+                .stepByMinute()
                 .start(minutesAgo.minusDays(1))
                 .end(now)
         );
@@ -194,17 +201,12 @@ public class SampleVerificationITCase {
     }
 
     private void verifyInstancesMetrics(Instances instances) throws Exception {
-        final String[] instanceMetricsNames = new String[] {
-            SERVICE_INSTANCE_RESP_TIME,
-            SERVICE_INSTANCE_CPM,
-            SERVICE_INSTANCE_SLA
-        };
         for (Instance instance : instances.getInstances()) {
-            for (String metricsName : instanceMetricsNames) {
+            for (String metricsName : ALL_INSTANCE_METRICS) {
                 LOGGER.info("verifying service instance response time: {}", instance);
                 final Metrics instanceRespTime = queryClient.metrics(
                     new MetricsQuery()
-                        .step("MINUTE")
+                        .stepByMinute()
                         .metricsName(metricsName)
                         .id(instance.getKey())
                 );
@@ -219,22 +221,15 @@ public class SampleVerificationITCase {
     }
 
     private void verifyEndpointsMetrics(Endpoints endpoints) throws Exception {
-        final String[] endpointMetricsNames = {
-            ENDPOINT_P99,
-            ENDPOINT_P95,
-            ENDPOINT_P90,
-            ENDPOINT_P75,
-            ENDPOINT_P50
-        };
         for (Endpoint endpoint : endpoints.getEndpoints()) {
             if (!endpoint.getLabel().equals("/e2e/users")) {
                 continue;
             }
-            for (String metricName : endpointMetricsNames) {
+            for (String metricName : ALL_ENDPOINT_METRICS) {
                 LOGGER.info("verifying endpoint {}, metrics: {}", endpoint, metricName);
                 final Metrics metrics = queryClient.metrics(
                     new MetricsQuery()
-                        .step("MINUTE")
+                        .stepByMinute()
                         .metricsName(metricName)
                         .id(endpoint.getKey())
                 );
@@ -249,18 +244,11 @@ public class SampleVerificationITCase {
     }
 
     private void verifyServiceMetrics(Service service) throws Exception {
-        final String[] serviceMetrics = {
-            SERVICE_P99,
-            SERVICE_P95,
-            SERVICE_P90,
-            SERVICE_P75,
-            SERVICE_P50
-        };
-        for (String metricName : serviceMetrics) {
+        for (String metricName : ALL_SERVICE_METRICS) {
             LOGGER.info("verifying service {}, metrics: {}", service, metricName);
             final Metrics instanceRespTime = queryClient.metrics(
                 new MetricsQuery()
-                    .step("MINUTE")
+                    .stepByMinute()
                     .metricsName(metricName)
                     .id(service.getKey())
             );
@@ -289,4 +277,15 @@ public class SampleVerificationITCase {
         final TracesMatcher tracesMatcher = new Yaml().loadAs(expectedInputStream, TracesMatcher.class);
         tracesMatcher.verify(traces);
     }
+
+    private void doRetryableVerification(Runnable runnable) throws InterruptedException {
+        for (int i = 0; i < retryTimes; i++) {
+            try {
+                runnable.run();
+                break;
+            } catch (Throwable ignored) {
+                Thread.sleep(retryInterval);
+            }
+        }
+    }
 }

test/e2e/pom.xml

@@ -34,6 +34,7 @@
     <modules>
         <module>e2e-base</module>
         <module>e2e-single-service</module>
+        <module>e2e-cluster</module>
     </modules>
 
     <properties>
@@ -51,6 +52,9 @@
         <snake.version>1.23</snake.version>
         <gson.version>2.8.5</gson.version>
         <h2.version>1.4.199</h2.version>
+        <elasticsearch.version>6.3.2</elasticsearch.version>
+        <zookeeper.image.version>3.5</zookeeper.image.version>
+        <lombok.version>1.18.4</lombok.version>
 
         <!-- build.id is an available environment variable in Jenkins to
              distinguish the different build jobs, once Jenkins job is aborted,
@@ -64,6 +68,9 @@
         <maven-failsafe-plugin.version>2.22.0</maven-failsafe-plugin.version>
         <maven-compiler-plugin.version>3.8.0</maven-compiler-plugin.version>
         <docker-maven-plugin.version>0.30.0</docker-maven-plugin.version>
+        <surefire.version>2.12.4</surefire.version>
+
+        <skipSurefire>true</skipSurefire>
     </properties>
 
     <dependencies>
@@ -92,6 +99,12 @@
             <artifactId>guava</artifactId>
             <version>${guava.version}</version>
         </dependency>
+        <dependency>
+            <groupId>org.projectlombok</groupId>
+            <artifactId>lombok</artifactId>
+            <version>${lombok.version}</version>
+            <scope>provided</scope>
+        </dependency>
     </dependencies>
 
     <build>
@@ -150,6 +163,14 @@
                     <encoding>${project.build.sourceEncoding}</encoding>
                 </configuration>
             </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-surefire-plugin</artifactId>
+                <version>${surefire.version}</version>
+                <configuration>
+                    <skip>${skipSurefire}</skip>
+                </configuration>
+            </plugin>
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-failsafe-plugin</artifactId>