From 721f86485cdafb37921c3593e366344e2846ab9f Mon Sep 17 00:00:00 2001 From: kezhenxu94 Date: Tue, 10 Sep 2019 13:06:57 +0800 Subject: [PATCH] Add TTL E2E test (#3437) * Add TTL E2E test * Add to Jenkins stage and minor bugfix * Upgrade e2e container version * Polish and minor fix * Polish --- .gitmodules | 3 + Jenkinsfile-E2E | 11 +- docker/oap/docker-entrypoint.sh | 1 + docs/en/setup/backend/ttl.md | 1 + .../oap/server/core/CoreModuleConfig.java | 1 + .../oap/server/core/CoreModuleProvider.java | 2 +- .../core/storage/ttl/DataTTLKeeperTimer.java | 28 +- .../src/main/assembly/application.yml | 1 + .../src/main/resources/application.yml | 1 + .../apache/skywalking/e2e/AbstractQuery.java | 52 +++- .../e2e/metrics/AllOfMetricsMatcher.java | 60 ++++ .../e2e/verification/AbstractMatcher.java | 10 + test/e2e/e2e-ttl/e2e-ttl-es/pom.xml | 193 ++++++++++++ .../e2e-ttl-es/src/docker/es_storage.awk | 64 ++++ .../e2e-ttl-es/src/docker/rc.d/rc0-prepare.sh | 28 ++ .../e2e-ttl-es/src/docker/rc.d/rc1-startup.sh | 37 +++ test/e2e/e2e-ttl/e2e-ttl-es/src/main/proto | 1 + .../skywalking/e2e/StorageTTLITCase.java | 284 ++++++++++++++++++ test/e2e/e2e-ttl/pom.xml | 36 +++ test/e2e/pom.xml | 5 +- 20 files changed, 798 insertions(+), 21 deletions(-) create mode 100644 test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/AllOfMetricsMatcher.java create mode 100644 test/e2e/e2e-ttl/e2e-ttl-es/pom.xml create mode 100644 test/e2e/e2e-ttl/e2e-ttl-es/src/docker/es_storage.awk create mode 100755 test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc0-prepare.sh create mode 100755 test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc1-startup.sh create mode 160000 test/e2e/e2e-ttl/e2e-ttl-es/src/main/proto create mode 100644 test/e2e/e2e-ttl/e2e-ttl-es/src/test/java/org/apache/skywalking/e2e/StorageTTLITCase.java create mode 100644 test/e2e/e2e-ttl/pom.xml diff --git a/.gitmodules b/.gitmodules index 72f71a2c2c..e0b0dd2809 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "skywalking-ui"] path = skywalking-ui url = https://github.com/apache/skywalking-rocketbot-ui.git +[submodule "test/e2e/e2e-ttl/e2e-ttl-es/src/main/proto"] + path = test/e2e/e2e-ttl/e2e-ttl-es/src/main/proto + url = https://github.com/apache/skywalking-data-collect-protocol.git diff --git a/Jenkinsfile-E2E b/Jenkinsfile-E2E index 9546fd93fd..2275386042 100755 --- a/Jenkinsfile-E2E +++ b/Jenkinsfile-E2E @@ -44,12 +44,13 @@ pipeline { // we're using them as a barrier here to filter out some invalid PRs (fast-fail) // thus save unnecessary E2E builds(which is expensive) sh './mvnw checkstyle:check apache-rat:check' - sh './mvnw -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -Dmaven.compiler.maxmem=3072 -DskipTests clean package' + sh './mvnw -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -Dmaven.compiler.maxmem=3072 -DskipTests clean install' // Some of the tests will modify files in the distribution folder, e.g. cluster test will modify the application.yml // so we give each test a separate distribution folder here sh 'mkdir -p dist-for-single-node-service && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-single-node-service' sh 'mkdir -p dist-for-cluster && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-cluster' sh 'mkdir -p dist-for-agent-reboot && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-agent-reboot' + sh 'mkdir -p dist-for-ttl-es && tar -zxf dist/apache-skywalking-apm-bin.tar.gz -C dist-for-ttl-es' } } @@ -78,6 +79,12 @@ pipeline { sh './mvnw -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml -pl e2e-agent-reboot -am verify' } } + + stage('Run TTL Tests') { + steps { + sh './mvnw -Dbuild.id=${BUILD_ID} -f test/e2e/pom.xml -pl e2e-ttl/e2e-ttl-es -am verify' + } + } } } } @@ -95,7 +102,7 @@ pipeline { // inside the container // // Delete all distribution folder - sh 'docker run -v $(pwd):/sw alpine sleep 10 && rm -rf /sw/dist-for-cluster/* /sw/dist-for-single-node-service/* /sw/dist-for-agent-reboot/*' + sh 'docker run -v $(pwd):/sw alpine sh -c "sleep 10 && rm -rf /sw/dist-for-cluster/* /sw/dist-for-single-node-service/* /sw/dist-for-agent-reboot/* /sw/dist-for-ttl-es/*"' deleteDir() } } diff --git a/docker/oap/docker-entrypoint.sh b/docker/oap/docker-entrypoint.sh index 7c2756b370..b2f4c83af8 100755 --- a/docker/oap/docker-entrypoint.sh +++ b/docker/oap/docker-entrypoint.sh @@ -259,6 +259,7 @@ core: - Month # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted. enableDataKeeperExecutor: \${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close. + dataKeeperExecutePeriod: \${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute recordDataTTL: \${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute minuteMetricsDataTTL: \${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute hourMetricsDataTTL: \${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour diff --git a/docs/en/setup/backend/ttl.md b/docs/en/setup/backend/ttl.md index e78aa95334..7a4ed5088c 100644 --- a/docs/en/setup/backend/ttl.md +++ b/docs/en/setup/backend/ttl.md @@ -8,6 +8,7 @@ You have following settings for different types. ```yaml # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted. enableDataKeeperExecutor: ${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close. + dataKeeperExecutePeriod: ${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java index 75050115c5..c7c26e1f75 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java @@ -45,6 +45,7 @@ public class CoreModuleConfig extends ModuleConfig { */ @Setter private long persistentPeriod = 3; @Setter private boolean enableDataKeeperExecutor = true; + @Setter private int dataKeeperExecutePeriod = 5; @Setter private int recordDataTTL; @Setter private int minuteMetricsDataTTL; @Setter private int hourMetricsDataTTL; diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index 6237f68098..a188bef51c 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -203,7 +203,7 @@ public class CoreModuleProvider extends ModuleProvider { PersistenceTimer.INSTANCE.start(getManager(), moduleConfig); if (moduleConfig.isEnableDataKeeperExecutor()) { - DataTTLKeeperTimer.INSTANCE.start(getManager()); + DataTTLKeeperTimer.INSTANCE.start(getManager(), moduleConfig); } CacheUpdateTimer.INSTANCE.start(getManager()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/DataTTLKeeperTimer.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/DataTTLKeeperTimer.java index 5e7220fee9..17b41233ef 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/DataTTLKeeperTimer.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/DataTTLKeeperTimer.java @@ -18,18 +18,26 @@ package org.apache.skywalking.oap.server.core.storage.ttl; -import java.io.IOException; -import java.util.List; -import java.util.concurrent.*; import org.apache.skywalking.apm.util.RunnableWithExceptionProtection; import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.CoreModuleConfig; import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; -import org.apache.skywalking.oap.server.core.cluster.*; -import org.apache.skywalking.oap.server.core.storage.*; -import org.apache.skywalking.oap.server.core.storage.model.*; +import org.apache.skywalking.oap.server.core.cluster.ClusterModule; +import org.apache.skywalking.oap.server.core.cluster.ClusterNodesQuery; +import org.apache.skywalking.oap.server.core.cluster.RemoteInstance; +import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.model.IModelGetter; +import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.library.module.ModuleManager; import org.apache.skywalking.oap.server.library.util.CollectionUtils; -import org.slf4j.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; /** * @author peng-yongsheng @@ -42,13 +50,13 @@ public enum DataTTLKeeperTimer { private ModuleManager moduleManager; private ClusterNodesQuery clusterNodesQuery; - public void start(ModuleManager moduleManager) { + public void start(ModuleManager moduleManager, CoreModuleConfig moduleConfig) { this.moduleManager = moduleManager; this.clusterNodesQuery = moduleManager.find(ClusterModule.NAME).provider().getService(ClusterNodesQuery.class); Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate( - new RunnableWithExceptionProtection(this::delete, - t -> logger.error("Remove data in background failure.", t)), 5, 5, TimeUnit.MINUTES); + new RunnableWithExceptionProtection(this::delete, t -> logger.error("Remove data in background failure.", t)), + moduleConfig.getDataKeeperExecutePeriod(), moduleConfig.getDataKeeperExecutePeriod(), TimeUnit.MINUTES); } private void delete() { diff --git a/oap-server/server-starter/src/main/assembly/application.yml b/oap-server/server-starter/src/main/assembly/application.yml index a2dad82a8c..a110ecad57 100644 --- a/oap-server/server-starter/src/main/assembly/application.yml +++ b/oap-server/server-starter/src/main/assembly/application.yml @@ -61,6 +61,7 @@ core: - Month # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted. enableDataKeeperExecutor: ${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close. + dataKeeperExecutePeriod: ${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index 1f98082941..24cebff639 100755 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -60,6 +60,7 @@ core: - Month # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted. enableDataKeeperExecutor: ${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close. + dataKeeperExecutePeriod: ${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour diff --git a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/AbstractQuery.java b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/AbstractQuery.java index 2b38b9e2ea..f09c177105 100644 --- a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/AbstractQuery.java +++ b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/AbstractQuery.java @@ -28,6 +28,8 @@ import java.time.format.DateTimeFormatter; public abstract class AbstractQuery> { private static final DateTimeFormatter TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HHmmss"); private static final DateTimeFormatter MINUTE_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd HHmm"); + private static final DateTimeFormatter DAY_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM-dd"); + private static final DateTimeFormatter MONTH_TIME_FORMATTER = DateTimeFormatter.ofPattern("yyyy-MM"); private String start; private String end; @@ -37,9 +39,19 @@ public abstract class AbstractQuery> { if (start != null) { return start; } - return "SECOND".equals(step()) - ? LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(TIME_FORMATTER) - : LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(MINUTE_TIME_FORMATTER); + if ("SECOND".equals(step())) { + return LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(TIME_FORMATTER); + } + if ("MINUTE".equals(step())) { + return LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(MINUTE_TIME_FORMATTER); + } + if ("DAY".equals(step())) { + return LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(DAY_TIME_FORMATTER); + } + if ("MONTH".equals(step())) { + return LocalDateTime.now(ZoneOffset.UTC).minusMinutes(15).format(MONTH_TIME_FORMATTER); + } + return null; } public T start(String start) { @@ -52,6 +64,10 @@ public abstract class AbstractQuery> { this.start = start.format(MINUTE_TIME_FORMATTER); } else if ("SECOND".equals(step())) { this.start = start.format(TIME_FORMATTER); + } else if ("DAY".equals(step())) { + this.start = start.format(DAY_TIME_FORMATTER); + } else if ("MONTH".equals(step())) { + this.start = start.format(MONTH_TIME_FORMATTER); } return (T) this; } @@ -60,9 +76,19 @@ public abstract class AbstractQuery> { if (end != null) { return end; } - return "SECOND".equals(step()) - ? LocalDateTime.now(ZoneOffset.UTC).format(TIME_FORMATTER) - : LocalDateTime.now(ZoneOffset.UTC).format(MINUTE_TIME_FORMATTER); + if ("SECOND".equals(step())) { + return LocalDateTime.now(ZoneOffset.UTC).format(TIME_FORMATTER); + } + if ("MINUTE".equals(step())) { + return LocalDateTime.now(ZoneOffset.UTC).format(MINUTE_TIME_FORMATTER); + } + if ("DAY".equals(step())) { + return LocalDateTime.now(ZoneOffset.UTC).format(DAY_TIME_FORMATTER); + } + if ("MONTH".equals(step())) { + return LocalDateTime.now(ZoneOffset.UTC).format(MONTH_TIME_FORMATTER); + } + return null; } public AbstractQuery end(String end) { @@ -75,6 +101,10 @@ public abstract class AbstractQuery> { this.end = end.format(MINUTE_TIME_FORMATTER); } else if ("SECOND".equals(step())) { this.end = end.format(TIME_FORMATTER); + } else if ("DAY".equals(step())) { + this.end = end.format(DAY_TIME_FORMATTER); + } else if ("MONTH".equals(step())) { + this.end = end.format(MONTH_TIME_FORMATTER); } return (T) this; } @@ -88,6 +118,16 @@ public abstract class AbstractQuery> { return (T) this; } + public T stepByMonth() { + this.step = "MONTH"; + return (T) this; + } + + public T stepByDay() { + this.step = "DAY"; + return (T) this; + } + public T stepByMinute() { this.step = "MINUTE"; return (T) this; diff --git a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/AllOfMetricsMatcher.java b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/AllOfMetricsMatcher.java new file mode 100644 index 0000000000..570cfc4d44 --- /dev/null +++ b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/metrics/AllOfMetricsMatcher.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e.metrics; + +import org.apache.skywalking.e2e.verification.AbstractMatcher; +import org.assertj.core.api.Condition; + +import java.util.function.Predicate; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author kezhenxu94 + */ +public class AllOfMetricsMatcher extends AbstractMatcher { + private MetricsValueMatcher value; + + @Override + public void verify(Metrics metrics) { + assertThat(metrics.getValues()).isNotEmpty(); + assertThat(metrics.getValues()).allMatch(value -> { + try { + AllOfMetricsMatcher.this.getValue().verify(value); + return true; + } catch (Throwable t) { + return false; + } + }); + } + + public MetricsValueMatcher getValue() { + return value; + } + + public void setValue(MetricsValueMatcher value) { + this.value = value; + } + + @Override + public String toString() { + return "AllOfMetricsMatcher{" + + "value=" + value + + '}'; + } +} diff --git a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/verification/AbstractMatcher.java b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/verification/AbstractMatcher.java index a47f5f5f15..ae82914545 100644 --- a/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/verification/AbstractMatcher.java +++ b/test/e2e/e2e-base/src/main/java/org/apache/skywalking/e2e/verification/AbstractMatcher.java @@ -28,6 +28,7 @@ import static org.assertj.core.api.Assertions.assertThat; */ public abstract class AbstractMatcher { private static final Pattern NE_MATCHER = Pattern.compile("ne\\s+(?.+)"); + private static final Pattern EQ_MATCHER = Pattern.compile("eq\\s+(?.+)"); private static final Pattern GT_MATCHER = Pattern.compile("gt\\s+(?.+)"); private static final Pattern GE_MATCHER = Pattern.compile("ge\\s+(?.+)"); private static final Pattern NN_MATCHER = Pattern.compile("^not null$"); @@ -65,6 +66,15 @@ public abstract class AbstractMatcher { return; } + matcher = EQ_MATCHER.matcher(expected); + if (matcher.find()) { + String val = matcher.group("val"); + + assertThat(val).isNotBlank(); + assertThat(Double.parseDouble(actual)).isEqualTo(Double.parseDouble(val)); + return; + } + assertThat(actual).isEqualTo(expected); } diff --git a/test/e2e/e2e-ttl/e2e-ttl-es/pom.xml b/test/e2e/e2e-ttl/e2e-ttl-es/pom.xml new file mode 100644 index 0000000000..ab0480f45d --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-es/pom.xml @@ -0,0 +1,193 @@ + + + + + + e2e-ttl + org.apache.skywalking + 1.0.0 + + 4.0.0 + + e2e-ttl-es + + + dist-for-ttl-es + 1.2 + skywalking-e2e-container-${build.id}-ttl-es + 1.14.0 + 1.5.0.Final + 0.5.0 + + + + + org.apache.skywalking + e2e-base + ${project.version} + + + io.grpc + grpc-protobuf + ${grpc.version} + + + io.grpc + grpc-stub + ${grpc.version} + + + io.grpc + grpc-netty + ${grpc.version} + provided + + + + + + + io.fabric8 + docker-maven-plugin + + %a-%t-%i + Always + + + elastic/elasticsearch:${elasticsearch.version} + ${e2e.container.name.prefix}-elasticsearch + + + es.port:9200 + + + + http://localhost:${es.port} + GET + 200 + + + + + single-node + 500 + 500 + + + + + skyapm/e2e-container:${e2e.container.version} + ${e2e.container.name.prefix} + + + standalone + + ${e2e.container.name.prefix}-elasticsearch:9200 + + + + ${e2e.container.name.prefix}-elasticsearch + + + +webapp.host:webapp.port:8081 + +oap.host:oap.port:11800 + + + ${e2e.container.name.prefix}-elasticsearch + + + + + ../../../../${e2e.dist.directory}/apache-skywalking-apm-bin:/sw + + + ${project.basedir}/src/docker/rc.d:/rc.d:ro + + + ${project.basedir}/src/docker/es_storage.awk:/es_storage.awk + + + + + SkyWalking e2e container is ready for tests + + + + + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + ${webapp.host} + ${webapp.port} + ${oap.host} + ${oap.port} + + + + + + verify + + + + + + org.xolstice.maven.plugins + protobuf-maven-plugin + ${protobuf-maven-plugin.version} + + + com.google.protobuf:protoc:3.3.0:exe:${os.detected.classifier} + + grpc-java + io.grpc:protoc-gen-grpc-java:1.8.0:exe:${os.detected.classifier} + + + + + + compile + compile-custom + + + + + + + + kr.motd.maven + os-maven-plugin + ${os-maven-plugin.version} + + + + + \ No newline at end of file diff --git a/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/es_storage.awk b/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/es_storage.awk new file mode 100644 index 0000000000..8dbdf75839 --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/es_storage.awk @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/awk -f + +BEGIN { + in_storage_section=0; + in_storage_es_section=0; + in_storage_h2_section=0; +} + +{ + if (in_storage_section == 0) { + in_storage_section=$0 ~ /^storage:$/ + } else { + in_storage_section=$0 ~ /^(#|\s{2})/ + } + + if (in_storage_section == 1) { + # in the storage: section now + # disable h2 module + if (in_storage_es_section == 0) { + in_storage_es_section=$0 ~ /^#?\s+elasticsearch:$/ + } else { + in_storage_es_section=$0 ~ /^#?\s{4}/ + } + if (in_storage_h2_section == 0) { + in_storage_h2_section=$0 ~ /^#?\s+h2:$/ + } else { + in_storage_h2_section=$0 ~ /^#?\s{4}/ + } + if (in_storage_es_section == 1) { + # in the storage.elasticsearch section now + # uncomment es config + gsub("^#", "", $0) + print + } else if (in_storage_h2_section == 1) { + # comment out h2 config + if ($0 !~ /^#/) { + print "#" $0 + } else { + print + } + } else { + print + } + } else { + print + } +} + diff --git a/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc0-prepare.sh b/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc0-prepare.sh new file mode 100755 index 0000000000..46c77a8a76 --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc0-prepare.sh @@ -0,0 +1,28 @@ +# Licensed to the SkyAPM under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash + +original_wd=$(pwd) + +# substitute application.yml to be capable of es mode +cd ${SW_HOME}/config \ + && awk -f /es_storage.awk application.yml > es_storage_app.yml \ + && mv es_storage_app.yml application.yml \ + && sed '//a' log4j2.xml > log4j2debuggable.xml \ + && mv log4j2debuggable.xml log4j2.xml + +cd ${original_wd} \ No newline at end of file diff --git a/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc1-startup.sh b/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc1-startup.sh new file mode 100755 index 0000000000..cef127fedc --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc1-startup.sh @@ -0,0 +1,37 @@ +# Licensed to the SkyAPM under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash + +echo 'starting OAP server...' \ + && SW_STORAGE_ES_BULK_ACTIONS=1 \ + SW_CORE_DATA_KEEPER_EXECUTE_PERIOD=1 \ + SW_STORAGE_ES_MONTH_METRIC_DATA_TTL=4 \ + SW_STORAGE_ES_OTHER_METRIC_DATA_TTL=5 \ + SW_STORAGE_ES_FLUSH_INTERVAL=1 \ + SW_RECEIVER_BUFFER_PATH=/tmp/oap/trace_buffer1 \ + SW_SERVICE_MESH_BUFFER_PATH=/tmp/oap/mesh_buffer1 \ + start_oap 'init' + +echo 'starting Web app...' \ + && start_webapp '0.0.0.0' 8081 + +echo "SkyWalking e2e container is ready for tests" + +tail -f ${OAP_LOG_DIR}/* \ + ${WEBAPP_LOG_DIR}/* \ + ${ES_HOME}/logs/elasticsearch.log \ + ${ES_HOME}/logs/stdout.log diff --git a/test/e2e/e2e-ttl/e2e-ttl-es/src/main/proto b/test/e2e/e2e-ttl/e2e-ttl-es/src/main/proto new file mode 160000 index 0000000000..7b244ff7ec --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-es/src/main/proto @@ -0,0 +1 @@ +Subproject commit 7b244ff7ec350910295eee85633e02d92a6f6b1c diff --git a/test/e2e/e2e-ttl/e2e-ttl-es/src/test/java/org/apache/skywalking/e2e/StorageTTLITCase.java b/test/e2e/e2e-ttl/e2e-ttl-es/src/test/java/org/apache/skywalking/e2e/StorageTTLITCase.java new file mode 100644 index 0000000000..5c538de677 --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-es/src/test/java/org/apache/skywalking/e2e/StorageTTLITCase.java @@ -0,0 +1,284 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.internal.DnsNameResolverProvider; +import io.grpc.netty.NettyChannelBuilder; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.network.common.DetectPoint; +import org.apache.skywalking.apm.network.servicemesh.MeshProbeDownstream; +import org.apache.skywalking.apm.network.servicemesh.Protocol; +import org.apache.skywalking.apm.network.servicemesh.ServiceMeshMetric; +import org.apache.skywalking.apm.network.servicemesh.ServiceMeshMetricServiceGrpc; +import org.apache.skywalking.e2e.metrics.AllOfMetricsMatcher; +import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher; +import org.apache.skywalking.e2e.metrics.Metrics; +import org.apache.skywalking.e2e.metrics.MetricsQuery; +import org.apache.skywalking.e2e.metrics.MetricsValueMatcher; +import org.apache.skywalking.e2e.service.Service; +import org.apache.skywalking.e2e.service.ServicesQuery; +import org.junit.Before; +import org.junit.Test; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import static org.apache.skywalking.e2e.metrics.MetricsQuery.SERVICE_P99; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author kezhenxu94 + */ +@Slf4j +public class StorageTTLITCase { + private static final int SW_STORAGE_ES_MONTH_METRIC_DATA_TTL = 4; + private static final int SW_STORAGE_ES_OTHER_METRIC_DATA_TTL = 5; + + private static final int MAX_INBOUND_MESSAGE_SIZE = 1024 * 1024 * 50; + private static final boolean USE_PLAIN_TEXT = true; + private static final boolean SUCCESS = true; + + private SimpleQueryClient queryClient; + + private ServiceMeshMetricServiceGrpc.ServiceMeshMetricServiceStub grpcStub; + + @Before + public void setUp() { + + final String swWebappHost = System.getProperty("sw.webapp.host", "127.0.0.1"); + final String swWebappPort = System.getProperty("sw.webapp.port", "32789"); + final String oapPort = System.getProperty("oap.port", "32788"); + queryClient = new SimpleQueryClient(swWebappHost, swWebappPort); + + final ManagedChannelBuilder builder = + NettyChannelBuilder.forAddress("127.0.0.1", Integer.parseInt(oapPort)) + .nameResolverFactory(new DnsNameResolverProvider()) + .maxInboundMessageSize(MAX_INBOUND_MESSAGE_SIZE) + .usePlaintext(USE_PLAIN_TEXT); + + final ManagedChannel channel = builder.build(); + + grpcStub = ServiceMeshMetricServiceGrpc.newStub(channel); + } + + @Test(timeout = 360000) + public void dayMetricsDataShouldBeRemovedAfterTTL() throws Exception { + + final ServiceMeshMetric.Builder builder = ServiceMeshMetric + .newBuilder() + .setSourceServiceName("e2e-test-source-service") + .setSourceServiceInstance("e2e-test-source-service-instance") + .setDestServiceName("e2e-test-dest-service") + .setDestServiceInstance("e2e-test-dest-service-instance") + .setEndpoint("e2e/test") + .setLatency(2000) + .setResponseCode(200) + .setStatus(SUCCESS) + .setProtocol(Protocol.HTTP) + .setDetectPoint(DetectPoint.server); + + final LocalDateTime now = LocalDateTime.now(); + final LocalDateTime startTime = now.minusDays(SW_STORAGE_ES_OTHER_METRIC_DATA_TTL + 1); + final LocalDateTime endTime = startTime.plusMinutes(1); + + final LocalDateTime queryStart = startTime; + final LocalDateTime queryEnd = now.minusDays(SW_STORAGE_ES_OTHER_METRIC_DATA_TTL); + + ensureSendingMetricsWorks( + builder, + startTime.toEpochSecond(ZoneOffset.UTC) * 1000, + endTime.toEpochSecond(ZoneOffset.UTC) * 1000, + queryStart, + queryEnd, + "DAY" + ); + + shouldBeEmptyBetweenTimeRange(queryStart, queryEnd, "DAY"); + } + + @Test(timeout = 360000) + public void monthMetricsDataShouldBeRemovedAfterTTL() throws Exception { + + final ServiceMeshMetric.Builder builder = ServiceMeshMetric + .newBuilder() + .setSourceServiceName("e2e-test-source-service") + .setSourceServiceInstance("e2e-test-source-service-instance") + .setDestServiceName("e2e-test-dest-service") + .setDestServiceInstance("e2e-test-dest-service-instance") + .setEndpoint("e2e/test") + .setLatency(2000) + .setResponseCode(200) + .setStatus(SUCCESS) + .setProtocol(Protocol.HTTP) + .setDetectPoint(DetectPoint.server); + + final LocalDateTime now = LocalDateTime.now(); + final LocalDateTime startTime = now.minusMonths(SW_STORAGE_ES_MONTH_METRIC_DATA_TTL + 1); + final LocalDateTime endTime = startTime.plusMinutes(1); + + final LocalDateTime queryStart = startTime; + final LocalDateTime queryEnd = now.minusMonths(SW_STORAGE_ES_MONTH_METRIC_DATA_TTL); + + ensureSendingMetricsWorks( + builder, + startTime.toEpochSecond(ZoneOffset.UTC) * 1000, + endTime.toEpochSecond(ZoneOffset.UTC) * 1000, + queryStart, + queryEnd, + "MONTH" + ); + + shouldBeEmptyBetweenTimeRange(queryStart, queryEnd, "MONTH"); + } + + private void shouldBeEmptyBetweenTimeRange( + final LocalDateTime queryStart, + final LocalDateTime queryEnd, + final String step + ) throws InterruptedException { + + boolean valid = false; + for (int i = 0; i < 10 && !valid; i++) { + try { + final Metrics serviceMetrics = queryMetrics(queryStart, queryEnd, step); + + log.info("ServiceMetrics: {}", serviceMetrics); + + AllOfMetricsMatcher instanceRespTimeMatcher = new AllOfMetricsMatcher(); + MetricsValueMatcher equalsZero = new MetricsValueMatcher(); + equalsZero.setValue("eq 0"); + instanceRespTimeMatcher.setValue(equalsZero); + try { + instanceRespTimeMatcher.verify(serviceMetrics); + valid = true; + } catch (Throwable t) { + log.warn("History metrics data are not deleted yet, {}", t.getMessage()); + Thread.sleep(60000); + } + } catch (Throwable t) { + log.warn("History metrics data are not deleted yet", t); + Thread.sleep(60000); + } + } + } + + private void ensureSendingMetricsWorks( + final ServiceMeshMetric.Builder builder, + final long startTime, + final long endTime, + final LocalDateTime queryStart, + final LocalDateTime queryEnd, + final String step + ) throws Exception { + + boolean prepared = false; + while (!prepared) { + sendMetrics( + builder + .setStartTime(startTime) + .setEndTime(endTime) + .build() + ); + + final Metrics serviceMetrics = queryMetrics(queryStart, queryEnd, step); + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + try { + instanceRespTimeMatcher.verify(serviceMetrics); + prepared = true; + } catch (Throwable ignored) { + sendMetrics( + builder + .setStartTime(startTime) + .setEndTime(endTime) + .build() + ); + Thread.sleep(10000); + } + } + } + + private Metrics queryMetrics( + final LocalDateTime queryStart, + final LocalDateTime queryEnd, + final String step + ) throws Exception { + for (int i = 0; i < 10; i++) { + try { + final List services = queryClient.services( + new ServicesQuery() + .start(LocalDateTime.now().minusDays(SW_STORAGE_ES_OTHER_METRIC_DATA_TTL)) + .end(LocalDateTime.now()) + ); + + log.info("Services: {}", services); + + assertThat(services).isNotEmpty(); + + String serviceId = services.stream().filter(it -> "e2e-test-dest-service".equals(it.getLabel())).findFirst().get().getKey(); + + return queryClient.metrics( + new MetricsQuery() + .id(serviceId) + .metricsName(SERVICE_P99) + .step(step) + .start(queryStart) + .end(queryEnd) + ); + } catch (Throwable ignored) { + Thread.sleep(10000); + } + } + return null; + } + + private void sendMetrics(final ServiceMeshMetric metrics) throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + + StreamObserver collect = grpcStub.collect(new StreamObserver() { + @Override + public void onNext(final MeshProbeDownstream meshProbeDownstream) { + + } + + @Override + public void onError(final Throwable throwable) { + throwable.printStackTrace(); + latch.countDown(); + } + + @Override + public void onCompleted() { + latch.countDown(); + } + }); + + collect.onNext(metrics); + + collect.onCompleted(); + + latch.await(); + } +} diff --git a/test/e2e/e2e-ttl/pom.xml b/test/e2e/e2e-ttl/pom.xml new file mode 100644 index 0000000000..eab8c608e1 --- /dev/null +++ b/test/e2e/e2e-ttl/pom.xml @@ -0,0 +1,36 @@ + + + + + + apache-skywalking-e2e + org.apache.skywalking + 1.0.0 + + 4.0.0 + + e2e-ttl + pom + + e2e-ttl-es + + + \ No newline at end of file diff --git a/test/e2e/pom.xml b/test/e2e/pom.xml index 21a3ffe029..12cbefaac0 100644 --- a/test/e2e/pom.xml +++ b/test/e2e/pom.xml @@ -36,6 +36,7 @@ e2e-single-service e2e-cluster e2e-agent-reboot + e2e-ttl @@ -50,7 +51,7 @@ 2.9.7 4.12 2.9.7 - 28.0-jre + 20.0 1.23 2.8.5 1.4.199 @@ -61,7 +62,7 @@ local -- GitLab