diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmStandardPersistence.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmStandardPersistence.java index 48dcc8a03a0accc4eb598c51994d9a394a1cb5d9..6237bfd2af664928236385ad217ca0c86604d331 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmStandardPersistence.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/alarm/AlarmStandardPersistence.java @@ -45,7 +45,7 @@ public class AlarmStandardPersistence implements AlarmCallback { record.setName(message.getName()); record.setAlarmMessage(message.getAlarmMessage()); record.setStartTime(message.getStartTime()); - record.setTimeBucket(TimeBucket.getSecondTimeBucket(message.getStartTime())); + record.setTimeBucket(TimeBucket.getRecordTimeBucket(message.getStartTime())); RecordStreamProcessor.getInstance().in(record); }); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java index 18ff6f976ae43d15d6149ef73bf0519c2d95601c..3df3470fb8a7af2849692de72366c1a46c5be6f8 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java @@ -25,7 +25,13 @@ import org.apache.skywalking.oap.server.core.UnexpectedException; */ public class TimeBucket { - public static long getSecondTimeBucket(long time) { + /** + * Record time bucket format in Second Unit. + * + * @param time Timestamp + * @return time in second format. + */ + public static long getRecordTimeBucket(long time) { return getTimeBucket(time, Downsampling.Second); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/GeneralStorageTTL.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/GeneralStorageTTL.java index aefaa53c6181c54429ec3c863564c8aeb108f75a..971bcb764126227fa6aeb417f87fcfda219f71a0 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/GeneralStorageTTL.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/GeneralStorageTTL.java @@ -24,7 +24,7 @@ import org.apache.skywalking.oap.server.core.analysis.Downsampling; */ public class GeneralStorageTTL implements StorageTTL { - @Override public TTLCalculator calculator(Downsampling downsampling) { + @Override public TTLCalculator metricsCalculator(Downsampling downsampling) { switch (downsampling) { case Hour: return new HourTTLCalculator(); @@ -38,4 +38,8 @@ public class GeneralStorageTTL implements StorageTTL { return new MinuteTTLCalculator(); } } + + @Override public TTLCalculator recordCalculator() { + return new RecordTTLCalculator(); + } } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/RecordTTLCalculator.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/RecordTTLCalculator.java new file mode 100644 index 0000000000000000000000000000000000000000..c06f26f5f950b5960ef8cb654e04b5abf84b239f --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/RecordTTLCalculator.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.storage.ttl; + +import org.apache.skywalking.oap.server.core.DataTTLConfig; +import org.joda.time.DateTime; + +/** + * Calculate TTL for record. + * + * @author wusheng + */ +public class RecordTTLCalculator implements TTLCalculator { + + @Override public long timeBefore(DateTime currentTime, DataTTLConfig dataTTLConfig) { + return Long.valueOf(currentTime.plusMinutes(0 - dataTTLConfig.getRecordDataTTL()).toString("yyyyMMddHHmmss")); + } +} \ No newline at end of file diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java index 0f386597f19983d35ce37a5fe9107bf6600bac59..7a0b4e9c80d7c3fca3e8e57aace0813bf69c2d64 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/storage/ttl/StorageTTL.java @@ -23,5 +23,7 @@ import org.apache.skywalking.oap.server.core.analysis.Downsampling; * @author peng-yongsheng */ public interface StorageTTL { - TTLCalculator calculator(Downsampling downsampling); + TTLCalculator metricsCalculator(Downsampling downsampling); + + TTLCalculator recordCalculator(); } diff --git a/oap-server/server-receiver-plugin/jaeger-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/jaeger/JaegerGRPCHandler.java b/oap-server/server-receiver-plugin/jaeger-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/jaeger/JaegerGRPCHandler.java index cfcb26d92660738844a1d166ff5085aa6226575b..defcf4ed22e766e768e8cc021a5ad0db5b6cd362 100644 --- a/oap-server/server-receiver-plugin/jaeger-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/jaeger/JaegerGRPCHandler.java +++ b/oap-server/server-receiver-plugin/jaeger-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/jaeger/JaegerGRPCHandler.java @@ -99,7 +99,7 @@ public class JaegerGRPCHandler extends CollectorServiceGrpc.CollectorServiceImpl long duration = span.getDuration().getNanos() / 1_000_000; jaegerSpan.setStartTime(Instant.ofEpochSecond(span.getStartTime().getSeconds(), span.getStartTime().getNanos()).toEpochMilli()); - long timeBucket = TimeBucket.getSecondTimeBucket(jaegerSpan.getStartTime()); + long timeBucket = TimeBucket.getRecordTimeBucket(jaegerSpan.getStartTime()); jaegerSpan.setTimeBucket(timeBucket); jaegerSpan.setEndTime(jaegerSpan.getStartTime() + duration); jaegerSpan.setLatency((int)duration); diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/endpoint/MultiScopesSpanListener.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/endpoint/MultiScopesSpanListener.java index 20f3b9b7e82e11bb33d19beff3c9ecb2b3140c57..7459abd1fa97ab56a3ab2f94408f691903443b81 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/endpoint/MultiScopesSpanListener.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/endpoint/MultiScopesSpanListener.java @@ -157,7 +157,7 @@ public class MultiScopesSpanListener implements EntrySpanListener, ExitSpanListe statement.setId(segmentCoreInfo.getSegmentId() + "-" + spanDecorator.getSpanId()); statement.setDatabaseServiceId(sourceBuilder.getDestServiceId()); statement.setLatency(sourceBuilder.getLatency()); - statement.setTimeBucket(TimeBucket.getSecondTimeBucket(segmentCoreInfo.getStartTime())); + statement.setTimeBucket(TimeBucket.getRecordTimeBucket(segmentCoreInfo.getStartTime())); statement.setTraceId(traceId); for (KeyStringValuePair tag : spanDecorator.getAllTags()) { if (SpanTags.DB_STATEMENT.equals(tag.getKey())) { diff --git a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/segment/SegmentSpanListener.java b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/segment/SegmentSpanListener.java index 1b637ceb3a084eec3bd998e515d25269b759a5dd..48c261827431368ac2391644cb32bf51b59e5e97 100644 --- a/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/segment/SegmentSpanListener.java +++ b/oap-server/server-receiver-plugin/skywalking-trace-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/trace/provider/parser/listener/segment/SegmentSpanListener.java @@ -68,7 +68,7 @@ public class SegmentSpanListener implements FirstSpanListener, EntrySpanListener return; } - long timeBucket = TimeBucket.getSecondTimeBucket(segmentCoreInfo.getStartTime()); + long timeBucket = TimeBucket.getRecordTimeBucket(segmentCoreInfo.getStartTime()); segment.setSegmentId(segmentCoreInfo.getSegmentId()); segment.setServiceId(segmentCoreInfo.getServiceId()); diff --git a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/trace/SpanForward.java b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/trace/SpanForward.java index a05e005b2348ae8ccfd815127d37d5e0ae6e79e3..84353e8c6a94f7a9ac4713bd15e933e09757ad93 100644 --- a/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/trace/SpanForward.java +++ b/oap-server/server-receiver-plugin/zipkin-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/zipkin/trace/SpanForward.java @@ -93,7 +93,7 @@ public class SpanForward { long startTime = span.timestampAsLong() / 1000; zipkinSpan.setStartTime(startTime); if (startTime != 0) { - long timeBucket = TimeBucket.getSecondTimeBucket(zipkinSpan.getStartTime()); + long timeBucket = TimeBucket.getRecordTimeBucket(zipkinSpan.getStartTime()); zipkinSpan.setTimeBucket(timeBucket); } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java index 1299cc6732d6d90ab117e14b043e55a050d2abb7..e780e1fe1c06eefbd49bc9d4f6123d78fff7de3c 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/base/HistoryDeleteEsDAO.java @@ -25,6 +25,7 @@ import org.apache.skywalking.oap.server.core.config.ConfigService; import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.core.storage.ttl.StorageTTL; +import org.apache.skywalking.oap.server.core.storage.ttl.TTLCalculator; import org.apache.skywalking.oap.server.library.client.elasticsearch.ElasticSearchClient; import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; import org.joda.time.DateTime; @@ -51,7 +52,13 @@ public class HistoryDeleteEsDAO extends EsDAO implements IHistoryDeleteDAO { ConfigService configService = moduleDefineHolder.find(CoreModule.NAME).provider().getService(ConfigService.class); ElasticSearchClient client = getClient(); - long timeBefore = storageTTL.calculator(model.getDownsampling()).timeBefore(new DateTime(), configService.getDataTTLConfig()); + TTLCalculator ttlCalculator; + if (model.isRecord()) { + ttlCalculator = storageTTL.recordCalculator(); + } else { + ttlCalculator = storageTTL.metricsCalculator(model.getDownsampling()); + } + long timeBefore = ttlCalculator.timeBefore(new DateTime(), configService.getDataTTLConfig()); if (model.isCapableOfTimeSeries()) { List indexes = client.retrievalIndexByAliases(model.getName()); diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/ElasticsearchStorageTTL.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/ElasticsearchStorageTTL.java index 5779de42e31870e49b0ca9cea4886eb65b0cc975..46af52d8dd56c6f3079f9dad25fb90a74a91b3f7 100644 --- a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/ElasticsearchStorageTTL.java +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/ElasticsearchStorageTTL.java @@ -25,7 +25,7 @@ import org.apache.skywalking.oap.server.core.storage.ttl.*; */ public class ElasticsearchStorageTTL implements StorageTTL { - @Override public TTLCalculator calculator(Downsampling downsampling) { + @Override public TTLCalculator metricsCalculator(Downsampling downsampling) { switch (downsampling) { case Month: return new MonthTTLCalculator(); @@ -37,4 +37,8 @@ public class ElasticsearchStorageTTL implements StorageTTL { return new DayTTLCalculator(); } } + + @Override public TTLCalculator recordCalculator() { + return new EsRecordTTLCalculator(); + } } diff --git a/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/EsRecordTTLCalculator.java b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/EsRecordTTLCalculator.java new file mode 100644 index 0000000000000000000000000000000000000000..19ce59d8362e397e8258abd1b72eba81cbdeefc1 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-elasticsearch-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/elasticsearch/ttl/EsRecordTTLCalculator.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.elasticsearch.ttl; + +import org.apache.skywalking.oap.server.core.DataTTLConfig; +import org.apache.skywalking.oap.server.core.storage.ttl.TTLCalculator; +import org.joda.time.DateTime; + +/** + * Calculate TTL for record. + * + * @author wusheng + */ +public class EsRecordTTLCalculator implements TTLCalculator { + + @Override public long timeBefore(DateTime currentTime, DataTTLConfig dataTTLConfig) { + return Long.valueOf(currentTime.plusDays(0 - dataTTLConfig.getRecordDataTTL()).toString("yyyyMMdd")); + } +} \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java index 69fcd40f84331b2651842fd91d68bf3cce004603..f709566dcc11730413905d39a091d9ace0e6be38 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/h2/dao/H2HistoryDeleteDAO.java @@ -19,12 +19,14 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao; import java.io.IOException; -import java.sql.*; +import java.sql.Connection; +import java.sql.SQLException; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.config.ConfigService; import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; import org.apache.skywalking.oap.server.core.storage.model.Model; import org.apache.skywalking.oap.server.core.storage.ttl.StorageTTL; +import org.apache.skywalking.oap.server.core.storage.ttl.TTLCalculator; import org.apache.skywalking.oap.server.library.client.jdbc.JDBCClientException; import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; @@ -53,7 +55,13 @@ public class H2HistoryDeleteDAO implements IHistoryDeleteDAO { SQLBuilder dataDeleteSQL = new SQLBuilder("delete from " + model.getName() + " where ").append(timeBucketColumnName).append("<= ?"); try (Connection connection = client.getConnection()) { - long timeBefore = storageTTL.calculator(model.getDownsampling()).timeBefore(new DateTime(), configService.getDataTTLConfig()); + TTLCalculator ttlCalculator; + if (model.isRecord()) { + ttlCalculator = storageTTL.recordCalculator(); + } else { + ttlCalculator = storageTTL.metricsCalculator(model.getDownsampling()); + } + long timeBefore = ttlCalculator.timeBefore(new DateTime(), configService.getDataTTLConfig()); client.execute(connection, dataDeleteSQL.toString(), timeBefore); } catch (JDBCClientException | SQLException e) { throw new IOException(e.getMessage(), e);