diff --git a/CHANGES.md b/CHANGES.md index cf6a9ae699792e8b94e6c348a1175759b9eef5e5..bd548101edcdb230ce1177c0be068e77ffb3b088 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -43,6 +43,9 @@ Release Notes. * Fix ElasticSearch implementation of `queryMetricsValues` and `readLabeledMetricsValues` doesn't fill default values when no available data in the ElasticSearch server. * Fix config yaml data type conversion bug when meets special character like !. +* Optimize metrics of minute dimensionality persistence. The value of metrics, which has declaration of the default + value and current value equals the default value logically, the whole row wouldn't be pushed into database. +* Fix `max` function in OAL doesn't support negative long. #### UI @@ -62,9 +65,9 @@ Release Notes. #### Documentation * Enhance documents about the data report and query protocols. -* Restructure documents about receivers and fetchers. - 1. Remove general receiver and fetcher docs - 2. Add more specific menu with docs to help users to find documents easier. +* Restructure documents about receivers and fetchers. + 1. Remove general receiver and fetcher docs + 2. Add more specific menu with docs to help users to find documents easier. All issues and pull requests are [here](https://github.com/apache/skywalking/milestone/101?closed=1) diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java index dc2dec36d304a7c04f8e011618ed76bb1ecbe8f7..85f128d46368063f5d154aa6f8952f6896ceaf02 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/meter/function/avg/AvgFunction.java @@ -213,4 +213,14 @@ public abstract class AvgFunction extends Metrics implements AcceptableValue { private final boolean supportUpdate; private long sessionTimeout; private CounterMetrics aggregationCounter; + private CounterMetrics skippedMetricsCounter; /** * The counter for the round of persistent. */ @@ -82,6 +83,11 @@ public class MetricsPersistentWorker extends PersistenceWorker { * @since 8.7.0 TTL settings from {@link org.apache.skywalking.oap.server.core.CoreModuleConfig#getMetricsDataTTL()} */ private int metricsDataTTL; + /** + * @since 8.9.0 The persistence of minute dimensionality metrics will be skipped if the value of the metric is the + * default. + */ + private boolean skipDefaultValueMetric; MetricsPersistentWorker(ModuleDefineHolder moduleDefineHolder, Model model, IMetricsDAO metricsDAO, AbstractWorker nextAlarmWorker, AbstractWorker nextExportWorker, @@ -100,6 +106,7 @@ public class MetricsPersistentWorker extends PersistenceWorker { this.persistentCounter = 0; this.persistentMod = 1; this.metricsDataTTL = metricsDataTTL; + this.skipDefaultValueMetric = true; String name = "METRICS_L2_AGGREGATION"; int size = BulkConsumePool.Creator.recommendMaxSize() / 8; @@ -124,6 +131,11 @@ public class MetricsPersistentWorker extends PersistenceWorker { new MetricsTag.Keys("metricName", "level", "dimensionality"), new MetricsTag.Values(model.getName(), "2", model.getDownsampling().getName()) ); + skippedMetricsCounter = metricsCreator.createCounter( + "metrics_persistence_skipped", "The number of metrics skipped in persistence due to be in default value", + new MetricsTag.Keys("metricName", "dimensionality"), + new MetricsTag.Values(model.getName(), model.getDownsampling().getName()) + ); SESSION_TIMEOUT_OFFSITE_COUNTER++; } @@ -141,6 +153,12 @@ public class MetricsPersistentWorker extends PersistenceWorker { null, null, null, enableDatabaseSession, supportUpdate, storageSessionTimeout, metricsDataTTL ); + + // Skipping default value mechanism only works for minute dimensionality. + // Metrics in hour and day dimensionalities would not apply this as duration is too long, + // applying this could make precision of hour/day metrics to be lost easily. + this.skipDefaultValueMetric = false; + // For a down-sampling metrics, we prolong the session timeout for 4 times, nearly 5 minutes. // And add offset according to worker creation sequence, to avoid context clear overlap, // eventually optimize load of IDs reading. @@ -225,12 +243,22 @@ public class MetricsPersistentWorker extends PersistenceWorker { continue; } cachedMetrics.calculate(); - prepareRequests.add(metricsDAO.prepareBatchUpdate(model, cachedMetrics)); + if (skipDefaultValueMetric && cachedMetrics.haveDefault() && cachedMetrics.isDefaultValue()) { + // Skip metrics in default value + skippedMetricsCounter.inc(); + } else { + prepareRequests.add(metricsDAO.prepareBatchUpdate(model, cachedMetrics)); + } nextWorker(cachedMetrics); cachedMetrics.setLastUpdateTimestamp(timestamp); } else { metrics.calculate(); - prepareRequests.add(metricsDAO.prepareBatchInsert(model, metrics)); + if (skipDefaultValueMetric && metrics.haveDefault() && metrics.isDefaultValue()) { + // Skip metrics in default value + skippedMetricsCounter.inc(); + } else { + prepareRequests.add(metricsDAO.prepareBatchInsert(model, metrics)); + } nextWorker(metrics); metrics.setLastUpdateTimestamp(timestamp); }