From 53e42bce5d8f3ead5e5d99ac09b63c048743a4da Mon Sep 17 00:00:00 2001 From: Gao Hongtao Date: Thu, 26 Nov 2020 14:38:10 +0800 Subject: [PATCH] Insert group name into services of so11y and istio relevant metrics (#5897) --- CHANGES.md | 4 ++++ docs/en/setup/backend/backend-fetcher.md | 8 ++++---- .../skywalking/oap/meter/analyzer/MetricConvert.java | 6 +++--- .../oap/meter/analyzer/prometheus/rule/Rule.java | 4 ++-- .../skywalking/oap/meter/analyzer/dsl/FunctionTest.java | 7 +++++++ .../src/main/resources/fetcher-prom-rules/self.yaml | 4 ++-- .../main/resources/otel-oc-rules/istio-controlplane.yaml | 2 ++ .../src/main/resources/otel-oc-rules/oap.yaml | 5 ++--- .../src/main/resources/ui-initialized-templates/istio.yml | 1 + .../ui-initialized-templates/self-observability.yml | 1 + skywalking-ui | 2 +- .../test/java/org/apache/skywalking/e2e/PythonE2E.java | 2 +- .../java/org/apache/skywalking/e2e/kafka/KafkaE2E.java | 4 ++-- .../java/org/apache/skywalking/e2e/meter/MeterE2E.java | 2 +- .../java/org/apache/skywalking/e2e/simple/SimpleE2E.java | 4 ++-- .../org/apache/skywalking/e2e/storage/StorageE2E.java | 4 ++-- .../src/test/resources/expected/simple/so11y-services.yml | 2 +- 17 files changed, 38 insertions(+), 24 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 1918e5a058..e7da55d0aa 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -54,10 +54,14 @@ Release Notes. * Fix "transaction too large error" when use TiDB as storage. * Add otel rules to ui template to observe Istio control plane. * Remove istio mixer +* Insert group name into services of so11y and istio relevant metrics #### UI * Fix incorrect label in radial chart in topology. * Replace node-sass with dart-sass. +* Replace serviceFilter with serviceGroup +* Removed "Les Miserables" from radial chart in topology. +* Add the Promise dropdown option #### Documentation * Add VNode FAQ doc. diff --git a/docs/en/setup/backend/backend-fetcher.md b/docs/en/setup/backend/backend-fetcher.md index 5ff8752caa..95a58188fa 100644 --- a/docs/en/setup/backend/backend-fetcher.md +++ b/docs/en/setup/backend/backend-fetcher.md @@ -46,10 +46,10 @@ staticConfig: # Labels assigned to all metrics fetched from the targets. labels: [ : ... ] -# insert group_name into metric name, for example, meter__ -group: -# default metric level function appends to all expression in this file. -defaultMetricLevel: +# expSuffix is appended to all expression in this file. +expSuffix: +# insert metricPrefix into metric name: _ +metricPrefix: # Metrics rule allow you to recompute queries. metricsRules: [ - ] diff --git a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/MetricConvert.java b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/MetricConvert.java index 4f2a1f0bdf..747d1df8f8 100644 --- a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/MetricConvert.java +++ b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/MetricConvert.java @@ -48,9 +48,9 @@ public class MetricConvert { private final List analyzers; public MetricConvert(Rule rule, MeterSystem service) { - Preconditions.checkState(!Strings.isNullOrEmpty(rule.getGroup())); + Preconditions.checkState(!Strings.isNullOrEmpty(rule.getMetricPrefix())); this.analyzers = rule.getMetricsRules().stream().map(r -> Analyzer.build(formatMetricName(rule, r.getName()), - Strings.isEmpty(rule.getDefaultMetricLevel()) ? r.getExp() : String.format("(%s).%s", r.getExp(), rule.getDefaultMetricLevel()), service)) + Strings.isEmpty(rule.getExpSuffix()) ? r.getExp() : String.format("(%s).%s", r.getExp(), rule.getExpSuffix()), service)) .collect(toList()); } @@ -75,7 +75,7 @@ public class MetricConvert { private String formatMetricName(Rule rule, String meterRuleName) { StringJoiner metricName = new StringJoiner("_"); - metricName.add("meter").add(rule.getGroup()).add(meterRuleName); + metricName.add(rule.getMetricPrefix()).add(meterRuleName); return metricName.toString(); } } diff --git a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/prometheus/rule/Rule.java b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/prometheus/rule/Rule.java index 673d54329f..8fa5a01326 100644 --- a/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/prometheus/rule/Rule.java +++ b/oap-server/analyzer/meter-analyzer/src/main/java/org/apache/skywalking/oap/meter/analyzer/prometheus/rule/Rule.java @@ -33,7 +33,7 @@ public class Rule { private String fetcherTimeout; private String metricsPath; private StaticConfig staticConfig; - private String group; - private String defaultMetricLevel; + private String metricPrefix; + private String expSuffix; private List metricsRules; } diff --git a/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/FunctionTest.java b/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/FunctionTest.java index 876f25a205..3efb906735 100644 --- a/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/FunctionTest.java +++ b/oap-server/analyzer/meter-analyzer/src/test/java/org/apache/skywalking/oap/meter/analyzer/dsl/FunctionTest.java @@ -81,6 +81,13 @@ public class FunctionTest { Result.success(SampleFamily.build(Sample.builder().labels(of("region", "zh")).build())), false, }, + { + "tag-append", + of("instance_cpu_percentage", SampleFamily.build(Sample.builder().labels(of("region", "us")).build())), + "instance_cpu_percentage.tag({tags -> tags.region = 'prefix::' + tags.region})", + Result.success(SampleFamily.build(Sample.builder().labels(of("region", "prefix::us")).build())), + false, + }, { "histogram", of("instance_cpu_percentage", SampleFamily.build( diff --git a/oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml b/oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml index b78b320e50..fff44645b2 100644 --- a/oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml +++ b/oap-server/server-bootstrap/src/main/resources/fetcher-prom-rules/self.yaml @@ -38,8 +38,8 @@ staticConfig: sslCaFilePath: labels: service: oap-server -defaultMetricLevel: instance(['service'], ['instance']) -group: oap +expSuffix: tag({tags -> tags.service = 'oap::' + tags.service}).instance(['service'], ['instance']) +metricPrefix: meter_oap metricsRules: - name: instance_cpu_percentage exp: (process_cpu_seconds_total * 100).sum(['service', 'instance']).rate('PT1M') diff --git a/oap-server/server-bootstrap/src/main/resources/otel-oc-rules/istio-controlplane.yaml b/oap-server/server-bootstrap/src/main/resources/otel-oc-rules/istio-controlplane.yaml index 345e26b7f4..dec2498f92 100644 --- a/oap-server/server-bootstrap/src/main/resources/otel-oc-rules/istio-controlplane.yaml +++ b/oap-server/server-bootstrap/src/main/resources/otel-oc-rules/istio-controlplane.yaml @@ -30,6 +30,8 @@ # group: istio defaultMetricLevel: service(['cluster', 'app']) +expSuffix: tag({tags -> tags.cluster = 'istio-ctrl::' + tags.cluster}).service(['cluster', 'app']) +metricPrefix: meter_istio metricsRules: ## Resource usage # Pilot Versions diff --git a/oap-server/server-bootstrap/src/main/resources/otel-oc-rules/oap.yaml b/oap-server/server-bootstrap/src/main/resources/otel-oc-rules/oap.yaml index 373f766e18..ac89bab83e 100644 --- a/oap-server/server-bootstrap/src/main/resources/otel-oc-rules/oap.yaml +++ b/oap-server/server-bootstrap/src/main/resources/otel-oc-rules/oap.yaml @@ -28,9 +28,8 @@ # "-P6H3M" -- parses as "-6 hours and -3 minutes" # "-P-6H+3M" -- parses as "+6 hours and -3 minutes" # -group: oap -# Self-monitoring defaults to prometheus fetcher. -defaultMetricLevel: instance(['service'], ['host_name']) +expSuffix: tag({tags -> tags.service = 'oap::' + tags.service}).instance(['service'], ['host_name']) +metricPrefix: meter_oap metricsRules: - name: instance_cpu_percentage exp: (process_cpu_seconds_total * 100).sum(['service', 'host_name']).rate('PT1M') diff --git a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio.yml b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio.yml index b21ea90c11..bea98db2ea 100644 --- a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio.yml +++ b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/istio.yml @@ -30,6 +30,7 @@ templates: { "name": "Istio", "type": "service", + "serviceGroup": "istio-ctrl", "children": [ { "name": "Control Plane", diff --git a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/self-observability.yml b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/self-observability.yml index 693019703c..6e3f1d25ef 100644 --- a/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/self-observability.yml +++ b/oap-server/server-bootstrap/src/main/resources/ui-initialized-templates/self-observability.yml @@ -26,6 +26,7 @@ templates: { "name": "SelfObservability", "type": "service", + "serviceGroup": "oap", "children": [ { "name": "oap-server", diff --git a/skywalking-ui b/skywalking-ui index 00fedacc5f..eb7717e9da 160000 --- a/skywalking-ui +++ b/skywalking-ui @@ -1 +1 @@ -Subproject commit 00fedacc5fe842f44952f58719fc6a9659147367 +Subproject commit eb7717e9daf5dd13eb561c4905301017470ec528 diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java index 3efdee5981..a18f292d2c 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/PythonE2E.java @@ -101,7 +101,7 @@ public class PythonE2E extends SkyWalkingTestAdapter { @RetryableTest void services() throws Exception { List services = graphql.services(new ServicesQuery().start(startTime).end(now())); - services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList()); + services = services.stream().filter(s -> !s.getLabel().equals("oap::oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/python/services.yml").as(ServicesMatcher.class).verify(services); diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java index 91a69b2b8a..c1272a294a 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/kafka/KafkaE2E.java @@ -109,7 +109,7 @@ public class KafkaE2E extends SkyWalkingTestAdapter { void services() throws Exception { List services = graphql.services(new ServicesQuery().start(startTime).end(now())); - services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList()); + services = services.stream().filter(s -> !s.getLabel().equals("oap::oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/simple/services.yml").as(ServicesMatcher.class).verify(services); @@ -171,7 +171,7 @@ public class KafkaE2E extends SkyWalkingTestAdapter { void so11y() throws Exception { List services = graphql.services(new ServicesQuery().start(startTime).end(now())); - services = services.stream().filter(s -> s.getLabel().equals("oap-server")).collect(Collectors.toList()); + services = services.stream().filter(s -> s.getLabel().equals("oap::oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/simple/so11y-services.yml").as(ServicesMatcher.class).verify(services); for (final Service service : services) { diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/meter/MeterE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/meter/MeterE2E.java index 828bb52bf9..9d486b3259 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/meter/MeterE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/meter/MeterE2E.java @@ -77,7 +77,7 @@ public class MeterE2E extends SkyWalkingTestAdapter { void meters() throws Exception { List services = graphql.services(new ServicesQuery().start(startTime).end(now())); - services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList()); + services = services.stream().filter(s -> !s.getLabel().equals("oap::oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/meter/services.yml").as(ServicesMatcher.class).verify(services); diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java index 790da62bcc..bc5268b065 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/simple/SimpleE2E.java @@ -141,7 +141,7 @@ public class SimpleE2E extends SkyWalkingTestAdapter { void services() throws Exception { List services = graphql.services(new ServicesQuery().start(startTime).end(now())); - services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList()); + services = services.stream().filter(s -> !s.getLabel().equals("oap::oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/simple/services.yml").as(ServicesMatcher.class).verify(services); @@ -203,7 +203,7 @@ public class SimpleE2E extends SkyWalkingTestAdapter { void so11y() throws Exception { List services = graphql.services(new ServicesQuery().start(startTime).end(now())); - services = services.stream().filter(s -> s.getLabel().equals("oap-server")).collect(Collectors.toList()); + services = services.stream().filter(s -> s.getLabel().equals("oap::oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/simple/so11y-services.yml").as(ServicesMatcher.class).verify(services); for (final Service service : services) { diff --git a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/storage/StorageE2E.java b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/storage/StorageE2E.java index 899dd60c8f..54f70a1aa0 100644 --- a/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/storage/StorageE2E.java +++ b/test/e2e/e2e-test/src/test/java/org/apache/skywalking/e2e/storage/StorageE2E.java @@ -123,7 +123,7 @@ public class StorageE2E extends SkyWalkingTestAdapter { void services() throws Exception { List services = graphql.services(new ServicesQuery().start(startTime).end(now())); - services = services.stream().filter(s -> !s.getLabel().equals("oap-server")).collect(Collectors.toList()); + services = services.stream().filter(s -> !s.getLabel().equals("oap::oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/storage/services.yml").as(ServicesMatcher.class).verify(services); @@ -233,7 +233,7 @@ public class StorageE2E extends SkyWalkingTestAdapter { void so11y() throws Exception { List services = graphql.services(new ServicesQuery().start(startTime).end(now())); - services = services.stream().filter(s -> s.getLabel().equals("oap-server")).collect(Collectors.toList()); + services = services.stream().filter(s -> s.getLabel().equals("oap::oap-server")).collect(Collectors.toList()); LOGGER.info("services: {}", services); load("expected/simple/so11y-services.yml").as(ServicesMatcher.class).verify(services); for (final Service service : services) { diff --git a/test/e2e/e2e-test/src/test/resources/expected/simple/so11y-services.yml b/test/e2e/e2e-test/src/test/resources/expected/simple/so11y-services.yml index c19deb25c6..2cc35e820a 100644 --- a/test/e2e/e2e-test/src/test/resources/expected/simple/so11y-services.yml +++ b/test/e2e/e2e-test/src/test/resources/expected/simple/so11y-services.yml @@ -15,4 +15,4 @@ services: - key: not null - label: "oap-server" + label: "oap::oap-server" -- GitLab