Allow multiple definitions as fallback in `k8sServiceNameRule`. (#7006)

CHANGES.md

@@ -45,7 +45,7 @@ Release Notes.
 * Support analyzing Envoy TCP access logs and persist error TCP logs.
 * Fix: Envoy error logs are not persisted when no metrics are generated
 * Fix: Memory leakage of low version etcd client. [fix-issue](https://github.com/jurmous/etcd4j/pull/185)
-* Allow multiple definitions as fallback in metadata-service-mapping.yaml file.
+* Allow multiple definitions as fallback in metadata-service-mapping.yaml file and `k8sServiceNameRule`.
 * Fix: NPE when configmap has no data.
 * Fix: Dynamic Configuration key `slowTraceSegmentThreshold` not work
 * Fix: `!=` is not supported in oal when parameters are numbers.

oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/ServiceMetaInfoFactoryImpl.java

@@ -23,6 +23,8 @@ import org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo;
 import org.apache.skywalking.oap.server.receiver.envoy.als.mx.ServiceMetaInfoAdapter;
 
 public class ServiceMetaInfoFactoryImpl implements ServiceMetaInfoFactory {
+    private static final ServiceMetaInfo UNKNOWN = new ServiceMetaInfo("UNKNOWN", "UNKNOWN");
+
     @Override
     public Class<? extends ServiceMetaInfo> clazz() {
         return ServiceMetaInfo.class;
@@ -30,7 +32,7 @@ public class ServiceMetaInfoFactoryImpl implements ServiceMetaInfoFactory {
 
     @Override
     public ServiceMetaInfo unknown() {
-        return new ServiceMetaInfo("UNKNOWN", "UNKNOWN");
+        return UNKNOWN;
     }
 
     @Override

oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/ServiceMetaInfo.java

@@ -54,6 +54,4 @@ public class ServiceMetaInfo {
 
         private final String value;
     }
-
-    public static final ServiceMetaInfo UNKNOWN = new ServiceMetaInfo("UNKNOWN", "UNKNOWN");
 }

oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/k8s/K8SServiceRegistry.java

@@ -64,7 +64,11 @@ public class K8SServiceRegistry {
 
     protected final ServiceNameFormatter serviceNameFormatter;
 
+    private final EnvoyMetricReceiverConfig config;
+
     public K8SServiceRegistry(final EnvoyMetricReceiverConfig config) {
+        this.config = config;
+
         serviceNameFormatter = new ServiceNameFormatter(config.getK8sServiceNameRule());
         ipServiceMetaInfoMap = new ConcurrentHashMap<>();
         idServiceMap = new ConcurrentHashMap<>();
@@ -267,7 +271,7 @@ public class K8SServiceRegistry {
         final ServiceMetaInfo service = ipServiceMetaInfoMap.get(ip);
         if (isNull(service)) {
             log.debug("Unknown ip {}, ip -> service is null", ip);
-            return ServiceMetaInfo.UNKNOWN;
+            return config.serviceMetaInfoFactory().unknown();
         }
         return service;
     }
@@ -297,7 +301,7 @@ public class K8SServiceRegistry {
                     final V1ObjectMeta serviceMetadata = service.getMetadata();
                     if (isNull(serviceMetadata)) {
                         log.warn("Service metadata is null, {}", service);
-                        return ServiceMetaInfo.UNKNOWN;
+                        return config.serviceMetaInfoFactory().unknown();
                     }
                     serviceMetaInfo.setServiceName(serviceMetadata.getName());
                 }

oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/k8s/K8sALSServiceMeshHTTPAnalysis.java

@@ -30,6 +30,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.apm.network.servicemesh.v3.ServiceMeshMetric;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.receiver.envoy.EnvoyMetricReceiverConfig;
+import org.apache.skywalking.oap.server.receiver.envoy.ServiceMetaInfoFactory;
 import org.apache.skywalking.oap.server.receiver.envoy.als.AbstractALSAnalyzer;
 import org.apache.skywalking.oap.server.receiver.envoy.als.Role;
 import org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo;
@@ -46,6 +47,8 @@ import static org.apache.skywalking.oap.server.receiver.envoy.als.k8s.Addresses.
 public class K8sALSServiceMeshHTTPAnalysis extends AbstractALSAnalyzer {
     protected K8SServiceRegistry serviceRegistry;
 
+    protected EnvoyMetricReceiverConfig config;
+
     @Override
     public String name() {
         return "k8s-mesh";
@@ -54,6 +57,7 @@ public class K8sALSServiceMeshHTTPAnalysis extends AbstractALSAnalyzer {
     @Override
     @SneakyThrows
     public void init(ModuleManager manager, EnvoyMetricReceiverConfig config) {
+        this.config = config;
         serviceRegistry = new K8SServiceRegistry(config);
         serviceRegistry.start();
     }
@@ -107,7 +111,7 @@ public class K8sALSServiceMeshHTTPAnalysis extends AbstractALSAnalyzer {
         if (cluster.startsWith("inbound|")) {
             // Server side
             final ServiceMeshMetric.Builder metrics;
-            if (downstreamService.equals(ServiceMetaInfo.UNKNOWN)) {
+            if (downstreamService.equals(config.serviceMetaInfoFactory().unknown())) {
                 // Ingress -> sidecar(server side)
                 // Mesh telemetry without source, the relation would be generated.
                 metrics = newAdapter(entry, null, localService).adaptToDownstreamMetrics();
@@ -178,7 +182,7 @@ public class K8sALSServiceMeshHTTPAnalysis extends AbstractALSAnalyzer {
     }
 
     /**
-     * @return found service info, or {@link ServiceMetaInfo#UNKNOWN} to represent not found.
+     * @return found service info, or {@link ServiceMetaInfoFactory#unknown()} to represent not found.
      */
     protected ServiceMetaInfo find(String ip) {
         return serviceRegistry.findService(ip);

oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/k8s/ServiceNameFormatter.java

@@ -22,8 +22,10 @@ import com.google.common.base.Strings;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
+import java.util.Objects;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Stream;
 import org.apache.commons.beanutils.PropertyUtils;
 import org.apache.commons.lang3.StringUtils;
 
@@ -34,7 +36,7 @@ public class ServiceNameFormatter {
     private final StringBuffer serviceNamePattern;
 
     public ServiceNameFormatter(String rule) {
-        rule = StringUtils.defaultIfBlank(rule, "${pod.metadata.labels.(service.istio.io/canonical-name)}");
+        rule = StringUtils.defaultIfBlank(rule, "${pod.metadata.labels.(service.istio.io/canonical-name),pod.metadata.labels.(app.kubernetes.io/name),pod.metadata.labels.app)}");
 
         this.properties = new ArrayList<>();
         this.serviceNamePattern = new StringBuffer();
@@ -52,7 +54,18 @@ public class ServiceNameFormatter {
         final Object[] values = new Object[properties.size()];
 
         for (int i = 0; i < properties.size(); i++) {
-            final Object value = PropertyUtils.getProperty(context, properties.get(i));
+            final String property = properties.get(i);
+            final Object value = Stream.of(property.split(","))
+                                       .map(it -> {
+                                           try {
+                                               return PropertyUtils.getProperty(context, it);
+                                           } catch (Exception e) {
+                                               return null;
+                                           }
+                                       })
+                                       .filter(it -> Objects.nonNull(it) && !Strings.isNullOrEmpty(it.toString()))
+                                       .findFirst()
+                                       .orElse("-");
             values[i] = value;
         }
 

oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/mx/MetaExchangeALSHTTPAnalyzer.java

@@ -39,7 +39,6 @@ import org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo;
 
 import static org.apache.skywalking.oap.server.library.util.CollectionUtils.isNotEmpty;
 import static org.apache.skywalking.oap.server.receiver.envoy.als.LogEntry2MetricsAdapter.NON_TLS;
-import static org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo.UNKNOWN;
 
 @Slf4j
 public class MetaExchangeALSHTTPAnalyzer extends AbstractALSAnalyzer {
@@ -126,7 +125,7 @@ public class MetaExchangeALSHTTPAnalyzer extends AbstractALSAnalyzer {
             }
         });
         if (role.equals(Role.PROXY) && !downstreamExists.get()) {
-            final ServiceMeshMetric.Builder metric = newAdapter(entry, UNKNOWN, currSvc).adaptToDownstreamMetrics();
+            final ServiceMeshMetric.Builder metric = newAdapter(entry, config.serviceMetaInfoFactory().unknown(), currSvc).adaptToDownstreamMetrics();
             if (log.isDebugEnabled()) {
                 log.debug("Transformed a {} inbound mesh metric {}", role, TextFormat.shortDebugString(metric));
             }

oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/tcp/k8s/K8sALSServiceMeshTCPAnalysis.java

@@ -30,6 +30,7 @@ import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.apm.network.servicemesh.v3.ServiceMeshMetric;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.receiver.envoy.EnvoyMetricReceiverConfig;
+import org.apache.skywalking.oap.server.receiver.envoy.ServiceMetaInfoFactory;
 import org.apache.skywalking.oap.server.receiver.envoy.als.Role;
 import org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo;
 import org.apache.skywalking.oap.server.receiver.envoy.als.k8s.K8SServiceRegistry;
@@ -45,6 +46,8 @@ import static org.apache.skywalking.oap.server.receiver.envoy.als.LogEntry2Metri
 public class K8sALSServiceMeshTCPAnalysis extends AbstractTCPAccessLogAnalyzer {
     protected K8SServiceRegistry serviceRegistry;
 
+    private EnvoyMetricReceiverConfig config;
+
     @Override
     public String name() {
         return "k8s-mesh";
@@ -53,6 +56,7 @@ public class K8sALSServiceMeshTCPAnalysis extends AbstractTCPAccessLogAnalyzer {
     @Override
     @SneakyThrows
     public void init(ModuleManager manager, EnvoyMetricReceiverConfig config) {
+        this.config = config;
         serviceRegistry = new K8SServiceRegistry(config);
         serviceRegistry.start();
     }
@@ -103,7 +107,7 @@ public class K8sALSServiceMeshTCPAnalysis extends AbstractTCPAccessLogAnalyzer {
         if (cluster.startsWith("inbound|")) {
             // Server side
             final ServiceMeshMetric.Builder metrics;
-            if (downstreamService.equals(ServiceMetaInfo.UNKNOWN)) {
+            if (downstreamService.equals(config.serviceMetaInfoFactory().unknown())) {
                 // Ingress -> sidecar(server side)
                 // Mesh telemetry without source, the relation would be generated.
                 metrics = newAdapter(entry, null, localService).adaptToDownstreamMetrics();
@@ -171,7 +175,7 @@ public class K8sALSServiceMeshTCPAnalysis extends AbstractTCPAccessLogAnalyzer {
     }
 
     /**
-     * @return found service info, or {@link ServiceMetaInfo#UNKNOWN} to represent not found.
+     * @return found service info, or {@link ServiceMetaInfoFactory#unknown()} to represent not found.
      */
     protected ServiceMetaInfo find(String ip) {
         return serviceRegistry.findService(ip);

oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/tcp/mx/MetaExchangeTCPAccessLogAnalyzer.java

@@ -41,7 +41,6 @@ import org.apache.skywalking.oap.server.receiver.envoy.als.tcp.AbstractTCPAccess
 
 import static org.apache.skywalking.oap.server.library.util.CollectionUtils.isNotEmpty;
 import static org.apache.skywalking.oap.server.receiver.envoy.als.LogEntry2MetricsAdapter.NON_TLS;
-import static org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo.UNKNOWN;
 import static org.apache.skywalking.oap.server.receiver.envoy.als.mx.MetaExchangeALSHTTPAnalyzer.DOWNSTREAM_KEY;
 import static org.apache.skywalking.oap.server.receiver.envoy.als.mx.MetaExchangeALSHTTPAnalyzer.UPSTREAM_KEY;
 
@@ -125,7 +124,7 @@ public class MetaExchangeTCPAccessLogAnalyzer extends AbstractTCPAccessLogAnalyz
             }
         });
         if (role.equals(Role.PROXY) && !downstreamExists.get()) {
-            final ServiceMeshMetric.Builder metric = newAdapter(entry, UNKNOWN, currSvc).adaptToDownstreamMetrics();
+            final ServiceMeshMetric.Builder metric = newAdapter(entry, config.serviceMetaInfoFactory().unknown(), currSvc).adaptToDownstreamMetrics();
             if (log.isDebugEnabled()) {
                 log.debug("Transformed a {} inbound mesh metric {}", role, TextFormat.shortDebugString(metric));
             }

oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/envoy/als/k8s/K8SALSServiceMeshHTTPAnalysisTest.java

@@ -28,6 +28,8 @@ import org.apache.skywalking.apm.network.servicemesh.v3.ServiceMeshMetric;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
 import org.apache.skywalking.oap.server.receiver.envoy.EnvoyMetricReceiverConfig;
 import org.apache.skywalking.oap.server.receiver.envoy.MetricServiceGRPCHandlerTestMain;
+import org.apache.skywalking.oap.server.receiver.envoy.ServiceMetaInfoFactory;
+import org.apache.skywalking.oap.server.receiver.envoy.ServiceMetaInfoFactoryImpl;
 import org.apache.skywalking.oap.server.receiver.envoy.als.AccessLogAnalyzer;
 import org.apache.skywalking.oap.server.receiver.envoy.als.Role;
 import org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo;
@@ -46,7 +48,12 @@ public class K8SALSServiceMeshHTTPAnalysisTest {
     @Before
     public void setUp() {
         analysis = new MockK8SAnalysis();
-        analysis.init(null, null);
+        analysis.init(null, new EnvoyMetricReceiverConfig() {
+            @Override
+            public ServiceMetaInfoFactory serviceMetaInfoFactory() {
+                return new ServiceMetaInfoFactoryImpl();
+            }
+        });
     }
 
     @Test
@@ -148,8 +155,9 @@ public class K8SALSServiceMeshHTTPAnalysisTest {
 
         @Override
         public void init(ModuleManager manager, EnvoyMetricReceiverConfig config) {
+            super.init(manager, config);
             serviceRegistry = mock(K8SServiceRegistry.class);
-            when(serviceRegistry.findService(anyString())).thenReturn(ServiceMetaInfo.UNKNOWN);
+            when(serviceRegistry.findService(anyString())).thenReturn(config.serviceMetaInfoFactory().unknown());
             when(serviceRegistry.findService("10.44.2.56")).thenReturn(new ServiceMetaInfo("ingress", "ingress-Inst"));
             when(serviceRegistry.findService("10.44.2.54")).thenReturn(new ServiceMetaInfo("productpage", "productpage-Inst"));
             when(serviceRegistry.findService("10.44.6.66")).thenReturn(new ServiceMetaInfo("detail", "detail-Inst"));

oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/envoy/als/k8s/ServiceNameFormatterTest.java

@@ -58,6 +58,16 @@ public class ServiceNameFormatterTest {
                 "${pod.metadata.labels.(service.istio.io/canonical-name)}",
                 ImmutableMap.of("service", service("Clash"), "pod", pod(of("service.istio.io/canonical-name", "ClashX-alpha"))),
                 "ClashX-alpha"
+            ),
+            new Case(
+                "${pod.metadata.labels.NOT_EXISTS}",
+                ImmutableMap.of("service", service("Clash"), "pod", pod(of("service.istio.io/canonical-name", "ClashX-alpha"))),
+                "-"
+            ),
+            new Case(
+                "${pod.metadata.labels.NOT_EXISTS,pod.metadata.labels.(service.istio.io/canonical-name),pod.metadata.labels.app}",
+                ImmutableMap.of("service", service("Clash"), "pod", pod(of("app", "ClashX-alpha"))),
+                "ClashX-alpha"
             )
         };
     }