From 85a993faf5f09a88ba3a640c4e3b4284a5e64b47 Mon Sep 17 00:00:00 2001 From: Zhenxu Ke Date: Thu, 13 May 2021 06:53:05 +0800 Subject: [PATCH] Allow multiple definitions as fallback in metadata-service-mapping.yaml file (#6933) --- CHANGES.md | 1 + .../resources/metadata-service-mapping.yaml | 2 +- .../core/query/MetadataQueryService.java | 3 +- .../receiver/envoy/als/mx/FieldsHelper.java | 84 ++++++++++++------- .../envoy/als/mx/FieldsHelperTest.java | 10 +++ 5 files changed, 69 insertions(+), 31 deletions(-) diff --git a/CHANGES.md b/CHANGES.md index 6091d6408c..e209ae1c55 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -40,6 +40,7 @@ Release Notes. * Support analyzing Envoy TCP access logs and persist error TCP logs. * Fix: Envoy error logs are not persisted when no metrics are generated * Fix: Memory leakage of low version etcd client. [fix-issue](https://github.com/jurmous/etcd4j/pull/185) +* Allow multiple definitions as fallback in metadata-service-mapping.yaml file. #### UI * Add logo for kong plugin. diff --git a/oap-server/server-bootstrap/src/main/resources/metadata-service-mapping.yaml b/oap-server/server-bootstrap/src/main/resources/metadata-service-mapping.yaml index 941ece6d41..7737432db2 100644 --- a/oap-server/server-bootstrap/src/main/resources/metadata-service-mapping.yaml +++ b/oap-server/server-bootstrap/src/main/resources/metadata-service-mapping.yaml @@ -13,5 +13,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -serviceName: ${LABELS."service.istio.io/canonical-name"} +serviceName: ${LABELS."service.istio.io/canonical-name",LABELS."app.kubernetes.io/name",LABELS.app} serviceInstanceName: ${NAME} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetadataQueryService.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetadataQueryService.java index a5f1e8b2f6..7f6d67442f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetadataQueryService.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/query/MetadataQueryService.java @@ -50,11 +50,10 @@ public class MetadataQueryService implements org.apache.skywalking.oap.server.li public List getAllServices(final String group) throws IOException { return getMetadataQueryDAO().getAllServices(group).stream() - .map(service -> { + .peek(service -> { if (service.getGroup() == null) { service.setGroup(Const.EMPTY_STRING); } - return service; }).collect(Collectors.toList()); } diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/mx/FieldsHelper.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/mx/FieldsHelper.java index 04ad51154f..a615284176 100644 --- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/mx/FieldsHelper.java +++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/mx/FieldsHelper.java @@ -30,7 +30,9 @@ import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import lombok.RequiredArgsConstructor; +import lombok.experimental.Delegate; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.skywalking.oap.server.library.module.ModuleStartException; @@ -39,9 +41,6 @@ import org.apache.skywalking.oap.server.receiver.envoy.als.ServiceMetaInfo; import org.yaml.snakeyaml.Yaml; @Slf4j -/** - * FieldsHelper - */ public enum FieldsHelper { SINGLETON; @@ -79,30 +78,33 @@ public enum FieldsHelper { final String serviceMetaInfoFieldName = entry.getKey(); final String flatBuffersFieldName = entry.getValue(); - final Pattern p = Pattern.compile("(\\$\\{(?.+?)})"); + final Pattern p = Pattern.compile("(\\$\\{(?.+?)})"); final Matcher m = p.matcher(flatBuffersFieldName); - final List> flatBuffersFieldNames = new ArrayList<>(m.groupCount()); + final List flatBuffersFieldNames = new ArrayList<>(m.groupCount()); final StringBuffer serviceNamePattern = new StringBuffer(); while (m.find()) { - final String property = m.group("property"); - List tokens = Splitter.on('.').omitEmptyStrings().splitToList(property); - - StringBuilder tokenBuffer = new StringBuilder(); - List compactedTokens = new ArrayList<>(tokens.size()); - for (String token : tokens) { - if (tokenBuffer.length() == 0 && token.startsWith("\"")) { - tokenBuffer.append(token); - } else if (tokenBuffer.length() > 0) { - tokenBuffer.append(".").append(token); - if (token.endsWith("\"")) { - compactedTokens.add(tokenBuffer.toString().replaceAll("\"", "")); - tokenBuffer.setLength(0); + final String properties = m.group("properties"); + final List fields = Splitter.on(',').omitEmptyStrings().splitToList(properties).stream().map(candidate -> { + List tokens = Splitter.on('.').omitEmptyStrings().splitToList(candidate); + + StringBuilder tokenBuffer = new StringBuilder(); + List candidateFields = new ArrayList<>(tokens.size()); + for (String token : tokens) { + if (tokenBuffer.length() == 0 && token.startsWith("\"")) { + tokenBuffer.append(token); + } else if (tokenBuffer.length() > 0) { + tokenBuffer.append(".").append(token); + if (token.endsWith("\"")) { + candidateFields.add(tokenBuffer.toString().replaceAll("\"", "")); + tokenBuffer.setLength(0); + } + } else { + candidateFields.add(token); } - } else { - compactedTokens.add(token); } - } - flatBuffersFieldNames.add(compactedTokens); + return new Field(candidateFields); + }).collect(Collectors.toList()); + flatBuffersFieldNames.add(new Property(fields)); m.appendReplacement(serviceNamePattern, "%s"); } @@ -136,12 +138,18 @@ public enum FieldsHelper { final ServiceNameFormat serviceNameFormat = entry.getValue(); final Object[] values = new String[serviceNameFormat.properties.size()]; for (int i = 0; i < serviceNameFormat.properties.size(); i++) { - final List properties = serviceNameFormat.properties.get(i); - Value value = root; - for (final String property : properties) { - value = value.getStructValue().getFieldsOrDefault(property, empty); + values[i] = "-"; // Give it a default value + final Property property = serviceNameFormat.properties.get(i); + for (final Field field : property) { + Value value = root; + for (final String segment : field.dsvSegments) { + value = value.getStructValue().getFieldsOrDefault(segment, empty); + } + if (Strings.isNullOrEmpty(value.getStringValue()) || "-".equals(value.getStringValue())) { + continue; + } + values[i] = value.getStringValue(); } - values[i] = value.getStringValue(); } final String value = Strings.lenientFormat(serviceNameFormat.format, values); if (!Strings.isNullOrEmpty(value)) { @@ -154,6 +162,26 @@ public enum FieldsHelper { private static class ServiceNameFormat { private final String format; - private final List> properties; + private final List properties; + } + + /** + * A property in the metadata map, it may have multiple candidates, of which the first is non empty will be used. + * For example, to look up the service name, you may set candidates like ${LABELS."service.istio.io/canonical-name",LABELS."app.kubernetes.io/name","app"}. + */ + @RequiredArgsConstructor + private static class Property implements Iterable { + @Delegate + private final List candidateFields; + } + + /** + * A field in the property, it may be nested such as LABELS.app, LABELS.revision, etc. + * {@link #dsvSegments} are the `.` separated segment list, such as ["LABELS", "app"], ["LABELS", "revision"]. + */ + @RequiredArgsConstructor + private static class Field implements Iterable { + @Delegate + private final List dsvSegments; } } diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/envoy/als/mx/FieldsHelperTest.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/envoy/als/mx/FieldsHelperTest.java index 8a38c18f52..448b343512 100644 --- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/envoy/als/mx/FieldsHelperTest.java +++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/envoy/als/mx/FieldsHelperTest.java @@ -69,6 +69,16 @@ public class FieldsHelperTest { "serviceName: fixed-${LABELS.\"service.istio.io/canonical-name\"}\nserviceInstanceName: yeah_${NAME}", "fixed-productpage", "yeah_productpage-v1-65576bb7bf-4mzsp" + }, + { + "serviceName: fixed-${LABELS.\"service.istio.io/not-exist\",LABELS.\"service.istio.io/canonical-name\"}\nserviceInstanceName: yeah_${NAME}", + "fixed-productpage", + "yeah_productpage-v1-65576bb7bf-4mzsp" + }, + { + "serviceName: fixed-${LABELS.\"service.istio.io/not-exist\",LABELS.\"service.istio.io/not-exist-2\"}\nserviceInstanceName: yeah_${NAME}", + "fixed--", + "yeah_productpage-v1-65576bb7bf-4mzsp" } }); } -- GitLab