Enhance `get` generation mechanism of OAL engine, support map type of source's field. (#7904)

CHANGES.md

@@ -11,6 +11,7 @@ Release Notes.
 * Replace e2e cases to e2e-v2: Lua Nginx/SelfObservability.
 * Upgrade Armeria to 1.12, upgrade OpenSearch test version to 1.1.0.
 * Add component definition for `Apache-Kylin`.
+* Enhance `get` generation mechanism of OAL engine, support map type of source's field.
 
 #### UI
 * Optimize endpoint dependency.

docs/en/concepts-and-designs/oal.md

@@ -133,7 +133,10 @@ endpoint_url_sum = from(Endpoint.*).filter(name in ["/v1", "/v2"]).count()
 endpoint_calls = from(Endpoint.*).count()
 
 // Calculate the CPM with the GET method for each service.The value is made up with `tagKey:tagValue`.
+// Option 1, use `tags contain`.
 service_cpm_http_get = from(Service.*).filter(tags contain "http.method:GET").cpm()
+// Option 2, use `tag[key]`.
+service_cpm_http_get = from(Service.*).filter(tag["http.method"] == "GET").cpm();
 
 // Calculate the CPM with the HTTP method except for the GET method for each service.The value is made up with `tagKey:tagValue`.
 service_cpm_http_other = from(Service.*).filter(tags not contain "http.method:GET").cpm()

docs/en/concepts-and-designs/scope-definitions.md

@@ -15,6 +15,7 @@ Using the Aggregation Function, the requests will be grouped by time and **Group
 | rpcStatusCode | The string value of the rpc response code. | | string |
 | type | The type of each request, such as Database, HTTP, RPC, or gRPC. | | enum |
 | tags | The labels of each request. Each value is made up by `TagKey:TagValue` in the segment. | | `List<String>` |
+| tag | The key-value pair of span tags in the segment. | | `Map<String, String>` |
 
 ### SCOPE `Service`
 
@@ -33,6 +34,7 @@ This calculates the metrics data from each request of the service.
 | rpcStatusCode | The string value of the rpc response code. | | string |
 | type | The type of each request. Such as: Database, HTTP, RPC, gRPC. | | enum |
 | tags | The labels of each request. Each value is made up by `TagKey:TagValue` in the segment. | | `List<String>` |
+| tag | The key-value pair of span tags in the segment. | | `Map<String, String>` |
 | sideCar.internalErrorCode | The sidecar/gateway proxy internal error code. The value is based on the implementation. | | string|
 | tcpInfo.receivedBytes | The received bytes of the TCP traffic, if this request is a TCP call. | | long |
 | tcpInfo.sentBytes | The sent bytes of the TCP traffic, if this request is a TCP call. | | long |
@@ -54,6 +56,7 @@ This calculates the metrics data from each request of the service instance.
 | rpcStatusCode | The string value of the rpc response code. | | string |
 | type | The type of each request, such as Database, HTTP, RPC, or gRPC. | | enum |
 | tags | The labels of each request. Each value is made up by `TagKey:TagValue` in the segment. | | `List<String>` |
+| tag | The key-value pair of span tags in the segment. | | `Map<String, String>` |
 | sideCar.internalErrorCode | The sidecar/gateway proxy internal error code. The value is based on the implementation. | | string|
 | tcpInfo.receivedBytes | The received bytes of the TCP traffic, if this request is a TCP call. | | long |
 | tcpInfo.sentBytes | The sent bytes of the TCP traffic, if this request is a TCP call. | | long |
@@ -145,6 +148,7 @@ This calculates the metrics data from each request of the endpoint in the servic
 | rpcStatusCode | The string value of the rpc response code. | | string |
 | type | The type of each request, such as Database, HTTP, RPC, or gRPC. | | enum |
 | tags | The labels of each request. Each value is made up by `TagKey:TagValue` in the segment. | | `List<String>` |
+| tag | The key-value pair of span tags in the segment. | | `Map<String, String>` |
 | sideCar.internalErrorCode | The sidecar/gateway proxy internal error code. The value is based on the implementation. | | string|
 | tcpInfo.receivedBytes | The received bytes of the TCP traffic, if this request is a TCP call. | | long |
 | tcpInfo.sentBytes | The sent bytes of the TCP traffic, if this request is a TCP call. | | long |

oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/SourceBuilder.java

@@ -19,7 +19,9 @@
 package org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener;
 
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
+import java.util.Map;
 import lombok.Getter;
 import lombok.RequiredArgsConstructor;
 import lombok.Setter;
@@ -104,6 +106,8 @@ class SourceBuilder {
     private long timeBucket;
     @Getter
     private final List<String> tags = new ArrayList<>();
+    @Getter
+    private final Map<String, String> originalTags = new HashMap<>();
 
     void prepare() {
         this.sourceServiceName = namingControl.formatServiceName(sourceServiceName);
@@ -131,6 +135,7 @@ class SourceBuilder {
         all.setType(type);
         all.setTimeBucket(timeBucket);
         all.setTags(tags);
+        all.setOriginalTags(originalTags);
         return all;
     }
 
@@ -151,6 +156,7 @@ class SourceBuilder {
         service.setType(type);
         service.setTags(tags);
         service.setTimeBucket(timeBucket);
+        service.setOriginalTags(originalTags);
         return service;
     }
 
@@ -195,6 +201,7 @@ class SourceBuilder {
         serviceInstance.setRpcStatusCode(rpcStatusCode);
         serviceInstance.setType(type);
         serviceInstance.setTags(tags);
+        serviceInstance.setOriginalTags(originalTags);
         serviceInstance.setTimeBucket(timeBucket);
         return serviceInstance;
     }
@@ -242,6 +249,7 @@ class SourceBuilder {
         endpoint.setRpcStatusCode(rpcStatusCode);
         endpoint.setType(type);
         endpoint.setTags(tags);
+        endpoint.setOriginalTags(originalTags);
         endpoint.setTimeBucket(timeBucket);
         return endpoint;
     }
@@ -309,5 +317,6 @@ class SourceBuilder {
 
     public void setTag(KeyStringValuePair tag) {
         tags.add(tag.getKey().trim() + ":" + tag.getValue().trim());
+        originalTags.put(tag.getKey(), tag.getValue());
     }
 }

oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4

@@ -157,7 +157,11 @@ conditionAttributeStmt
     ;
 
 conditionAttribute
-    : IDENTIFIER
+    : IDENTIFIER | mapAttribute
+    ;
+
+mapAttribute
+    : IDENTIFIER LS_BRACKET STRING_LITERAL RS_BRACKET
     ;
 
 booleanConditionValue

oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java

@@ -47,7 +47,8 @@ public class DeepAnalysis {
         List<ConditionExpression> expressions = result.getFilterExpressionsParserResult();
         if (expressions != null && expressions.size() > 0) {
             for (ConditionExpression expression : expressions) {
-                final FilterMatchers.MatcherInfo matcherInfo = FilterMatchers.INSTANCE.find(expression.getExpressionType());
+                final FilterMatchers.MatcherInfo matcherInfo = FilterMatchers.INSTANCE.find(
+                    expression.getExpressionType());
 
                 final String getter = matcherInfo.isBooleanType()
                     ? ClassMethodUtil.toIsMethod(expression.getAttributes())
@@ -55,7 +56,7 @@ public class DeepAnalysis {
 
                 final Expression filterExpression = new Expression();
                 filterExpression.setExpressionObject(matcherInfo.getMatcher().getName());
-                filterExpression.setLeft("source." + getter + "()");
+                filterExpression.setLeft("source." + getter);
                 filterExpression.setRight(expression.getValue());
                 result.addFilterExpressions(filterExpression);
             }
@@ -93,15 +94,17 @@ public class DeepAnalysis {
             Annotation annotation = parameterAnnotations[0];
             if (annotation instanceof SourceFrom) {
                 entryMethod.addArg(
-                    parameterType, "source." + ClassMethodUtil.toGetMethod(result.getSourceAttribute()) + "()");
+                    parameterType, "source." + ClassMethodUtil.toGetMethod(result.getSourceAttribute()));
             } else if (annotation instanceof ConstOne) {
                 entryMethod.addArg(parameterType, "1");
             } else if (annotation instanceof org.apache.skywalking.oap.server.core.analysis.metrics.annotation.Expression) {
                 if (isNull(result.getFuncConditionExpressions()) || result.getFuncConditionExpressions().isEmpty()) {
-                    throw new IllegalArgumentException("Entrance method:" + entranceMethod + " argument can't find funcParamExpression.");
+                    throw new IllegalArgumentException(
+                        "Entrance method:" + entranceMethod + " argument can't find funcParamExpression.");
                 } else {
                     ConditionExpression expression = result.getNextFuncConditionExpression();
-                    final FilterMatchers.MatcherInfo matcherInfo = FilterMatchers.INSTANCE.find(expression.getExpressionType());
+                    final FilterMatchers.MatcherInfo matcherInfo = FilterMatchers.INSTANCE.find(
+                        expression.getExpressionType());
 
                     final String getter = matcherInfo.isBooleanType()
                         ? ClassMethodUtil.toIsMethod(expression.getAttributes())
@@ -110,7 +113,7 @@ public class DeepAnalysis {
                     final Expression argExpression = new Expression();
                     argExpression.setRight(expression.getValue());
                     argExpression.setExpressionObject(matcherInfo.getMatcher().getName());
-                    argExpression.setLeft("source." + getter + "()");
+                    argExpression.setLeft("source." + getter);
 
                     entryMethod.addArg(argExpression);
                 }

oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/EntryMethod.java

@@ -41,8 +41,10 @@ public class EntryMethod {
             addArg(parameterType, arg.getType(), arg.getText().get(0));
             return;
         }
-        addArg(parameterType, arg.getType(), parameterType.equals(boolean.class) ? "source." + ClassMethodUtil.toIsMethod(arg
-            .getText()) + "()" : "source." + ClassMethodUtil.toGetMethod(arg.getText()) + "()");
+        addArg(parameterType, arg.getType(), parameterType.equals(boolean.class) ?
+            "source." + ClassMethodUtil.toIsMethod(arg.getText())
+            :
+            "source." + ClassMethodUtil.toGetMethod(arg.getText()));
     }
 
     void addArg(Class<?> parameterType, String expression) {

oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/util/ClassMethodUtil.java

@@ -42,10 +42,10 @@ public class ClassMethodUtil {
             if (i != 0) {
                 method.append(".");
             }
-            if (i != attributes.size() - 1) {
-                method.append(toGetMethod(attributes.get(i))).append("()");
+            if (isMapExpression(attributes.get(i))) {
+                method.append(mapExpression(attributes.get(i)));
             } else {
-                method.append(toGetMethod(attributes.get(i)));
+                method.append(toGetMethod(attributes.get(i))).append("()");
             }
         }
         return method.toString();
@@ -63,9 +63,22 @@ public class ClassMethodUtil {
             if (i != attributes.size() - 1) {
                 method.append(toGetMethod(attributes.get(i))).append("()");
             } else {
-                method.append(toIsMethod(attributes.get(i)));
+                method.append(toIsMethod(attributes.get(i))).append("()");
             }
         }
         return method.toString();
     }
+
+    /**
+     * @return empty if this attribute is not type of map.
+     */
+    private static String mapExpression(String attribute) {
+        final int indexOf = attribute.indexOf("[");
+        return toGetMethod(attribute.substring(0, indexOf))
+            + "(" + attribute.substring(indexOf + 1, attribute.length() - 1) + ")";
+    }
+
+    private static boolean isMapExpression(String attribute) {
+        return attribute.indexOf("[") > 0 && attribute.endsWith("]");
+    }
 }

oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java

@@ -287,6 +287,17 @@ public class ScriptParserTest {
         Assert.assertEquals(1, methodArgsExpressions.size());
     }
 
+    @Test
+    public void testParse11() throws IOException {
+        ScriptParser parser = ScriptParser.createFromScriptText(
+            "GetCallTraffic = from(Service.*).filter(tag[\"http.method\"] == \"get\").cpm();", TEST_SOURCE_PACKAGE);
+        List<AnalysisResult> results = parser.parse().getMetricsStmts();
+        AnalysisResult clientCpm = results.get(0);
+        final List<Expression> filterExpressions = clientCpm.getFilterExpressions();
+        Assert.assertEquals(1, filterExpressions.size());
+        Assert.assertEquals("source.getTag(\"http.method\")", filterExpressions.get(0).getLeft());
+    }
+
     @Test
     public void testDisable() throws IOException {
         ScriptParser parser = ScriptParser.createFromScriptText("disable(segment);", TEST_SOURCE_PACKAGE);

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/All.java

@@ -19,8 +19,10 @@
 package org.apache.skywalking.oap.server.core.source;
 
 import java.util.List;
+import java.util.Map;
 import lombok.Getter;
 import lombok.Setter;
+import org.apache.skywalking.oap.server.core.Const;
 
 import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ALL;
 
@@ -67,4 +69,10 @@ public class All extends Source {
     @Getter
     @Setter
     private List<String> tags;
+    @Setter
+    private Map<String, String> originalTags;
+
+    public String getTag(String key) {
+        return originalTags.getOrDefault(key, Const.EMPTY_STRING);
+    }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java

@@ -19,10 +19,12 @@
 package org.apache.skywalking.oap.server.core.source;
 
 import java.util.List;
+import java.util.Map;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.apm.util.StringUtil;
+import org.apache.skywalking.oap.server.core.Const;
 import org.apache.skywalking.oap.server.core.analysis.IDManager;
 import org.apache.skywalking.oap.server.core.analysis.NodeType;
 
@@ -86,6 +88,8 @@ public class Endpoint extends Source {
     @Getter
     @Setter
     private List<String> tags;
+    @Setter
+    private Map<String, String> originalTags;
     @Getter
     @Setter
     private SideCar sideCar = new SideCar();
@@ -94,4 +98,8 @@ public class Endpoint extends Source {
     public void prepare() {
         serviceId = IDManager.ServiceID.buildId(serviceName, serviceNodeType);
     }
+
+    public String getTag(String key) {
+        return originalTags.getOrDefault(key, Const.EMPTY_STRING);
+    }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java

@@ -18,13 +18,14 @@
 
 package org.apache.skywalking.oap.server.core.source;
 
+import java.util.List;
+import java.util.Map;
 import lombok.Getter;
 import lombok.Setter;
+import org.apache.skywalking.oap.server.core.Const;
 import org.apache.skywalking.oap.server.core.analysis.IDManager;
 import org.apache.skywalking.oap.server.core.analysis.NodeType;
 
-import java.util.List;
-
 import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE;
 import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_CATALOG_NAME;
 
@@ -81,10 +82,16 @@ public class Service extends Source {
     @Getter
     @Setter
     private List<String> tags;
+    @Setter
+    private Map<String, String> originalTags;
     @Getter
     @Setter
     private SideCar sideCar = new SideCar();
     @Getter
     @Setter
     private TCPInfo tcpInfo = new TCPInfo();
+
+    public String getTag(String key) {
+        return originalTags.getOrDefault(key, Const.EMPTY_STRING);
+    }
 }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java

@@ -18,8 +18,10 @@
 
 package org.apache.skywalking.oap.server.core.source;
 
+import java.util.Map;
 import lombok.Getter;
 import lombok.Setter;
+import org.apache.skywalking.oap.server.core.Const;
 import org.apache.skywalking.oap.server.core.analysis.IDManager;
 import org.apache.skywalking.oap.server.core.analysis.NodeType;
 
@@ -84,6 +86,8 @@ public class ServiceInstance extends Source {
     @Getter
     @Setter
     private List<String> tags;
+    @Setter
+    private Map<String, String> originalTags;
     @Getter
     @Setter
     private SideCar sideCar = new SideCar();
@@ -96,4 +100,8 @@ public class ServiceInstance extends Source {
     public void prepare() {
         serviceId = IDManager.ServiceID.buildId(serviceName, nodeType);
     }
+
+    public String getTag(String key) {
+        return originalTags.getOrDefault(key, Const.EMPTY_STRING);
+    }
 }