From 9c486a05ee3ad8947011168e0039f5b48922e4ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=99=9F=20Wu=20Sheng?= Date: Tue, 17 Nov 2020 09:03:48 +0800 Subject: [PATCH] Support `sideCar.internalErrorCode` in the sources (#5849) * Support nested sidecar object in the scope. * Update the changelog. * Fix comments and docs. --- CHANGES.md | 2 + apm-protocol/apm-network/src/main/proto | 2 +- .../concepts-and-designs/scope-definitions.md | 5 ++ .../skywalking/oal/rt/grammar/OALParser.g4 | 34 ++++++----- .../oal/rt/parser/AnalysisResult.java | 3 +- .../skywalking/oal/rt/parser/Argument.java | 3 +- .../oal/rt/parser/ConditionExpression.java | 8 ++- .../oal/rt/parser/DeepAnalysis.java | 8 +-- .../skywalking/oal/rt/parser/EntryMethod.java | 3 +- .../skywalking/oal/rt/parser/Expression.java | 2 +- .../skywalking/oal/rt/parser/OALListener.java | 9 +-- .../oal/rt/util/ClassMethodUtil.java | 38 +++++++++++++ .../oal/rt/parser/DeepAnalysisTest.java | 10 ++-- .../oal/rt/parser/ScriptParserTest.java | 45 ++++++++++----- .../oap/server/core/source/Endpoint.java | 3 + .../oap/server/core/source/Service.java | 4 +- .../server/core/source/ServiceInstance.java | 3 + .../core/source/ServiceInstanceRelation.java | 3 + .../server/core/source/ServiceRelation.java | 3 + .../oap/server/core/source/SideCar.java | 35 ++++++++++++ .../envoy/als/AbstractALSAnalyzer.java | 3 +- .../envoy/als/LogEntry2MetricsAdapter.java | 57 ++++++++++++++++++- .../mesh/TelemetryDataDispatcher.java | 10 ++++ .../server/receiver/mesh/MeshDataMock.java | 1 + test/e2e/e2e-protocol/src/main/proto | 2 +- 25 files changed, 243 insertions(+), 53 deletions(-) create mode 100644 oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/SideCar.java diff --git a/CHANGES.md b/CHANGES.md index c8d1cc66d1..b0b3c08365 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -28,6 +28,8 @@ Release Notes. * Add Envoy ALS analyzer based on metadata exchange. * Support keeping collecting the slowly segments in the sampling mechanism. * Support choose files to active the meter analyzer. +* Support nested class definition in the Service, ServiceInstance, Endpoint, ServiceRelation, and ServiceInstanceRelation sources. +* Support `sideCar.internalErrorCode` in the Service, ServiceInstance, Endpoint, ServiceRelation, and ServiceInstanceRelation sources. * Improve Kubernetes service registry for ALS analysis. * Add health checker for cluster management * Improve the queryable tags generation. Remove the duplicated tags to reduce the storage payload. diff --git a/apm-protocol/apm-network/src/main/proto b/apm-protocol/apm-network/src/main/proto index 9933e2d170..5c0fe3a3a6 160000 --- a/apm-protocol/apm-network/src/main/proto +++ b/apm-protocol/apm-network/src/main/proto @@ -1 +1 @@ -Subproject commit 9933e2d17078c2bf07cd1c8d5ef36d52b5cbb917 +Subproject commit 5c0fe3a3a6f2118f86357385616dccd5ebade3c4 diff --git a/docs/en/concepts-and-designs/scope-definitions.md b/docs/en/concepts-and-designs/scope-definitions.md index b340fd8489..ae8207a314 100644 --- a/docs/en/concepts-and-designs/scope-definitions.md +++ b/docs/en/concepts-and-designs/scope-definitions.md @@ -29,6 +29,7 @@ Calculate the metrics data from each request of the service. | responseCode | Represent the response code of HTTP response, if this request is the HTTP call | | int| | type | Represent the type of each request. Such as: Database, HTTP, RPC, gRPC. | | enum | | tags | Represent the labels of each request and each value is made up with the `TagKey:TagValue` in the segment. | | `List` | +| sideCar.internalErrorCode | Represent the sidecar/gateway proxy internal error code, the value bases on the implementation. | | string| ### SCOPE `ServiceInstance` @@ -45,6 +46,7 @@ Calculate the metrics data from each request of the service instance. | responseCode | Represent the response code of HTTP response, if this request is the HTTP call. | | int | | type | Represent the type of each request. Such as: Database, HTTP, RPC, gRPC. | | enum | | tags | Represent the labels of each request and each value is made up with the `TagKey:TagValue` in the segment. | | `List` | +| sideCar.internalErrorCode | Represent the sidecar/gateway proxy internal error code, the value bases on the implementation. | | string| #### Secondary scopes of `ServiceInstance` @@ -117,6 +119,7 @@ Calculate the metrics data from each request of the endpoint in the service. | responseCode | Represent the response code of HTTP response, if this request is the HTTP call. | | int | | type | Represent the type of each request. Such as: Database, HTTP, RPC, gRPC. | | enum | | tags | Represent the labels of each request and each value is made up with the `TagKey:TagValue` in the segment. | | `List` | +| sideCar.internalErrorCode | Represent the sidecar/gateway proxy internal error code, the value bases on the implementation. | | string| ### SCOPE `ServiceRelation` @@ -138,6 +141,7 @@ Calculate the metrics data from each request between one service and the other s | type | Represent the type of each request. Such as: Database, HTTP, RPC, gRPC. | | enum | | detectPoint | Represent where is the relation detected. Values: client, server, proxy. | yes | enum| | tlsMode | Represent TLS mode between source and destination services. For example `service_relation_mtls_cpm = from(ServiceRelation.*).filter(tlsMode == "mTLS").cpm()` || string| +| sideCar.internalErrorCode | Represent the sidecar/gateway proxy internal error code, the value bases on the implementation. | | string| ### SCOPE `ServiceInstanceRelation` @@ -160,6 +164,7 @@ Calculate the metrics data from each request between one service instance and th | type | Represent the type of each request. Such as: Database, HTTP, RPC, gRPC. | | enum | | detectPoint | Represent where is the relation detected. Values: client, server, proxy. | yes | enum| | tlsMode | Represent TLS mode between source and destination service instances. For example, `service_instance_relation_mtls_cpm = from(ServiceInstanceRelation.*).filter(tlsMode == "mTLS").cpm()` || string| +| sideCar.internalErrorCode | Represent the sidecar/gateway proxy internal error code, the value bases on the implementation. | | string| ### SCOPE `EndpointRelation` diff --git a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 index f0871fa792..4fb74ace53 100644 --- a/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 +++ b/oap-server/oal-grammar/src/main/antlr4/org/apache/skywalking/oal/rt/grammar/OALParser.g4 @@ -38,7 +38,7 @@ disableStatement ; metricStatement - : FROM LR_BRACKET source DOT sourceAttribute RR_BRACKET (filterStatement+)? DOT aggregateFunction + : FROM LR_BRACKET source (sourceAttributeStmt+) RR_BRACKET (filterStatement+)? DOT aggregateFunction ; filterStatement @@ -65,6 +65,10 @@ disableSource SRC_PROFILE_TASK | SRC_PROFILE_TASK_LOG | SRC_PROFILE_THREAD_SHANPSHOT ; +sourceAttributeStmt + : DOT sourceAttribute + ; + sourceAttribute : IDENTIFIER | ALL ; @@ -94,57 +98,61 @@ expression ; containMatch - : conditionAttribute CONTAIN stringConditionValue + : conditionAttributeStmt CONTAIN stringConditionValue ; notContainMatch - : conditionAttribute NOT_CONTAIN stringConditionValue + : conditionAttributeStmt NOT_CONTAIN stringConditionValue ; booleanMatch - : conditionAttribute DUALEQUALS booleanConditionValue + : conditionAttributeStmt DUALEQUALS booleanConditionValue ; stringMatch - : conditionAttribute DUALEQUALS (stringConditionValue | enumConditionValue) + : conditionAttributeStmt DUALEQUALS (stringConditionValue | enumConditionValue) ; greaterMatch - : conditionAttribute GREATER numberConditionValue + : conditionAttributeStmt GREATER numberConditionValue ; lessMatch - : conditionAttribute LESS numberConditionValue + : conditionAttributeStmt LESS numberConditionValue ; greaterEqualMatch - : conditionAttribute GREATER_EQUAL numberConditionValue + : conditionAttributeStmt GREATER_EQUAL numberConditionValue ; lessEqualMatch - : conditionAttribute LESS_EQUAL numberConditionValue + : conditionAttributeStmt LESS_EQUAL numberConditionValue ; booleanNotEqualMatch - : conditionAttribute NOT_EQUAL booleanConditionValue + : conditionAttributeStmt NOT_EQUAL booleanConditionValue ; notEqualMatch - : conditionAttribute NOT_EQUAL (numberConditionValue | stringConditionValue | enumConditionValue) + : conditionAttributeStmt NOT_EQUAL (numberConditionValue | stringConditionValue | enumConditionValue) ; likeMatch - : conditionAttribute LIKE stringConditionValue + : conditionAttributeStmt LIKE stringConditionValue ; inMatch - : conditionAttribute IN multiConditionValue + : conditionAttributeStmt IN multiConditionValue ; multiConditionValue : LS_BRACKET (numberConditionValue ((COMMA numberConditionValue)*) | stringConditionValue ((COMMA stringConditionValue)*) | enumConditionValue ((COMMA enumConditionValue)*)) RS_BRACKET ; +conditionAttributeStmt + : conditionAttribute ((DOT conditionAttribute)*) + ; + conditionAttribute : IDENTIFIER ; diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java index 06ef512b16..e954c78dc5 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/AnalysisResult.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oal.rt.parser; +import java.util.ArrayList; import java.util.LinkedList; import java.util.List; import lombok.Getter; @@ -43,7 +44,7 @@ public class AnalysisResult { private int sourceScopeId; - private String sourceAttribute; + private List sourceAttribute = new ArrayList<>(); private String aggregationFunctionName; diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Argument.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Argument.java index 7519706e78..f84f6db2b6 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Argument.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Argument.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oal.rt.parser; +import java.util.List; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -30,5 +31,5 @@ public class Argument { private final int type; - private final String text; + private final List text; } diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java index 8f7bc10152..a9466f4d0b 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/ConditionExpression.java @@ -18,6 +18,8 @@ package org.apache.skywalking.oal.rt.parser; +import java.util.ArrayList; +import java.util.Arrays; import java.util.LinkedList; import java.util.List; import lombok.Getter; @@ -30,13 +32,13 @@ import lombok.Setter; public class ConditionExpression { // original from script private String expressionType; - private String attribute; + private List attributes = new ArrayList<>(); private String value; private List values; - public ConditionExpression(final String expressionType, final String attribute, final String value) { + public ConditionExpression(final String expressionType, final String attributes, final String value) { this.expressionType = expressionType; - this.attribute = attribute; + this.attributes = Arrays.asList(attributes.split("\\.")); this.value = value; } diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java index e968c33003..19380dbd4b 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/DeepAnalysis.java @@ -50,8 +50,8 @@ public class DeepAnalysis { final FilterMatchers.MatcherInfo matcherInfo = FilterMatchers.INSTANCE.find(expression.getExpressionType()); final String getter = matcherInfo.isBooleanType() - ? ClassMethodUtil.toIsMethod(expression.getAttribute()) - : ClassMethodUtil.toGetMethod(expression.getAttribute()); + ? ClassMethodUtil.toIsMethod(expression.getAttributes()) + : ClassMethodUtil.toGetMethod(expression.getAttributes()); final Expression filterExpression = new Expression(); filterExpression.setExpressionObject(matcherInfo.getMatcher().getName()); @@ -104,8 +104,8 @@ public class DeepAnalysis { final FilterMatchers.MatcherInfo matcherInfo = FilterMatchers.INSTANCE.find(expression.getExpressionType()); final String getter = matcherInfo.isBooleanType() - ? ClassMethodUtil.toIsMethod(expression.getAttribute()) - : ClassMethodUtil.toGetMethod(expression.getAttribute()); + ? ClassMethodUtil.toIsMethod(expression.getAttributes()) + : ClassMethodUtil.toGetMethod(expression.getAttributes()); final Expression argExpression = new Expression(); argExpression.setRight(expression.getValue()); diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/EntryMethod.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/EntryMethod.java index 61f5605435..cb0f651767 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/EntryMethod.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/EntryMethod.java @@ -37,7 +37,8 @@ public class EntryMethod { void addArg(Class parameterType, Argument arg) { if (arg.getType() == LITERAL_TYPE) { - addArg(parameterType, arg.getType(), arg.getText()); + // As literal type, there is always one element. + addArg(parameterType, arg.getType(), arg.getText().get(0)); return; } addArg(parameterType, arg.getType(), parameterType.equals(boolean.class) ? "source." + ClassMethodUtil.toIsMethod(arg diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Expression.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Expression.java index b911d342b1..aed15128d4 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Expression.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/Expression.java @@ -22,8 +22,8 @@ import lombok.Getter; import lombok.Setter; @Getter -@Setter public class Expression { + @Setter private String expressionObject; private String left; private String right; diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java index 61d001afb6..ef93b77854 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/parser/OALListener.java @@ -18,6 +18,7 @@ package org.apache.skywalking.oal.rt.parser; +import java.util.Arrays; import java.util.List; import org.antlr.v4.runtime.misc.NotNull; import org.apache.skywalking.oal.rt.grammar.OALParser; @@ -59,7 +60,7 @@ public class OALListener extends OALParserBaseListener { @Override public void enterSourceAttribute(OALParser.SourceAttributeContext ctx) { - current.setSourceAttribute(ctx.getText()); + current.getSourceAttribute().add(ctx.getText()); } @Override @@ -105,7 +106,7 @@ public class OALListener extends OALParserBaseListener { //////////// @Override public void enterConditionAttribute(OALParser.ConditionAttributeContext ctx) { - conditionExpression.setAttribute(ctx.getText()); + conditionExpression.getAttributes().add(ctx.getText()); } @Override @@ -213,10 +214,10 @@ public class OALListener extends OALParserBaseListener { @Override public void enterLiteralExpression(OALParser.LiteralExpressionContext ctx) { if (ctx.IDENTIFIER() == null) { - current.addFuncArg(new Argument(EntryMethod.LITERAL_TYPE, ctx.getText())); + current.addFuncArg(new Argument(EntryMethod.LITERAL_TYPE, Arrays.asList(ctx.getText()))); return; } - current.addFuncArg(new Argument(EntryMethod.IDENTIFIER_TYPE, ctx.getText())); + current.addFuncArg(new Argument(EntryMethod.IDENTIFIER_TYPE, Arrays.asList(ctx.getText().split("\\.")))); } private String metricsNameFormat(String source) { diff --git a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/util/ClassMethodUtil.java b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/util/ClassMethodUtil.java index 4a880a6be5..40573ea455 100644 --- a/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/util/ClassMethodUtil.java +++ b/oap-server/oal-rt/src/main/java/org/apache/skywalking/oal/rt/util/ClassMethodUtil.java @@ -18,6 +18,8 @@ package org.apache.skywalking.oal.rt.util; +import java.util.List; + public class ClassMethodUtil { public static String toGetMethod(String attribute) { return "get" + attribute.substring(0, 1).toUpperCase() + attribute.substring(1); @@ -30,4 +32,40 @@ public class ClassMethodUtil { public static String toIsMethod(String attribute) { return "is" + attribute.substring(0, 1).toUpperCase() + attribute.substring(1); } + + /** + * @return nested get methods. + */ + public static String toGetMethod(List attributes) { + StringBuilder method = new StringBuilder(); + for (int i = 0; i < attributes.size(); i++) { + if (i != 0) { + method.append("."); + } + if (i != attributes.size() - 1) { + method.append(toGetMethod(attributes.get(i))).append("()"); + } else { + method.append(toGetMethod(attributes.get(i))); + } + } + return method.toString(); + } + + /** + * @return nested get/is methods. + */ + public static String toIsMethod(List attributes) { + StringBuilder method = new StringBuilder(); + for (int i = 0; i < attributes.size(); i++) { + if (i != 0) { + method.append("."); + } + if (i != attributes.size() - 1) { + method.append(toGetMethod(attributes.get(i))).append("()"); + } else { + method.append(toIsMethod(attributes.get(i))); + } + } + return method.toString(); + } } diff --git a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/DeepAnalysisTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/DeepAnalysisTest.java index e745ac10d5..1eb6fa524d 100644 --- a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/DeepAnalysisTest.java +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/DeepAnalysisTest.java @@ -53,7 +53,7 @@ public class DeepAnalysisTest { AnalysisResult result = new AnalysisResult(); result.setSourceName("Service"); result.setPackageName("service.serviceavg"); - result.setSourceAttribute("latency"); + result.getSourceAttribute().add("latency"); result.setMetricsName("ServiceAvg"); result.setAggregationFunctionName("longAvg"); @@ -77,7 +77,7 @@ public class DeepAnalysisTest { AnalysisResult result = new AnalysisResult(); result.setSourceName("Endpoint"); result.setPackageName("endpoint.endpointavg"); - result.setSourceAttribute("latency"); + result.getSourceAttribute().add("latency"); result.setMetricsName("EndpointAvg"); result.setAggregationFunctionName("longAvg"); @@ -101,12 +101,12 @@ public class DeepAnalysisTest { AnalysisResult result = new AnalysisResult(); result.setSourceName("Endpoint"); result.setPackageName("endpoint.endpointavg"); - result.setSourceAttribute("latency"); + result.getSourceAttribute().add("latency"); result.setMetricsName("EndpointAvg"); result.setAggregationFunctionName("longAvg"); ConditionExpression expression = new ConditionExpression(); expression.setExpressionType("stringMatch"); - expression.setAttribute("name"); + expression.getAttributes().add("name"); expression.setValue("\"/service/prod/save\""); result.addFilterExpressionsParserResult(expression); @@ -138,7 +138,7 @@ public class DeepAnalysisTest { AnalysisResult result = new AnalysisResult(); result.setSourceName("Endpoint"); result.setPackageName("endpoint.endpointavg"); - result.setSourceAttribute("latency"); + result.getSourceAttribute().add("latency"); result.setMetricsName("EndpointAvg"); result.setAggregationFunctionName("longAvg"); diff --git a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java index c92703852e..1d28d13735 100644 --- a/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java +++ b/oap-server/oal-rt/src/test/java/org/apache/skywalking/oal/rt/parser/ScriptParserTest.java @@ -57,13 +57,13 @@ public class ScriptParserTest { AnalysisResult endpointAvg = results.get(0); Assert.assertEquals("EndpointAvg", endpointAvg.getMetricsName()); Assert.assertEquals("Endpoint", endpointAvg.getSourceName()); - Assert.assertEquals("latency", endpointAvg.getSourceAttribute()); + Assert.assertEquals("[latency]", endpointAvg.getSourceAttribute().toString()); Assert.assertEquals("longAvg", endpointAvg.getAggregationFunctionName()); AnalysisResult serviceAvg = results.get(1); Assert.assertEquals("ServiceAvg", serviceAvg.getMetricsName()); Assert.assertEquals("Service", serviceAvg.getSourceName()); - Assert.assertEquals("latency", serviceAvg.getSourceAttribute()); + Assert.assertEquals("[latency]", serviceAvg.getSourceAttribute().toString()); Assert.assertEquals("longAvg", serviceAvg.getAggregationFunctionName()); } @@ -76,7 +76,7 @@ public class ScriptParserTest { AnalysisResult endpointPercent = results.get(0); Assert.assertEquals("EndpointPercent", endpointPercent.getMetricsName()); Assert.assertEquals("Endpoint", endpointPercent.getSourceName()); - Assert.assertEquals("*", endpointPercent.getSourceAttribute()); + Assert.assertEquals("[*]", endpointPercent.getSourceAttribute().toString()); Assert.assertEquals("percent", endpointPercent.getAggregationFunctionName()); EntryMethod entryMethod = endpointPercent.getEntryMethod(); List methodArgsExpressions = entryMethod.getArgsExpressions(); @@ -94,19 +94,19 @@ public class ScriptParserTest { AnalysisResult endpointPercent = results.get(0); Assert.assertEquals("EndpointPercent", endpointPercent.getMetricsName()); Assert.assertEquals("Endpoint", endpointPercent.getSourceName()); - Assert.assertEquals("*", endpointPercent.getSourceAttribute()); + Assert.assertEquals("[*]", endpointPercent.getSourceAttribute().toString()); Assert.assertEquals("longAvg", endpointPercent.getAggregationFunctionName()); List expressions = endpointPercent.getFilterExpressionsParserResult(); Assert.assertEquals(2, expressions.size()); ConditionExpression booleanMatchExp = expressions.get(0); - Assert.assertEquals("status", booleanMatchExp.getAttribute()); + Assert.assertEquals("[status]", booleanMatchExp.getAttributes().toString()); Assert.assertEquals("true", booleanMatchExp.getValue()); Assert.assertEquals("booleanMatch", booleanMatchExp.getExpressionType()); ConditionExpression stringMatchExp = expressions.get(1); - Assert.assertEquals("name", stringMatchExp.getAttribute()); + Assert.assertEquals("[name]", stringMatchExp.getAttributes().toString()); Assert.assertEquals("\"/product/abc\"", stringMatchExp.getValue()); Assert.assertEquals("stringMatch", stringMatchExp.getExpressionType()); } @@ -125,14 +125,14 @@ public class ScriptParserTest { AnalysisResult responseSummary = results.get(0); Assert.assertEquals("ServiceResponseS1Summary", responseSummary.getMetricsName()); Assert.assertEquals("Service", responseSummary.getSourceName()); - Assert.assertEquals("latency", responseSummary.getSourceAttribute()); + Assert.assertEquals("[latency]", responseSummary.getSourceAttribute().toString()); Assert.assertEquals("sum", responseSummary.getAggregationFunctionName()); List expressions = responseSummary.getFilterExpressionsParserResult(); Assert.assertEquals(1, expressions.size()); ConditionExpression booleanMatchExp = expressions.get(0); - Assert.assertEquals("latency", booleanMatchExp.getAttribute()); + Assert.assertEquals("[latency]", booleanMatchExp.getAttributes().toString()); Assert.assertEquals("1000", booleanMatchExp.getValue()); Assert.assertEquals("greaterMatch", booleanMatchExp.getExpressionType()); @@ -142,7 +142,7 @@ public class ScriptParserTest { Assert.assertEquals(1, expressions.size()); booleanMatchExp = expressions.get(0); - Assert.assertEquals("latency", booleanMatchExp.getAttribute()); + Assert.assertEquals("[latency]", booleanMatchExp.getAttributes().toString()); Assert.assertEquals("2000", booleanMatchExp.getValue()); Assert.assertEquals("lessMatch", booleanMatchExp.getExpressionType()); @@ -152,7 +152,7 @@ public class ScriptParserTest { Assert.assertEquals(1, expressions.size()); booleanMatchExp = expressions.get(0); - Assert.assertEquals("latency", booleanMatchExp.getAttribute()); + Assert.assertEquals("[latency]", booleanMatchExp.getAttributes().toString()); Assert.assertEquals("3000", booleanMatchExp.getValue()); Assert.assertEquals("greaterEqualMatch", booleanMatchExp.getExpressionType()); @@ -162,7 +162,7 @@ public class ScriptParserTest { Assert.assertEquals(1, expressions.size()); booleanMatchExp = expressions.get(0); - Assert.assertEquals("latency", booleanMatchExp.getAttribute()); + Assert.assertEquals("[latency]", booleanMatchExp.getAttributes().toString()); Assert.assertEquals("4000", booleanMatchExp.getValue()); Assert.assertEquals("lessEqualMatch", booleanMatchExp.getExpressionType()); } @@ -180,12 +180,12 @@ public class ScriptParserTest { Assert.assertEquals(2, result.getFuncConditionExpressions().size()); ConditionExpression expression1 = result.getFuncConditionExpressions().get(0); - Assert.assertEquals("param1", expression1.getAttribute()); + Assert.assertEquals("[param1]", expression1.getAttributes().toString()); Assert.assertEquals("booleanMatch", expression1.getExpressionType()); Assert.assertEquals("true", expression1.getValue()); ConditionExpression expression2 = result.getFuncConditionExpressions().get(1); - Assert.assertEquals("param2", expression2.getAttribute()); + Assert.assertEquals("[param2]", expression2.getAttributes().toString()); Assert.assertEquals("booleanMatch", expression2.getExpressionType()); Assert.assertEquals("false", expression2.getValue()); } @@ -230,6 +230,25 @@ public class ScriptParserTest { Assert.assertEquals("new Object[]{1,2,3}", expression.getRight()); } + @Test + public void testParse8() throws IOException { + ScriptParser parser = ScriptParser.createFromScriptText( + "ServicePercent = from(Service.sidecar.internalError).filter(sidecar.internalError == \"abc\").percent(sidecar.internalError != \"\");", TEST_SOURCE_PACKAGE); + List results = parser.parse().getMetricsStmts(); + + AnalysisResult servicePercent = results.get(0); + Assert.assertEquals("ServicePercent", servicePercent.getMetricsName()); + Assert.assertEquals("Service", servicePercent.getSourceName()); + Assert.assertEquals("[sidecar, internalError]", servicePercent.getSourceAttribute().toString()); + final List filterExpressions = servicePercent.getFilterExpressions(); + Assert.assertEquals(1, filterExpressions.size()); + Assert.assertEquals("source.getSidecar().getInternalError()", filterExpressions.get(0).getLeft()); + Assert.assertEquals("percent", servicePercent.getAggregationFunctionName()); + EntryMethod entryMethod = servicePercent.getEntryMethod(); + List methodArgsExpressions = entryMethod.getArgsExpressions(); + Assert.assertEquals(1, methodArgsExpressions.size()); + } + @Test public void testDisable() throws IOException { ScriptParser parser = ScriptParser.createFromScriptText("disable(segment);", TEST_SOURCE_PACKAGE); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java index 5d8babce23..c5c438d267 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java @@ -79,6 +79,9 @@ public class Endpoint extends Source { @Getter @Setter private List tags; + @Getter + @Setter + private SideCar sideCar = new SideCar(); @Override public void prepare() { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java index c720ab7936..110f7fda1c 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Service.java @@ -68,5 +68,7 @@ public class Service extends Source { @Getter @Setter private List tags; - + @Getter + @Setter + private SideCar sideCar = new SideCar(); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java index a428c583f9..d8564e152e 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstance.java @@ -71,6 +71,9 @@ public class ServiceInstance extends Source { @Getter @Setter private List tags; + @Getter + @Setter + private SideCar sideCar = new SideCar(); @Override public void prepare() { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceRelation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceRelation.java index e4f328c55d..9b6ee0c700 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceRelation.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceInstanceRelation.java @@ -106,6 +106,9 @@ public class ServiceInstanceRelation extends Source { @Getter @Setter private String tlsMode; + @Getter + @Setter + private SideCar sideCar = new SideCar(); @Override public void prepare() { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceRelation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceRelation.java index cf9fb618ab..ca19163c52 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceRelation.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceRelation.java @@ -96,6 +96,9 @@ public class ServiceRelation extends Source { @Getter @Setter private String tlsMode; + @Getter + @Setter + private SideCar sideCar = new SideCar(); @Override public void prepare() { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/SideCar.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/SideCar.java new file mode 100644 index 0000000000..3d1710d2b4 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/SideCar.java @@ -0,0 +1,35 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.source; + +import lombok.Getter; +import lombok.Setter; + +/** + * As service mesh is becoming the next generation standard infrastructure for k8s and out-of-k8s env, the sidecar + * source would be an attachment for sources of Service, Instance, Endpoint, and their relationship. + */ +public class SideCar { + /** + * the sidecar/gateway proxy internal error code, the value bases on the implementation. + */ + @Setter + @Getter + private String internalErrorCode = ""; +} diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/AbstractALSAnalyzer.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/AbstractALSAnalyzer.java index fae8cd1d52..41e2a3cb9d 100644 --- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/AbstractALSAnalyzer.java +++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/AbstractALSAnalyzer.java @@ -56,8 +56,7 @@ public abstract class AbstractALSAnalyzer implements ALSHTTPAnalysis { protected LogEntry2MetricsAdapter newAdapter( final HTTPAccessLogEntry entry, final ServiceMetaInfo sourceService, - final ServiceMetaInfo targetService - ) { + final ServiceMetaInfo targetService) { return new LogEntry2MetricsAdapter(entry, sourceService, targetService); } diff --git a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/LogEntry2MetricsAdapter.java b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/LogEntry2MetricsAdapter.java index b3dee9ff37..df3bab8fb7 100644 --- a/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/LogEntry2MetricsAdapter.java +++ b/oap-server/server-receiver-plugin/envoy-metrics-receiver-plugin/src/main/java/org/apache/skywalking/oap/server/receiver/envoy/als/LogEntry2MetricsAdapter.java @@ -25,6 +25,7 @@ import io.envoyproxy.envoy.data.accesslog.v2.AccessLogCommon; import io.envoyproxy.envoy.data.accesslog.v2.HTTPAccessLogEntry; import io.envoyproxy.envoy.data.accesslog.v2.HTTPRequestProperties; import io.envoyproxy.envoy.data.accesslog.v2.HTTPResponseProperties; +import io.envoyproxy.envoy.data.accesslog.v2.ResponseFlags; import io.envoyproxy.envoy.data.accesslog.v2.TLSProperties; import java.time.Instant; import java.util.Optional; @@ -95,10 +96,13 @@ public class LogEntry2MetricsAdapter { protected ServiceMeshMetric.Builder adaptCommonPart() { final AccessLogCommon properties = entry.getCommonProperties(); final String endpoint = endpoint(); - final int responseCode = ofNullable(entry.getResponse()).map(HTTPResponseProperties::getResponseCode).map(UInt32Value::getValue).orElse(200); + final int responseCode = ofNullable(entry.getResponse()).map(HTTPResponseProperties::getResponseCode) + .map(UInt32Value::getValue) + .orElse(200); final boolean status = responseCode >= 200 && responseCode < 400; final Protocol protocol = requestProtocol(entry.getRequest()); final String tlsMode = parseTLS(properties.getTlsProperties()); + final String internalErrorCode = parseInternalErrorCode(properties.getResponseFlags()); final ServiceMeshMetric.Builder builder = ServiceMeshMetric.newBuilder() @@ -106,7 +110,8 @@ public class LogEntry2MetricsAdapter { .setResponseCode(Math.toIntExact(responseCode)) .setStatus(status) .setProtocol(protocol) - .setTlsMode(tlsMode); + .setTlsMode(tlsMode) + .setInternalErrorCode(internalErrorCode); Optional.ofNullable(sourceService) .map(ServiceMetaInfo::getServiceName) @@ -164,4 +169,52 @@ public class LogEntry2MetricsAdapter { return M_TLS; } + /** + * Refer to https://www.envoyproxy.io/docs/envoy/latest/api-v2/data/accesslog/v2/accesslog.proto#data-accesslog-v2-responseflags + * + * @param responseFlags in the ALS v2 + * @return empty string if no internal error code, or literal string representing the code. + */ + protected static String parseInternalErrorCode(final ResponseFlags responseFlags) { + if (responseFlags != null) { + if (responseFlags.getFailedLocalHealthcheck()) { + return "failed_local_healthcheck"; + } else if (responseFlags.getNoHealthyUpstream()) { + return "no_healthy_upstream"; + } else if (responseFlags.getUpstreamRequestTimeout()) { + return "upstream_request_timeout"; + } else if (responseFlags.getLocalReset()) { + return "local_reset"; + } else if (responseFlags.getUpstreamConnectionFailure()) { + return "upstream_connection_failure"; + } else if (responseFlags.getUpstreamConnectionTermination()) { + return "upstream_connection_termination"; + } else if (responseFlags.getUpstreamOverflow()) { + return "upstream_overflow"; + } else if (responseFlags.getNoRouteFound()) { + return "no_route_found"; + } else if (responseFlags.getDelayInjected()) { + return "delay_injected"; + } else if (responseFlags.getFaultInjected()) { + return "fault_injected"; + } else if (responseFlags.getRateLimited()) { + return "rate_limited"; + } else if (responseFlags.getUnauthorizedDetails() != null) { + return "unauthorized_details"; + } else if (responseFlags.getRateLimitServiceError()) { + return "rate_limit_service_error"; + } else if (responseFlags.getDownstreamConnectionTermination()) { + return "downstream_connection_termination"; + } else if (responseFlags.getUpstreamRetryLimitExceeded()) { + return "upstream_retry_limit_exceeded"; + } else if (responseFlags.getStreamIdleTimeout()) { + return "stream_idle_timeout"; + } else if (responseFlags.getInvalidEnvoyRequestHeaders()) { + return "invalid_envoy_request_headers"; + } else if (responseFlags.getDownstreamProtocolError()) { + return "downstream_protocol_error"; + } + } + return ""; + } } diff --git a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/TelemetryDataDispatcher.java b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/TelemetryDataDispatcher.java index 890c79711e..06356554ef 100644 --- a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/TelemetryDataDispatcher.java +++ b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/main/java/org/apache/skywalking/aop/server/receiver/mesh/TelemetryDataDispatcher.java @@ -22,6 +22,7 @@ import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.network.servicemesh.v3.Protocol; import org.apache.skywalking.apm.network.servicemesh.v3.ServiceMeshMetric; import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.core.Const; import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.IDManager; import org.apache.skywalking.oap.server.core.analysis.NodeType; @@ -88,6 +89,10 @@ public class TelemetryDataDispatcher { if (data.getEndpoint() != null) { data.setEndpoint(NAME_LENGTH_CONTROL.formatEndpointName(data.getDestServiceName(), data.getEndpoint())); } + if (data.getInternalErrorCode() == null) { + // Add this since 8.2.0, set the default value. + data.setInternalErrorCode(Const.EMPTY_STRING); + } doDispatch(data); } finally { @@ -166,6 +171,7 @@ public class TelemetryDataDispatcher { service.setStatus(metrics.getStatus()); service.setResponseCode(metrics.getResponseCode()); service.setType(protocol2Type(metrics.getProtocol())); + service.getSideCar().setInternalErrorCode(metrics.getInternalErrorCode()); SOURCE_RECEIVER.receive(service); } @@ -187,6 +193,7 @@ public class TelemetryDataDispatcher { serviceRelation.setDetectPoint(detectPointMapping(metrics.getDetectPoint())); serviceRelation.setComponentId(protocol2Component(metrics.getProtocol())); serviceRelation.setTlsMode(metrics.getTlsMode()); + serviceRelation.getSideCar().setInternalErrorCode(metrics.getInternalErrorCode()); SOURCE_RECEIVER.receive(serviceRelation); } @@ -202,6 +209,7 @@ public class TelemetryDataDispatcher { serviceInstance.setStatus(metrics.getStatus()); serviceInstance.setResponseCode(metrics.getResponseCode()); serviceInstance.setType(protocol2Type(metrics.getProtocol())); + serviceInstance.getSideCar().setInternalErrorCode(metrics.getInternalErrorCode()); SOURCE_RECEIVER.receive(serviceInstance); } @@ -223,6 +231,7 @@ public class TelemetryDataDispatcher { serviceRelation.setDetectPoint(detectPointMapping(metrics.getDetectPoint())); serviceRelation.setComponentId(protocol2Component(metrics.getProtocol())); serviceRelation.setTlsMode(metrics.getTlsMode()); + serviceRelation.getSideCar().setInternalErrorCode(metrics.getInternalErrorCode()); SOURCE_RECEIVER.receive(serviceRelation); } @@ -238,6 +247,7 @@ public class TelemetryDataDispatcher { endpoint.setStatus(metrics.getStatus()); endpoint.setResponseCode(metrics.getResponseCode()); endpoint.setType(protocol2Type(metrics.getProtocol())); + endpoint.getSideCar().setInternalErrorCode(metrics.getInternalErrorCode()); SOURCE_RECEIVER.receive(endpoint); } diff --git a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/mesh/MeshDataMock.java b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/mesh/MeshDataMock.java index 13a5927f7b..69e6d1db87 100644 --- a/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/mesh/MeshDataMock.java +++ b/oap-server/server-receiver-plugin/skywalking-mesh-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/mesh/MeshDataMock.java @@ -53,6 +53,7 @@ public class MeshDataMock { .setStatus(true) .setProtocol(Protocol.HTTP) .setDetectPoint(DetectPoint.server) + .setInternalErrorCode("rate_limited") .build()); } meshObserver.onCompleted(); diff --git a/test/e2e/e2e-protocol/src/main/proto b/test/e2e/e2e-protocol/src/main/proto index 9933e2d170..5c0fe3a3a6 160000 --- a/test/e2e/e2e-protocol/src/main/proto +++ b/test/e2e/e2e-protocol/src/main/proto @@ -1 +1 @@ -Subproject commit 9933e2d17078c2bf07cd1c8d5ef36d52b5cbb917 +Subproject commit 5c0fe3a3a6f2118f86357385616dccd5ebade3c4 -- GitLab