提交 9a09e962 编写于 作者: wu-sheng's avatar wu-sheng 提交者: 彭勇升 pengys

Support disable statement in OAL (#2402)

* Try to make disable work.

* Make disable works.

* Add disable document.
上级 a482eaf3
......@@ -10,10 +10,13 @@ The OAL scripts will be compiled to normal Java codes in package stage.
## Grammar
Scripts should be named as `*.oal`
```
// Declare the metric.
METRIC_NAME = from(SCOPE.(* | [FIELD][,FIELD ...]))
[.filter(FIELD OP [INT | STRING])]
.FUNCTION([PARAM][, PARAM ...])
// Disable hard code
disable(METRIC_NAME);
```
## Scope
......@@ -67,6 +70,13 @@ All metric data will be grouped by Scope.ID and min-level TimeBucket.
- In `Endpoint` scope, the Scope.ID = Endpoint id (the unique id based on service and its Endpoint)
## Disable
`Disable` is an advanced statement in OAL, which is only used in certain case.
Some of the aggregation and metric are defined through core hard codes,
this `disable` statement is designed for make them de-active,
such as `segment`, `top_n_database_statement`.
In default, no one is being disable.
## Examples
```
// Caculate p99 of both Endpoint1 and Endpoint2
......@@ -93,4 +103,8 @@ Endpoint_500 = from(Endpoint.*).filter(responseCode like "5%").percent()
// Caculate the sum of calls for each service.
EndpointCalls = from(Endpoint.*).sum()
disable(segment);
disable(endpoint_relation_server_side);
disable(top_n_database_statement);
```
......@@ -26,6 +26,7 @@ lexer grammar OALLexer;
FROM: 'from';
FILTER: 'filter';
DISABLE: 'disable';
SRC_ALL: 'All';
SRC_SERVICE: 'Service';
SRC_SERVICE_INSTANCE: 'ServiceInstance';
......@@ -43,6 +44,14 @@ SRC_SERVICE_INSTANCE_CLR_GC: 'ServiceInstanceCLRGC';
SRC_SERVICE_INSTANCE_CLR_THREAD: 'ServiceInstanceCLRThread';
SRC_ENVOY_INSTANCE_METRIC: 'EnvoyInstanceMetric';
//hard code sources, only used when need to be deactived.
SRC_SEGMENT: 'segment';
SRC_TOP_N_DB_STATEMENT: 'top_n_database_statement';
SRC_ENDPOINT_RELATION_SERVER_SIDE: 'endpoint_relation_server_side';
SRC_SERVICE_RELATION_SERVER_SIDE: 'service_relation_server_side';
SRC_SERVICE_RELATION_CLIENT_SIDE: 'service_relation_client_side';
SRC_ALARM_RECORD: 'alarm_record';
// Literals
BOOL_LITERAL: 'true'
......
......@@ -26,13 +26,17 @@ options { tokenVocab=OALLexer; }
// Top Level Description
root
: (aggregationStatement)*
: (aggregationStatement | disableStatement)*
;
aggregationStatement
: variable (SPACE)? EQUAL (SPACE)? metricStatement DelimitedComment? LineComment? (SEMI|EOF)
;
disableStatement
: DISABLE LR_BRACKET disableSource RR_BRACKET DelimitedComment? LineComment? (SEMI|EOF)
;
metricStatement
: FROM LR_BRACKET source DOT sourceAttribute RR_BRACKET (filterStatement+)? DOT aggregateFunction
;
......@@ -53,6 +57,11 @@ source
SRC_ENVOY_INSTANCE_METRIC
;
disableSource
: SRC_SEGMENT | SRC_TOP_N_DB_STATEMENT | SRC_ENDPOINT_RELATION_SERVER_SIDE | SRC_SERVICE_RELATION_SERVER_SIDE |
SRC_SERVICE_RELATION_CLIENT_SIDE | SRC_ALARM_RECORD
;
sourceAttribute
: IDENTIFIER | ALL
;
......
......@@ -20,7 +20,6 @@ package org.apache.skywalking.oal.tool;
import freemarker.template.TemplateException;
import java.io.*;
import java.util.List;
import org.apache.skywalking.apm.util.StringUtil;
import org.apache.skywalking.oal.tool.meta.*;
import org.apache.skywalking.oal.tool.output.FileGenerator;
......@@ -59,9 +58,9 @@ public class Main {
SourceColumnsFactory.setSettings(metaSettings);
ScriptParser scriptParser = ScriptParser.createFromFile(scriptFilePath);
List<AnalysisResult> analysisResults = scriptParser.parse();
OALScripts oalScripts = scriptParser.parse();
FileGenerator generator = new FileGenerator(analysisResults, outputPath);
FileGenerator generator = new FileGenerator(oalScripts, outputPath);
generator.generate();
}
}
......@@ -21,16 +21,18 @@ package org.apache.skywalking.oal.tool.output;
import freemarker.template.*;
import java.io.*;
import java.util.*;
import org.apache.skywalking.oal.tool.parser.AnalysisResult;
import org.apache.skywalking.oal.tool.parser.*;
public class FileGenerator {
private List<AnalysisResult> results;
private DisableCollection collection;
private String outputPath;
private Configuration configuration;
private AllDispatcherContext allDispatcherContext;
public FileGenerator(List<AnalysisResult> results, String outputPath) {
this.results = results;
public FileGenerator(OALScripts oalScripts, String outputPath) {
this.results = oalScripts.getIndicatorStmts();
this.collection = oalScripts.getDisableCollection();
this.outputPath = outputPath;
configuration = new Configuration(new Version("2.3.28"));
configuration.setEncoding(Locale.ENGLISH, "UTF-8");
......@@ -48,6 +50,7 @@ public class FileGenerator {
createFile(file);
generateDispatcher(result, new FileWriter(file));
}
generateDisable();
}
private void generate(AnalysisResult result, String fileSuffix,
......@@ -103,4 +106,10 @@ public class FileGenerator {
context.getIndicators().add(result);
}
}
private void generateDisable() throws IOException, TemplateException {
File file = new File(outputPath, "generated/DisableSourceDefinition.java");
createFile(file);
configuration.getTemplate("DisableSourceDefinition.ftl").process(collection, new FileWriter(file));
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oal.tool.parser;
import java.util.*;
import lombok.Getter;
/**
* @author wusheng
*/
@Getter
public class DisableCollection {
private List<String> allDisableSources = new ArrayList<>();
public void add(String source) {
allDisableSources.add(source);
}
}
......@@ -26,11 +26,13 @@ import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
public class OALListener extends OALParserBaseListener {
private List<AnalysisResult> results;
private AnalysisResult current;
private DisableCollection collection;
private ConditionExpression conditionExpression;
public OALListener(List<AnalysisResult> results) {
this.results = results;
public OALListener(OALScripts scripts) {
this.results = scripts.getIndicatorStmts();
this.collection = scripts.getDisableCollection();
}
@Override
......@@ -150,6 +152,15 @@ public class OALListener extends OALParserBaseListener {
return source;
}
/**
* Disable source
*
* @param ctx
*/
@Override public void enterDisableSource(OALParser.DisableSourceContext ctx) {
collection.add(ctx.getText());
}
private String firstLetterUpper(String source) {
return source.substring(0, 1).toUpperCase() + source.substring(1);
}
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oal.tool.parser;
import java.util.*;
import lombok.*;
@Getter
public class OALScripts {
private List<AnalysisResult> indicatorStmts;
private DisableCollection disableCollection;
public OALScripts() {
indicatorStmts = new LinkedList<>();
disableCollection = new DisableCollection();
}
}
......@@ -43,8 +43,9 @@ public class ScriptParser {
return parser;
}
public List<AnalysisResult> parse() throws IOException {
List<AnalysisResult> results = new LinkedList<>();
public OALScripts parse() throws IOException {
OALScripts scripts = new OALScripts();
CommonTokenStream tokens = new CommonTokenStream(lexer);
OALParser parser = new OALParser(tokens);
......@@ -52,9 +53,9 @@ public class ScriptParser {
ParseTree tree = parser.root();
ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(new OALListener(results), tree);
walker.walk(new OALListener(scripts), tree);
return results;
return scripts;
}
public void close() {
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.analysis.generated;
<#if (allDisableSources?size>0)>
import org.apache.skywalking.oap.server.core.analysis.Disable;
</#if>
/**
* This class is auto generated. Please don't change this class manually.
*
* @author Observability Analysis Language code generator
*/
<#list allDisableSources as disableSource>
@Disable("${disableSource}")
</#list>
public class DisableSourceDefinition {
}
\ No newline at end of file
......@@ -79,10 +79,10 @@ public class FileGeneratorTest {
public void testGenerateIndicatorImplementor() throws IOException, TemplateException {
AnalysisResult result = buildResult();
List<AnalysisResult> results = new LinkedList<>();
results.add(result);
OALScripts oalScripts = new OALScripts();
oalScripts.getIndicatorStmts().add(result);
FileGenerator fileGenerator = new FileGenerator(results, ".");
FileGenerator fileGenerator = new FileGenerator(oalScripts, ".");
StringWriter writer = new StringWriter();
fileGenerator.generateIndicatorImplementor(result, writer);
Assert.assertEquals(readExpectedFile("IndicatorImplementorExpected.java"), writer.toString());
......@@ -94,10 +94,10 @@ public class FileGeneratorTest {
public void testServiceDispatcher() throws IOException, TemplateException {
AnalysisResult result = buildResult();
List<AnalysisResult> results = new LinkedList<>();
results.add(result);
OALScripts oalScripts = new OALScripts();
oalScripts.getIndicatorStmts().add(result);
FileGenerator fileGenerator = new FileGenerator(results, ".");
FileGenerator fileGenerator = new FileGenerator(oalScripts, ".");
StringWriter writer = new StringWriter();
fileGenerator.generateDispatcher(result, writer);
Assert.assertEquals(readExpectedFile("ServiceDispatcherExpected.java"), writer.toString());
......
......@@ -50,7 +50,7 @@ public class ScriptParserTest {
"Endpoint_avg = from(Endpoint.latency).longAvg(); //comment test" + "\n" +
"Service_avg = from(Service.latency).longAvg()"
);
List<AnalysisResult> results = parser.parse();
List<AnalysisResult> results = parser.parse().getIndicatorStmts();
Assert.assertEquals(2, results.size());
......@@ -72,7 +72,7 @@ public class ScriptParserTest {
ScriptParser parser = ScriptParser.createFromScriptText(
"Endpoint_percent = from(Endpoint.*).percent(status == true);"
);
List<AnalysisResult> results = parser.parse();
List<AnalysisResult> results = parser.parse().getIndicatorStmts();
AnalysisResult endpointPercent = results.get(0);
Assert.assertEquals("EndpointPercent", endpointPercent.getMetricName());
......@@ -91,7 +91,7 @@ public class ScriptParserTest {
ScriptParser parser = ScriptParser.createFromScriptText(
"Endpoint_percent = from(Endpoint.*).filter(status == true).filter(name == \"/product/abc\").longAvg();"
);
List<AnalysisResult> results = parser.parse();
List<AnalysisResult> results = parser.parse().getIndicatorStmts();
AnalysisResult endpointPercent = results.get(0);
Assert.assertEquals("EndpointPercent", endpointPercent.getMetricName());
......@@ -121,7 +121,7 @@ public class ScriptParserTest {
"service_response_s3_summary = from(Service.latency).filter(latency >= 3000).sum();" + "\n" +
"service_response_s4_summary = from(Service.latency).filter(latency <= 4000).sum();"
);
List<AnalysisResult> results = parser.parse();
List<AnalysisResult> results = parser.parse().getIndicatorStmts();
AnalysisResult responseSummary = results.get(0);
Assert.assertEquals("ServiceResponseS1Summary", responseSummary.getMetricName());
......@@ -167,4 +167,14 @@ public class ScriptParserTest {
Assert.assertEquals("4000", booleanMatchExp.getValue());
Assert.assertEquals("lessEqualMatch", booleanMatchExp.getExpressionType());
}
@Test
public void testDisable() throws IOException {
ScriptParser parser = ScriptParser.createFromScriptText(
"disable(segment);");
DisableCollection collection = parser.parse().getDisableCollection();
List<String> sources = collection.getAllDisableSources();
Assert.assertEquals(1, sources.size());
Assert.assertEquals("segment", sources.get(0));
}
}
......@@ -96,3 +96,9 @@ instance_clr_max_worker_threads = from(ServiceInstanceCLRThread.maxWorkerThreads
envoy_heap_memory_max_used = from(EnvoyInstanceMetric.value).filter(metricName == "server.memory_heap_size").maxDouble();
envoy_total_connections_used = from(EnvoyInstanceMetric.value).filter(metricName == "server.total_connections").maxDouble();
envoy_parent_connections_used = from(EnvoyInstanceMetric.value).filter(metricName == "server.parent_connections").maxDouble();
// Disable unnecessary hard core sources
/////////
// disable(segment);
// disable(endpoint_relation_server_side);
// disable(top_n_database_statement);
\ No newline at end of file
......@@ -19,6 +19,7 @@
package org.apache.skywalking.oap.server.core;
import java.io.IOException;
import org.apache.skywalking.oap.server.core.analysis.DisableRegister;
import org.apache.skywalking.oap.server.core.analysis.indicator.annotation.IndicatorTypeListener;
import org.apache.skywalking.oap.server.core.analysis.record.annotation.RecordTypeListener;
import org.apache.skywalking.oap.server.core.analysis.topn.annotation.TopNTypeListener;
......@@ -88,6 +89,7 @@ public class CoreModuleProvider extends ModuleProvider {
@Override public void prepare() throws ServiceNotProvidedException, ModuleStartException {
AnnotationScan scopeScan = new AnnotationScan();
scopeScan.registerListener(new DefaultScopeDefine.Listener());
scopeScan.registerListener(DisableRegister.INSTANCE);
try {
scopeScan.scan(null);
} catch (IOException e) {
......
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.analysis;
import java.lang.annotation.*;
/**
* @author wusheng
*/
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
@Repeatable(MultipleDisable.class)
public @interface Disable {
String value();
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.analysis;
import java.lang.annotation.Annotation;
import java.util.*;
import org.apache.skywalking.oap.server.core.annotation.AnnotationListener;
/**
* Disable definition scanner and register.
*
* @author wusheng
*/
public class DisableRegister implements AnnotationListener {
public static DisableRegister INSTANCE = new DisableRegister();
private Set<String> disableEntitySet = new HashSet<>();
private DisableRegister() {
}
@Override public Class<? extends Annotation> annotation() {
return MultipleDisable.class;
}
@Override public void notify(Class aClass) {
MultipleDisable annotation = (MultipleDisable)aClass.getAnnotation(MultipleDisable.class);
Disable[] valueList = annotation.value();
if (valueList != null) {
for (Disable disable : valueList) {
disableEntitySet.add(disable.value());
}
}
}
public boolean include(String name) {
return disableEntitySet.contains(name);
}
}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/
package org.apache.skywalking.oap.server.core.analysis;
import java.lang.annotation.*;
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
public @interface MultipleDisable {
Disable[] value();
}
......@@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.core.analysis.worker;
import java.util.*;
import lombok.Getter;
import org.apache.skywalking.oap.server.core.*;
import org.apache.skywalking.oap.server.core.analysis.DisableRegister;
import org.apache.skywalking.oap.server.core.analysis.indicator.Indicator;
import org.apache.skywalking.oap.server.core.storage.*;
import org.apache.skywalking.oap.server.core.storage.annotation.StorageEntityAnnotationUtils;
......@@ -37,11 +38,19 @@ public enum IndicatorProcess {
@Getter private List<IndicatorPersistentWorker> persistentWorkers = new ArrayList<>();
public void in(Indicator indicator) {
entryWorkers.get(indicator.getClass()).in(indicator);
IndicatorAggregateWorker worker = entryWorkers.get(indicator.getClass());
if (worker != null) {
worker.in(indicator);
}
}
public void create(ModuleManager moduleManager, Class<? extends Indicator> indicatorClass) {
String modelName = StorageEntityAnnotationUtils.getModelName(indicatorClass);
if (DisableRegister.INSTANCE.include(modelName)) {
return;
}
Class<? extends StorageBuilder> builderClass = StorageEntityAnnotationUtils.getBuilder(indicatorClass);
StorageDAO storageDAO = moduleManager.find(StorageModule.NAME).provider().getService(StorageDAO.class);
......
......@@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.core.analysis.worker;
import java.util.*;
import lombok.Getter;
import org.apache.skywalking.oap.server.core.UnexpectedException;
import org.apache.skywalking.oap.server.core.analysis.DisableRegister;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.storage.*;
import org.apache.skywalking.oap.server.core.storage.annotation.StorageEntityAnnotationUtils;
......@@ -36,13 +37,21 @@ public enum RecordProcess {
private Map<Class<? extends Record>, RecordPersistentWorker> workers = new HashMap<>();
public void in(Record record) {
workers.get(record.getClass()).in(record);
RecordPersistentWorker worker = workers.get(record.getClass());
if (worker != null) {
worker.in(record);
}
}
@Getter private List<RecordPersistentWorker> persistentWorkers = new ArrayList<>();
public void create(ModuleManager moduleManager, Class<? extends Record> recordClass) {
String modelName = StorageEntityAnnotationUtils.getModelName(recordClass);
if (DisableRegister.INSTANCE.include(modelName)) {
return;
}
Class<? extends StorageBuilder> builderClass = StorageEntityAnnotationUtils.getBuilder(recordClass);
StorageDAO storageDAO = moduleManager.find(StorageModule.NAME).provider().getService(StorageDAO.class);
......
......@@ -21,6 +21,7 @@ package org.apache.skywalking.oap.server.core.analysis.worker;
import java.util.*;
import lombok.Getter;
import org.apache.skywalking.oap.server.core.UnexpectedException;
import org.apache.skywalking.oap.server.core.analysis.DisableRegister;
import org.apache.skywalking.oap.server.core.analysis.manual.database.TopNDatabaseStatement;
import org.apache.skywalking.oap.server.core.analysis.record.Record;
import org.apache.skywalking.oap.server.core.analysis.topn.TopN;
......@@ -30,8 +31,8 @@ import org.apache.skywalking.oap.server.core.worker.*;
import org.apache.skywalking.oap.server.library.module.ModuleManager;
/**
* TopN is a special process, which hold a certain size of windows,
* and cache all top N records, save to the persistence in low frequence.
* TopN is a special process, which hold a certain size of windows, and cache all top N records, save to the persistence
* in low frequence.
*
* @author wusheng
*/
......@@ -43,6 +44,11 @@ public enum TopNProcess {
public void create(ModuleManager moduleManager, Class<? extends TopN> topNClass) {
String modelName = StorageEntityAnnotationUtils.getModelName(topNClass);
if (DisableRegister.INSTANCE.include(modelName)) {
return;
}
Class<? extends StorageBuilder> builderClass = StorageEntityAnnotationUtils.getBuilder(topNClass);
StorageDAO storageDAO = moduleManager.find(StorageModule.NAME).provider().getService(StorageDAO.class);
......@@ -61,6 +67,9 @@ public enum TopNProcess {
}
public void in(TopNDatabaseStatement statement) {
workers.get(statement.getClass()).in(statement);
TopNWorker worker = workers.get(statement.getClass());
if (worker != null) {
worker.in(statement);
}
}
}
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册