Support choose files to activate the meter analyzer (#5748)

CHANGES.md

@@ -12,6 +12,7 @@ Release Notes.
 
 #### OAP-Backend
 * Add the `@SuperDataset` annotation for BrowserErrorLog.
+* Support choose files to active the meter analyzer.
 * Improve Kubernetes service registry for ALS analysis.
 
 #### UI

apm-dist-es7/src/main/assembly/binary-es7.xml

@@ -53,7 +53,7 @@
                 <include>endpoint-name-grouping.yml</include>
                 <include>oal/*.oal</include>
                 <include>fetcher-prom-rules/*.yaml</include>
-                <include>meter-receive-config/*.yaml</include>
+                <include>meter-analyzer-config/*.yaml</include>
                 <include>oc-rules/*</include>
                 <include>ui-initialized-templates/*</include>
             </includes>

apm-dist/src/main/assembly/binary.xml

@@ -53,7 +53,7 @@
                 <include>endpoint-name-grouping.yml</include>
                 <include>oal/*.oal</include>
                 <include>fetcher-prom-rules/*.yaml</include>
-                <include>meter-receive-config/*.yaml</include>
+                <include>meter-analyzer-config/*.yaml</include>
                 <include>oc-rules/*</include>
                 <include>ui-initialized-templates/*</include>
             </includes>

docs/en/setup/backend/backend-meter.md

@@ -23,11 +23,11 @@ Meter receiver is configured via a configuration file. The configuration file de
  from agents, as well as which rule files to load.
  
 OAP can load the configuration at bootstrap. If the new configuration is not well-formed, OAP fails to start up. The files
-are located at `$CLASSPATH/meter-receive-config`.
+are located at `$CLASSPATH/meter-analyzer-config`.
 
 The file is written in YAML format, defined by the scheme described below. Brackets indicate that a parameter is optional.
 
-A example can be found [here](../../../../oap-server/server-bootstrap/src/main/resources/meter-receive-config/spring-sleuth.yaml).
+A example can be found [here](../../../../oap-server/server-bootstrap/src/main/resources/meter-analyzer-config/spring-sleuth.yaml).
 If you're using Spring sleuth, you could use [Spring Sleuth Setup](spring-sleuth-setup.md).
 
 ### Meters configure

docs/en/setup/backend/configuration-vocabulary.md

@@ -145,6 +145,7 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode
 | - | - |forceSampleErrorSegment|When sampling mechanism activated, this config would make the error status segment sampled, ignoring the sampling rate.|SW_FORCE_SAMPLE_ERROR_SEGMENT|true|
 | - | - |segmentStatusAnalysisStrategy|Determine the final segment status from the status of spans. Available values are `FROM_SPAN_STATUS` , `FROM_ENTRY_SPAN` and `FROM_FIRST_SPAN`. `FROM_SPAN_STATUS` represents the segment status would be error if any span is in error status. `FROM_ENTRY_SPAN` means the segment status would be determined by the status of entry spans only. `FROM_FIRST_SPAN` means the segment status would be determined by the status of the first span only.|SW_SEGMENT_STATUS_ANALYSIS_STRATEGY|FROM_SPAN_STATUS|
 | - | - |noUpstreamRealAddressAgents|Exit spans with the component in the list would not generate the client-side instance relation metrics. As some tracing plugins can't collect the real peer ip address, such as Nginx-LUA and Envoy. |SW_NO_UPSTREAM_REAL_ADDRESS|6000,9000|
+| - | - |meterAnalyzerActiveFiles|Which files could be meter analyzed, files split by ","|SW_METER_ANALYZER_ACTIVE_FILES||
 | receiver-sharing-server|default| Sharing server provides new gRPC and restful servers for data collection. Ana make the servers in the core module working for internal communication only.| - | - |
 | - | - | restHost| Binding IP of restful service. Services include GraphQL query and HTTP data report| - | - |
 | - | - | restPort | Binding port of restful service |  - | - |

docs/en/setup/backend/spring-sleuth-setup.md

@@ -36,7 +36,7 @@ receiver-meter:
   default:
 ```
 
-2. Configure the meter config file, It already has the [spring sleuth meter config](../../../../oap-server/server-bootstrap/src/main/resources/meter-receive-config/spring-sleuth.yaml).
+2. Configure the meter config file, It already has the [spring sleuth meter config](../../../../oap-server/server-bootstrap/src/main/resources/meter-analyzer-config/spring-sleuth.yaml).
 If you also has some customized meter at the agent side, please read [meter document](backend-meter.md#meters-configure) to configure meter.
 
 ## Add UI dashboard

oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleConfig.java

@@ -18,11 +18,10 @@
 
 package org.apache.skywalking.oap.server.analyzer.provider;
 
-import java.util.ArrayList;
-import java.util.List;
 import lombok.Getter;
 import lombok.Setter;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.lang3.StringUtils;
 import org.apache.skywalking.oap.server.analyzer.provider.trace.DBLatencyThresholdsAndWatcher;
 import org.apache.skywalking.oap.server.analyzer.provider.trace.TraceSampleRateWatcher;
 import org.apache.skywalking.oap.server.analyzer.provider.trace.UninstrumentedGatewaysConfig;
@@ -30,6 +29,9 @@ import org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.
 import org.apache.skywalking.oap.server.core.Const;
 import org.apache.skywalking.oap.server.library.module.ModuleConfig;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import static org.apache.skywalking.oap.server.analyzer.provider.trace.parser.listener.strategy.SegmentStatusStrategy.FROM_SPAN_STATUS;
 
 @Slf4j
@@ -83,7 +85,13 @@ public class AnalyzerModuleConfig extends ModuleConfig {
     private int maxSlowSQLLength = 2000;
 
     @Getter
-    private final String configPath = "meter-receive-config";
+    private final String configPath = "meter-analyzer-config";
+
+    /**
+     * Which files could be meter analyzed, files split by ","
+     */
+    @Setter
+    private String meterAnalyzerActiveFiles = Const.EMPTY_STRING;
 
     /**
      * Sample the trace segment if the segment has span(s) tagged as error status, and ignore the sampleRate
@@ -123,4 +131,14 @@ public class AnalyzerModuleConfig extends ModuleConfig {
         }
         return virtualPeers.contains(componentId);
     }
+
+    /**
+     * Get all files could be meter analyzed, files split by ","
+     */
+    public String[] meterAnalyzerActiveFileNames() {
+        if (StringUtils.isEmpty(this.meterAnalyzerActiveFiles)) {
+            return null;
+        }
+        return this.meterAnalyzerActiveFiles.split(",");
+    }
 }

oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/AnalyzerModuleProvider.java

@@ -18,7 +18,6 @@
 
 package org.apache.skywalking.oap.server.analyzer.provider;
 
-import java.util.List;
 import lombok.Getter;
 import org.apache.skywalking.oap.server.analyzer.module.AnalyzerModule;
 import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig;
@@ -46,6 +45,8 @@ import org.apache.skywalking.oap.server.library.module.ModuleStartException;
 import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException;
 import org.apache.skywalking.oap.server.telemetry.TelemetryModule;
 
+import java.util.List;
+
 public class AnalyzerModuleProvider extends ModuleProvider {
     @Getter
     private final AnalyzerModuleConfig moduleConfig;
@@ -96,7 +97,7 @@ public class AnalyzerModuleProvider extends ModuleProvider {
         segmentParserService = new SegmentParserServiceImpl(getManager(), moduleConfig);
         this.registerServiceImplementation(ISegmentParserService.class, segmentParserService);
 
-        meterConfigs = MeterConfigs.loadConfig(moduleConfig.getConfigPath());
+        meterConfigs = MeterConfigs.loadConfig(moduleConfig.getConfigPath(), moduleConfig.meterAnalyzerActiveFileNames());
         processService = new MeterProcessService(getManager());
         this.registerServiceImplementation(IMeterProcessService.class, processService);
     }

oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/meter/config/MeterConfigs.java

@@ -30,6 +30,7 @@ import java.io.FileReader;
 import java.io.IOException;
 import java.io.Reader;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.stream.Collectors;
@@ -43,10 +44,14 @@ public class MeterConfigs {
     /**
      * Load all configs from path
      */
-    public static List<MeterConfig> loadConfig(String path) throws ModuleStartException {
+    public static List<MeterConfig> loadConfig(String path, String[] fileNames) throws ModuleStartException {
+        if (fileNames == null || fileNames.length == 0) {
+            return Collections.emptyList();
+        }
+
         File[] configs;
         try {
-            configs = ResourceUtils.getPathFiles(path);
+            configs = ResourceUtils.getPathFiles(path, fileNames);
         } catch (FileNotFoundException e) {
             throw new ModuleStartException("Load meter configs failed", e);
         }

oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterBaseTest.java

@@ -19,8 +19,6 @@
 package org.apache.skywalking.oap.server.analyzer.provider.meter.process;
 
 import com.google.common.collect.Maps;
-import java.util.HashMap;
-import java.util.List;
 import org.apache.skywalking.apm.network.language.agent.v3.Label;
 import org.apache.skywalking.apm.network.language.agent.v3.MeterBucketValue;
 import org.apache.skywalking.apm.network.language.agent.v3.MeterData;
@@ -41,10 +39,13 @@ import org.mockito.Mock;
 import org.mockito.Mockito;
 import org.powermock.reflect.Whitebox;
 
+import java.util.HashMap;
+import java.util.List;
+
 import static org.mockito.Mockito.when;
 
 public abstract class MeterBaseTest {
-    private static final String CONFIG_PATH = "meter-receive-config";
+    private static final String CONFIG_PATH = "meter-analyzer-config";
 
     @Mock
     protected CoreModuleProvider moduleProvider;
@@ -80,7 +81,7 @@ public abstract class MeterBaseTest {
         Whitebox.setInternalState(meterSystem, "functionRegister", map);
 
         // load context
-        List<MeterConfig> meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH);
+        List<MeterConfig> meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH, new String[] {"config.yaml"});
         final MeterProcessService service = new MeterProcessService(moduleManager);
         service.start(meterConfigs);
 

oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/meter/process/MeterProcessContextTest.java

@@ -18,7 +18,6 @@
 
 package org.apache.skywalking.oap.server.analyzer.provider.meter.process;
 
-import java.util.List;
 import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfig;
 import org.apache.skywalking.oap.server.analyzer.provider.meter.config.MeterConfigs;
 import org.apache.skywalking.oap.server.library.module.ModuleStartException;
@@ -29,6 +28,8 @@ import org.powermock.core.classloader.annotations.PowerMockIgnore;
 import org.powermock.modules.junit4.PowerMockRunner;
 import org.powermock.reflect.Whitebox;
 
+import java.util.List;
+
 import static org.mockito.Matchers.any;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
@@ -36,11 +37,11 @@ import static org.mockito.Mockito.verify;
 @RunWith(PowerMockRunner.class)
 @PowerMockIgnore({"com.sun.org.apache.xerces.*", "javax.xml.*", "org.xml.*", "javax.management.*", "org.w3c.*"})
 public class MeterProcessContextTest extends MeterBaseTest {
-    private static final String CONFIG_PATH = "meter-receive-config";
+    private static final String CONFIG_PATH = "meter-analyzer-config";
 
     @Test
     public void testInitMeter() throws ModuleStartException {
-        List<MeterConfig> meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH);
+        List<MeterConfig> meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH, new String[] {"config.yaml"});
         final MeterProcessService service = new MeterProcessService(moduleManager);
         service.start(meterConfigs);
 
@@ -51,7 +52,7 @@ public class MeterProcessContextTest extends MeterBaseTest {
 
     @Test
     public void testCreateNewProcessor() throws ModuleStartException {
-        List<MeterConfig> meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH);
+        List<MeterConfig> meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH, new String[] {"config.yaml"});
         final MeterProcessService service = new MeterProcessService(moduleManager);
         service.start(meterConfigs);
 

oap-server/server-bootstrap/pom.xml

@@ -259,7 +259,7 @@
                         <exclude>endpoint-name-grouping.yml</exclude>
                         <exclude>oal/</exclude>
                         <exclude>fetcher-prom-rules/</exclude>
-                        <exclude>meter-receive-config/</exclude>
+                        <exclude>meter-analyzer-config/</exclude>
                         <exclude>oc-rules/</exclude>
                         <exclude>ui-initialized-templates/</exclude>
                     </excludes>

oap-server/server-bootstrap/src/main/resources/application.yml

@@ -191,6 +191,7 @@ agent-analyzer:
     # Nginx and Envoy agents can't get the real remote address.
     # Exit spans with the component in the list would not generate the client-side instance relation metrics.
     noUpstreamRealAddressAgents: ${SW_NO_UPSTREAM_REAL_ADDRESS:6000,9000}
+    meterAnalyzerActiveFiles: ${SW_METER_ANALYZER_ACTIVE_FILES:} # Which files could be meter analyzed, files split by ","
 
 receiver-sharing-server:
   selector: ${SW_RECEIVER_SHARING_SERVER:default}
@@ -269,7 +270,7 @@ kafka-fetcher:
     consumePartitions: ${SW_KAFKA_FETCHER_CONSUME_PARTITIONS:""}
 
 receiver-meter:
-  selector: ${SW_RECEIVER_METER:-}
+  selector: ${SW_RECEIVER_METER:default}
   default:
 
 receiver-oc:

oap-server/server-fetcher-plugin/kafka-fetcher-plugin/src/main/java/org/apache/skywalking/oap/server/analyzer/agent/kafka/module/KafkaFetcherConfig.java

@@ -18,10 +18,11 @@
 
 package org.apache.skywalking.oap.server.analyzer.agent.kafka.module;
 
-import java.util.Properties;
 import lombok.Data;
 import org.apache.skywalking.oap.server.library.module.ModuleConfig;
 
+import java.util.Properties;
+
 @Data
 public class KafkaFetcherConfig extends ModuleConfig {
 
@@ -68,7 +69,7 @@ public class KafkaFetcherConfig extends ModuleConfig {
 
     private boolean enableMeterSystem = false;
 
-    private String configPath = "meter-receive-config";
+    private String configPath = "meter-analyzer-config";
 
     private String topicNameOfMetrics = "skywalking-metrics";
 

oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java

@@ -24,7 +24,10 @@ import java.io.InputStream;
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.net.URL;
+import java.util.Arrays;
+import java.util.HashSet;
 import java.util.Objects;
+import java.util.Set;
 
 public class ResourceUtils {
 
@@ -47,4 +50,19 @@ public class ResourceUtils {
         }
         return Objects.requireNonNull(new File(url.getPath()).listFiles(), "No files in " + path);
     }
+
+    public static File[] getPathFiles(String parentPath, String[] fileNames) throws FileNotFoundException {
+        URL url = ResourceUtils.class.getClassLoader().getResource(parentPath);
+        if (url == null) {
+            throw new FileNotFoundException("path not found: " + parentPath);
+        }
+        final Set<String> nameSet = new HashSet<>(Arrays.asList(fileNames));
+        final File[] listFiles = Objects.requireNonNull(new File(url.getPath())
+            .listFiles((dir, name) -> nameSet.contains(name)), "No files in " + parentPath);
+
+        if (listFiles.length == 0) {
+            throw new FileNotFoundException("files not found:" + nameSet);
+        }
+        return listFiles;
+    }
 }

oap-server/server-library/library-util/src/test/java/org/apache/skywalking/oap/server/library/util/ResourceUtilsTest.java

@@ -17,13 +17,32 @@
 
 package org.apache.skywalking.oap.server.library.util;
 
-import java.io.FileNotFoundException;
+import org.junit.Assert;
 import org.junit.Test;
 
+import java.io.File;
+import java.io.FileNotFoundException;
+
 public class ResourceUtilsTest {
 
     @Test(expected = FileNotFoundException.class)
     public void shouldThrowWhenResourceNotFound() throws FileNotFoundException {
         ResourceUtils.read("/not-existed");
     }
+
+    @Test
+    public void testGetPathFilesSuccess() throws FileNotFoundException {
+        final File[] files = ResourceUtils.getPathFiles("testdata", new String[] {
+            "prometheus.txt"
+        });
+        Assert.assertNotNull(files);
+        Assert.assertEquals(1, files.length);
+    }
+
+    @Test(expected = FileNotFoundException.class)
+    public void testGetPathFilesNotFound() throws FileNotFoundException {
+        ResourceUtils.getPathFiles("testdata", new String[] {
+            "not-existed"
+        });
+    }
 }
\ No newline at end of file

oap-server/server-receiver-plugin/skywalking-meter-receiver-plugin/src/test/java/org/apache/skywalking/oap/server/receiver/meter/provider/config/MeterConfigsTest.java

@@ -29,11 +29,11 @@ import java.util.Arrays;
 import java.util.List;
 
 public class MeterConfigsTest {
-    private static final String CONFIG_PATH = "meter-receive-config";
+    private static final String CONFIG_PATH = "meter-analyzer-config";
 
     @Test
     public void testLoadConfig() throws ModuleStartException {
-        final List<MeterConfig> meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH);
+        final List<MeterConfig> meterConfigs = MeterConfigs.loadConfig(CONFIG_PATH, new String[] {"config.yaml"});
 
         Assert.assertEquals(3, meterConfigs.size());
 

test/e2e/e2e-test/docker/kafka/docker-compose.meter.yml

@@ -21,12 +21,12 @@ services:
       file: ../base-compose.yml
       service: oap
     environment:
-      SW_RECEIVER_METER: default
       SW_KAFKA_FETCHER: default
       SW_KAFKA_FETCHER_SERVERS: broker-a:9092,broker-b:9092
       SW_KAFKA_FETCHER_PARTITIONS: 2
       SW_KAFKA_FETCHER_PARTITIONS_FACTOR: 1
       SW_KAFKA_FETCHER_ENABLE_METER_SYSTEM: "true"
+      SW_METER_ANALYZER_ACTIVE_FILES: spring-sleuth.yaml
     depends_on:
       broker-a:
         condition: service_healthy

test/e2e/e2e-test/docker/meter/docker-compose.yml

@@ -18,7 +18,7 @@ version: '2.1'
 services:
   oap:
     environment:
-      SW_RECEIVER_METER: default
+      SW_METER_ANALYZER_ACTIVE_FILES: spring-sleuth.yaml
     extends:
       file: ../base-compose.yml
       service: oap