Add agent plugins selector (#5293)

.github/workflows/ci-it.yaml

@@ -39,6 +39,8 @@ jobs:
       - uses: actions/setup-java@v1
         with:
           java-version: 8
+      - name: 'Check Javaagent Plugin List'
+        run: tools/plugin/check-javaagent-plugin-list.sh
       - name: 'Install & Test'
         run: |
           ./mvnw --batch-mode -P"agent,backend,ui,dist,CI-with-IT" clean cobertura:cobertura verify install javadoc:javadoc

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java

@@ -266,6 +266,11 @@ public class Config {
          * Control the length of the peer field.
          */
         public static int PEER_MAX_LENGTH = 200;
+
+        /**
+         * Exclude activated plugins
+         */
+        public static String EXCLUDE_PLUGINS = "";
     }
 
     public static class Correlation {

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/plugin/PluginCfg.java

@@ -35,14 +35,15 @@ public enum PluginCfg {
     private static final ILog logger = LogManager.getLogger(PluginCfg.class);
 
     private List<PluginDefine> pluginClassList = new ArrayList<PluginDefine>();
+    private PluginSelector pluginSelector = new PluginSelector();
 
     void load(InputStream input) throws IOException {
         try {
             BufferedReader reader = new BufferedReader(new InputStreamReader(input));
-            String pluginDefine = null;
+            String pluginDefine;
             while ((pluginDefine = reader.readLine()) != null) {
                 try {
-                    if (pluginDefine == null || pluginDefine.trim().length() == 0 || pluginDefine.startsWith("#")) {
+                    if (pluginDefine.trim().length() == 0 || pluginDefine.startsWith("#")) {
                         continue;
                     }
                     PluginDefine plugin = PluginDefine.build(pluginDefine);
@@ -51,6 +52,7 @@ public enum PluginCfg {
                     logger.error(e, "Failed to format plugin({}) define.", pluginDefine);
                 }
             }
+            pluginClassList = pluginSelector.select(pluginClassList);
         } finally {
             input.close();
         }

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/plugin/PluginDefine.java

@@ -55,5 +55,9 @@ public class PluginDefine {
     public String getDefineClass() {
         return defineClass;
     }
+
+    public String getName() {
+        return name;
+    }
 }
 

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/plugin/PluginSelector.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.agent.core.plugin;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.skywalking.apm.agent.core.conf.Config;
+
+import static org.apache.skywalking.apm.agent.core.conf.Config.Plugin.EXCLUDE_PLUGINS;
+
+/**
+ * Select some plugins in activated plugins
+ */
+public class PluginSelector {
+    /**
+     * Exclude activated plugins
+     *
+     * @param pluginDefines the pluginDefines is loaded from activations directory or plugins directory
+     * @return real activate plugins
+     * @see Config.Plugin#EXCLUDE_PLUGINS
+     */
+    public List<PluginDefine> select(List<PluginDefine> pluginDefines) {
+        if (!EXCLUDE_PLUGINS.isEmpty()) {
+            List<String> excludes = Arrays.asList(EXCLUDE_PLUGINS.toLowerCase().split(","));
+            return pluginDefines.stream()
+                                .filter(item -> !excludes.contains(item.getName().toLowerCase()))
+                                .collect(Collectors.toList());
+        }
+        return pluginDefines;
+    }
+}

apm-sniffer/apm-agent-core/src/test/java/org/apache/skywalking/apm/agent/core/plugin/PluginSelectorTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.apm.agent.core.plugin;
+
+import java.util.ArrayList;
+import java.util.List;
+import org.apache.skywalking.apm.agent.core.plugin.exception.IllegalPluginDefineException;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.skywalking.apm.agent.core.conf.Config.Plugin.EXCLUDE_PLUGINS;
+
+public class PluginSelectorTest {
+
+    List<PluginDefine> pluginDefines;
+    PluginSelector selector;
+
+    @Before
+    public void prepare() throws IllegalPluginDefineException {
+        pluginDefines = new ArrayList<>();
+        selector = new PluginSelector();
+        pluginDefines.add(PluginDefine.build("elasticsearch=elasticsearchClass"));
+        pluginDefines.add(PluginDefine.build("mysql=mysqlClass"));
+    }
+
+    @Test
+    public void selectDefaultTest() {
+        EXCLUDE_PLUGINS = "";
+        List<PluginDefine> select = selector.select(pluginDefines);
+        Assert.assertEquals(2, select.size());
+    }
+
+    @Test
+    public void selectNormalTest() {
+        EXCLUDE_PLUGINS = "mysql";
+        List<PluginDefine> plugins = selector.select(pluginDefines);
+        Assert.assertEquals(1, plugins.size());
+        Assert.assertEquals("elasticsearch", plugins.get(0).getName());
+    }
+}
\ No newline at end of file

apm-sniffer/config/agent.config

@@ -90,3 +90,6 @@ logging.level=${SW_LOGGING_LEVEL:INFO}
 
 # Kafka producer configuration
 # plugin.kafka.bootstrap_servers=${SW_KAFKA_BOOTSTRAP_SERVERS:localhost:9092}
+
+# Exclude activated plugins
+# plugin.exclude_plugins=${SW_EXCLUDE_PLUGINS:""}

docs/en/setup/service-agent/java-agent/Plugin-list.md

+# Skywalking Agent List
+- activemq-5.x
+- armeria-063-084
+- armeria-085
+- armeria-086
+- armeria-098
+- avro-1.x
+- brpc-java
+- canal-1.x
+- cassandra-java-driver-3.x
+- dubbo
+- ehcache-2.x
+- elastic-job-2.x
+- elastic-job-3.x
+- elasticsearch-5.x
+- elasticsearch-6.x
+- feign-default-http-9.x
+- feign-pathvar-9.x
+- finagle
+- graphql
+- grpc-1.x
+- gson-2.8.x
+- h2-1.x
+- hbase-1.x
+- httpasyncclient-4.x
+- httpclient-3.x
+- httpclient-4.x
+- hystrix-1.x
+- influxdb-2.x
+- jdk-http-plugin
+- jdk-threading-plugin
+- jedis-2.x
+- jetty-client-9.0
+- jetty-client-9.x
+- jetty-server-9.x
+- kafka-0.11.x/1.x/2.x
+- kotlin-coroutine
+- lettuce-5.x
+- light4j
+- mariadb-2.x
+- memcache-2.x
+- mongodb-2.x
+- mongodb-3.x
+- motan-0.x
+- mysql-5.x
+- mysql-6.x
+- mysql-8.x
+- netty-socketio
+- nutz-http-1.x
+- nutz-mvc-annotation-1.x
+- okhttp-3.x
+- play-2.x
+- postgresql-8.x
+- pulsar
+- quasar
+- rabbitmq-5.x
+- redisson-3.x
+- resteasy-server-3.x
+- rocketMQ-3.x
+- rocketMQ-4.x
+- servicecomb-0.x
+- servicecomb-1.x
+- sharding-jdbc-1.5.x
+- sharding-sphere-3.x
+- sharding-sphere-4.0.0
+- sharding-sphere-4.1.0
+- sharding-sphere-4.x
+- sharding-sphere-4.x-rc3
+- sofarpc
+- solrj-7.x
+- spring-annotation
+- spring-async-annotation-5.x
+- spring-cloud-feign-1.x
+- spring-cloud-feign-2.x
+- spring-cloud-gateway-2.0.x
+- spring-cloud-gateway-2.1.x
+- spring-concurrent-util-4.x
+- spring-core-patch
+- spring-kafka-2.x
+- spring-mvc-annotation
+- spring-mvc-annotation-3.x
+- spring-mvc-annotation-4.x
+- spring-mvc-annotation-5.x
+- spring-resttemplate-4.x
+- spring-tx
+- spring-webflux-5.x
+- spymemcached-2.x
+- struts2-2.x
+- tomcat-7.x/8.x
+- toolkit-counter
+- toolkit-gauge
+- toolkit-histogram
+- toolkit-kafka
+- toolkit-log4j
+- toolkit-log4j2
+- toolkit-logback
+- toolkit-opentracing
+- toolkit-tag
+- toolkit-trace
+- undertow-2.x-plugin
+- vertx-core-3.x
+- zookeeper-3.4.x
\ No newline at end of file

docs/en/setup/service-agent/java-agent/README.md

@@ -111,6 +111,8 @@ property key | Description | Default |
 `meter.report_interval`|Report meters interval. The unit is second|`20`|
 `meter.max_meter_size`| Max size of the meter pool |`500`|
 `plugin.peer_max_length `|Peer maximum description limit.|`200`|
+`plugin.exclude_plugins `|Exclude some plugins define in plugins dir.Plugin names is defined in [Agent plugin list](Plugin-list.md)|`""`|
+
 `plugin.mongodb.trace_param`|If true, trace all the parameters in MongoDB access, default is false. Only trace the operation, not include parameters.|`false`|
 `plugin.mongodb.filter_length_limit`|If set to positive number, the `WriteRequest.params` would be truncated to this length, otherwise it would be completely saved, which may cause performance problem.|`256`|
 `plugin.elasticsearch.trace_dsl`|If true, trace all the DSL(Domain Specific Language) in ElasticSearch access, default is false.|`false`|

tools/plugin/check-javaagent-plugin-list.sh

+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+SRC_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+WORK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &&cd ../.. && pwd)"
+GENERNATE_PLUGINS_LIST=${SRC_DIR}/genernate-javaagent-plugin-list.txt
+MD_PLUGINS_LIST=${SRC_DIR}/md-javaagent-plugin-list.txt
+
+function genernateJavaagentPluginList() {
+    position_file="javaagent-position.txt"
+    find ${WORK_DIR}/apm-sniffer -name "skywalking-plugin.def"|grep "src/main/resources" > ${position_file}
+    cat ${position_file} | while read line
+    do
+        cat ${line}|grep -v "#"|awk -F "=" '{print $1}' >> temp.txt
+    done
+    cat temp.txt|sort|uniq|awk NF|grep -E '^[a-z].*'  > ${GENERNATE_PLUGINS_LIST}
+    rm -rf temp.txt
+}
+
+function getMdJavaagentPluginList() {
+    md_javaagent_plugins_file=${WORK_DIR}/docs/en/setup/service-agent/java-agent/Plugin-list.md
+    cat  ${md_javaagent_plugins_file}|grep -v "#" |awk -F " " '{ print $2}'|grep -E '^[a-z].*'|sort|uniq|awk NF >${MD_PLUGINS_LIST}
+}
+
+genernateJavaagentPluginList
+getMdJavaagentPluginList
+diff -w -bB -U0 ${MD_PLUGINS_LIST} ${GENERNATE_PLUGINS_LIST}
+
+