Support nacos grouped dynamic configurations. (#7632)

CHANGES.md

@@ -49,6 +49,7 @@ Release Notes.
 * Fix dynamic configuration watch implementation current value not null when the config is deleted.
 * Fix `LoggingConfigWatcher` return `watch.value` would not consistent with the real configuration content.
 * Fix `ZookeeperConfigWatcherRegister.readConfig()` could cause `NPE` when `data.getData()` is null.
+* Support nacos grouped dynamic configurations.
 
 #### UI
 

docs/en/setup/backend/dynamic-config-nacos.md

@@ -18,4 +18,65 @@ configuration:
     period: ${SW_CONFIG_NACOS_PERIOD:60}
     # the name of current cluster, set the name if you want to upstream system known.
     clusterName: ${SW_CONFIG_NACOS_CLUSTER_NAME:default}
-```
\ No newline at end of file
+```
+
+## Config Storage
+### Single Config
+
+| Data Id | Group | Config Value |
+|-----|-----|-----|
+| configKey | {group} | configValue |
+
+e.g. The config is:
+```
+{agent-analyzer.default.slowDBAccessThreshold}:{default:200,mongodb:50}
+```
+If `group = skywalking` the config in nacos is:
+
+| Data Id | Group | Config Value |
+|-----|-----|-----|
+| agent-analyzer.default.slowDBAccessThreshold | skywalking | default:200,mongodb:50 |
+
+### Group Config
+
+| Data Id | Group | Config Value | Config Type |
+|-----|-----|-----|-----|
+| configKey | {group} | subItemkey1</br>subItemkey2</br>... | TEXT |
+| subItemkey1 | {group} | subItemValue1 |
+| subItemkey2 | {group} | subItemValue2 |
+| ... | ... | ... |
+
+Notice: If you add/remove a subItem, you need to add/remove the subItemKey from the group which the subItem belongs:
+
+| Data Id | Group | Config Value | Config Type |
+|-----|-----|-----|-----|
+| configKey | {group} | subItemkey1</br>subItemkey2</br>... | TEXT |
+
+We separate subItemkeys by `\n` or `\r\n`, trim leading and trailing whitespace, if you set the config by `Nacos UI` each subItemkey should in a new line:
+```
+subItemValue1
+subItemValue2
+...
+
+```
+If you set the config by `API` each subItemkey should separated by `\n` or `\r\n`:
+```
+configService.publishConfig("test-module.default.testKeyGroup", "skywalking", "subItemkey1\n subItemkey2"));
+```
+
+e.g. The config is:
+```
+{core.default.endpoint-name-grouping-openapi}:|{customerAPI-v1}:{value of customerAPI-v1}
+                                              |{productAPI-v1}:{value of productAPI-v1}
+                                              |{productAPI-v2}:{value of productAPI-v2}
+```
+If `group = skywalking` the config in nacos is:
+
+| Data Id | Group | Config Value | Config Type |
+|-----|-----|-----|-----|
+| core.default.endpoint-name-grouping-openapi | skywalking | customerAPI-v1</br>productAPI-v1</br>productAPI-v2 | TEXT |
+| customerAPI-v1 | skywalking | value of customerAPI-v1 |
+| productAPI-v1 | skywalking | value of productAPI-v1 |
+| productAPI-v2 | skywalking | value of productAPI-v2 |
+
+

oap-server/server-configuration/configuration-nacos/src/main/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigWatcherRegister.java

@@ -23,6 +23,7 @@ import com.alibaba.nacos.api.PropertyKeyConst;
 import com.alibaba.nacos.api.config.ConfigService;
 import com.alibaba.nacos.api.config.listener.Listener;
 import com.alibaba.nacos.api.exception.NacosException;
+import java.util.Arrays;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Optional;
@@ -31,16 +32,14 @@ import java.util.Set;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.Executor;
 
+import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.apm.util.StringUtil;
 import org.apache.skywalking.oap.server.configuration.api.ConfigTable;
 import org.apache.skywalking.oap.server.configuration.api.ConfigWatcherRegister;
 import org.apache.skywalking.oap.server.configuration.api.GroupConfigTable;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
+@Slf4j
 public class NacosConfigWatcherRegister extends ConfigWatcherRegister {
-    private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigWatcherRegister.class);
-
     private final NacosServerSettings settings;
     private final ConfigService configService;
     private final Map<String, Optional<String>> configItemKeyedByName;
@@ -92,8 +91,32 @@ public class NacosConfigWatcherRegister extends ConfigWatcherRegister {
 
     @Override
     public Optional<GroupConfigTable> readGroupConfig(final Set<String> keys) {
-        // TODO: implement readGroupConfig
-        return Optional.empty();
+        GroupConfigTable groupConfigTable = new GroupConfigTable();
+        keys.forEach(key -> {
+            GroupConfigTable.GroupConfigItems groupConfigItems = new GroupConfigTable.GroupConfigItems(key);
+            groupConfigTable.addGroupConfigItems(groupConfigItems);
+            String config = null;
+            try {
+                config = configService.getConfig(key, settings.getGroup(), 1000);
+                if (StringUtil.isNotEmpty(config)) {
+                    String[] itemNames = config.split("\\n|\\r\\n");
+                    Arrays.stream(itemNames).map(String::trim).forEach(itemName -> {
+                        String itemValue = null;
+                        try {
+                            itemValue = configService.getConfig(itemName, settings.getGroup(), 1000);
+                        } catch (NacosException e) {
+                            log.error("Failed to register Nacos listener for dataId: {}", itemName, e);
+                        }
+                        groupConfigItems.add(
+                            new ConfigTable.ConfigItem(itemName, itemValue));
+                    });
+                }
+            } catch (NacosException e) {
+                log.error("Failed to register Nacos listener for dataId: {}", key, e);
+            }
+        });
+
+        return Optional.of(groupConfigTable);
     }
 
     private void registerKeyListeners(final Set<String> keys) {
@@ -121,7 +144,7 @@ public class NacosConfigWatcherRegister extends ConfigWatcherRegister {
                 final String config = configService.getConfig(dataId, group, 1000);
                 onDataIdValueChanged(dataId, config);
             } catch (NacosException e) {
-                LOGGER.warn("Failed to register Nacos listener for dataId: {}", dataId);
+                log.warn("Failed to register Nacos listener for dataId: {}", dataId);
             }
         }
     }
@@ -141,8 +164,8 @@ public class NacosConfigWatcherRegister extends ConfigWatcherRegister {
     }
 
     void onDataIdValueChanged(String dataId, String configInfo) {
-        if (LOGGER.isInfoEnabled()) {
-            LOGGER.info("Nacos config changed: {}: {}", dataId, configInfo);
+        if (log.isInfoEnabled()) {
+            log.info("Nacos config changed: {}: {}", dataId, configInfo);
         }
 
         configItemKeyedByName.put(dataId, Optional.ofNullable(configInfo));

oap-server/server-configuration/configuration-nacos/src/main/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigurationProvider.java

@@ -20,20 +20,18 @@ package org.apache.skywalking.oap.server.configuration.nacos;
 
 import com.alibaba.nacos.api.exception.NacosException;
 import com.google.common.base.Strings;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.apm.util.StringUtil;
 import org.apache.skywalking.oap.server.configuration.api.AbstractConfigurationProvider;
 import org.apache.skywalking.oap.server.configuration.api.ConfigWatcherRegister;
 import org.apache.skywalking.oap.server.library.module.ModuleConfig;
 import org.apache.skywalking.oap.server.library.module.ModuleStartException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * Get configuration from Nacos.
  */
+@Slf4j
 public class NacosConfigurationProvider extends AbstractConfigurationProvider {
-    private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurationProvider.class);
-
     private NacosServerSettings settings;
 
     public NacosConfigurationProvider() {
@@ -52,7 +50,7 @@ public class NacosConfigurationProvider extends AbstractConfigurationProvider {
 
     @Override
     protected ConfigWatcherRegister initConfigReader() throws ModuleStartException {
-        LOGGER.info("settings: {}", settings);
+        log.info("settings: {}", settings);
         if (Strings.isNullOrEmpty(settings.getServerAddr())) {
             throw new ModuleStartException("Nacos serverAddr cannot be null or empty.");
         }

oap-server/server-configuration/configuration-nacos/src/test/java/org/apache/skywalking/oap/server/configuration/nacos/ITNacosConfigurationTest.java

@@ -25,6 +25,7 @@ import java.io.FileNotFoundException;
 import java.io.Reader;
 import java.util.Map;
 import java.util.Properties;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.apm.util.PropertyPlaceholderHelper;
 import org.apache.skywalking.oap.server.library.module.ApplicationConfiguration;
 import org.apache.skywalking.oap.server.library.module.ModuleManager;
@@ -32,8 +33,6 @@ import org.apache.skywalking.oap.server.library.util.CollectionUtils;
 import org.apache.skywalking.oap.server.library.util.ResourceUtils;
 import org.junit.Before;
 import org.junit.Test;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 import org.yaml.snakeyaml.Yaml;
 
 import static org.junit.Assert.assertEquals;
@@ -41,9 +40,8 @@ import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
 import static org.junit.Assert.assertTrue;
 
+@Slf4j
 public class ITNacosConfigurationTest {
-    private static final Logger LOGGER = LoggerFactory.getLogger(ITNacosConfigurationTest.class);
-
     private final Yaml yaml = new Yaml();
 
     private NacosConfigurationTestProvider provider;
@@ -69,7 +67,7 @@ public class ITNacosConfigurationTest {
         final Properties properties = new Properties();
         final String nacosHost = System.getProperty("nacos.host");
         final String nacosPort = System.getProperty("nacos.port");
-        LOGGER.info("nacosHost: {}, nacosPort: {}", nacosHost, nacosPort);
+        log.info("nacosHost: {}, nacosPort: {}", nacosHost, nacosPort);
         properties.put("serverAddr", nacosHost + ":" + nacosPort);
 
         final ConfigService configService = NacosFactory.createConfigService(properties);
@@ -88,6 +86,50 @@ public class ITNacosConfigurationTest {
         assertNull(provider.watcher.value());
     }
 
+    @Test(timeout = 20000)
+    public void shouldReadUpdatedGroup() throws NacosException {
+        assertNull(provider.watcher.value());
+
+        final Properties properties = new Properties();
+        final String nacosHost = System.getProperty("nacos.host");
+        final String nacosPort = System.getProperty("nacos.port");
+        log.info("nacosHost: {}, nacosPort: {}", nacosHost, nacosPort);
+        properties.put("serverAddr", nacosHost + ":" + nacosPort);
+
+        final ConfigService configService = NacosFactory.createConfigService(properties);
+        //test add group key and item1 item2
+        assertTrue(configService.publishConfig("test-module.default.testKeyGroup", "skywalking", "item1\n item2"));
+        assertTrue(configService.publishConfig("item1", "skywalking", "100"));
+        assertTrue(configService.publishConfig("item2", "skywalking", "200"));
+        for (String v = provider.groupWatcher.groupItems().get("item1"); v == null; v = provider.groupWatcher.groupItems().get("item1")) {
+        }
+        for (String v = provider.groupWatcher.groupItems().get("item2"); v == null; v = provider.groupWatcher.groupItems().get("item2")) {
+        }
+        assertEquals("100", provider.groupWatcher.groupItems().get("item1"));
+        assertEquals("200", provider.groupWatcher.groupItems().get("item2"));
+
+        //test remove item1
+        assertTrue(configService.removeConfig("item1", "skywalking"));
+        for (String v = provider.groupWatcher.groupItems().get("item1"); v != null; v = provider.groupWatcher.groupItems().get("item1")) {
+        }
+        assertNull(provider.groupWatcher.groupItems().get("item1"));
+
+        //test modify item1
+        assertTrue(configService.publishConfig("item1", "skywalking", "300"));
+        for (String v = provider.groupWatcher.groupItems().get("item1"); v == null; v = provider.groupWatcher.groupItems().get("item1")) {
+        }
+        assertEquals("300", provider.groupWatcher.groupItems().get("item1"));
+
+        //test remove group key
+        assertTrue(configService.removeConfig("test-module.default.testKeyGroup", "skywalking"));
+        for (String v = provider.groupWatcher.groupItems().get("item2"); v != null; v = provider.groupWatcher.groupItems().get("item2")) {
+        }
+        assertNull(provider.groupWatcher.groupItems().get("item2"));
+        //chean
+        assertTrue(configService.removeConfig("item1", "skywalking"));
+        assertTrue(configService.removeConfig("item2", "skywalking"));
+    }
+
     @SuppressWarnings("unchecked")
     private void loadConfig(ApplicationConfiguration configuration) throws FileNotFoundException {
         Reader applicationReader = ResourceUtils.read("application.yml");

oap-server/server-configuration/configuration-nacos/src/test/java/org/apache/skywalking/oap/server/configuration/nacos/NacosConfigurationTestProvider.java

@@ -18,21 +18,23 @@
 
 package org.apache.skywalking.oap.server.configuration.nacos;
 
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import lombok.extern.slf4j.Slf4j;
 import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher;
 import org.apache.skywalking.oap.server.configuration.api.ConfigurationModule;
 import org.apache.skywalking.oap.server.configuration.api.DynamicConfigurationService;
+import org.apache.skywalking.oap.server.configuration.api.GroupConfigChangeWatcher;
 import org.apache.skywalking.oap.server.library.module.ModuleConfig;
 import org.apache.skywalking.oap.server.library.module.ModuleDefine;
 import org.apache.skywalking.oap.server.library.module.ModuleProvider;
 import org.apache.skywalking.oap.server.library.module.ModuleStartException;
 import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
+@Slf4j
 public class NacosConfigurationTestProvider extends ModuleProvider {
-    private static final Logger LOGGER = LoggerFactory.getLogger(NacosConfigurationTestProvider.class);
-
     ConfigChangeWatcher watcher;
+    GroupConfigChangeWatcher groupWatcher;
 
     @Override
     public String name() {
@@ -57,7 +59,7 @@ public class NacosConfigurationTestProvider extends ModuleProvider {
 
             @Override
             public void notify(ConfigChangeEvent value) {
-                LOGGER.info("ConfigChangeWatcher.ConfigChangeEvent: {}", value);
+                log.info("ConfigChangeWatcher.ConfigChangeEvent: {}", value);
                 if (EventType.DELETE.equals(value.getEventType())) {
                     testValue = null;
                 } else {
@@ -70,6 +72,27 @@ public class NacosConfigurationTestProvider extends ModuleProvider {
                 return testValue;
             }
         };
+
+        groupWatcher = new GroupConfigChangeWatcher(NacosConfigurationTestModule.NAME, this, "testKeyGroup") {
+            private Map<String, String> config = new ConcurrentHashMap<>();
+
+            @Override
+            public void notifyGroup(Map<String, ConfigChangeEvent> groupItems) {
+                log.info("GroupConfigChangeWatcher.ConfigChangeEvents: {}", groupItems);
+                groupItems.forEach((groupItemName, event) -> {
+                    if (EventType.DELETE.equals(event.getEventType())) {
+                        config.remove(groupItemName);
+                    } else {
+                        config.put(groupItemName, event.getNewValue());
+                    }
+                });
+            }
+
+            @Override
+            public Map<String, String> groupItems() {
+                return config;
+            }
+        };
     }
 
     @Override
@@ -78,6 +101,11 @@ public class NacosConfigurationTestProvider extends ModuleProvider {
                     .provider()
                     .getService(DynamicConfigurationService.class)
                     .registerConfigChangeWatcher(watcher);
+
+        getManager().find(ConfigurationModule.NAME)
+                    .provider()
+                    .getService(DynamicConfigurationService.class)
+                    .registerConfigChangeWatcher(groupWatcher);
     }
 
     @Override