diff --git a/CHANGES.md b/CHANGES.md index d9c7e13b1326300b36cf430ace039ac684afffd8..52b6d1a3c64e35d124038707f357f169ae133a8a 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -68,6 +68,7 @@ Release Notes. * Fix `LogHandler` of `kafka-fetcher-plugin` cannot recognize namespace. * Improve the speed of writing TiDB by batching the SQL execution. * Fix wrong service name when IP is node IP in `k8s-mesh`. +* Support dynamic configurations for openAPI endpoint name grouping rule. #### UI diff --git a/docs/en/setup/backend/dynamic-config.md b/docs/en/setup/backend/dynamic-config.md index 6cc46fa4374ad89138c2600bedce8d7c95fb33df..b233bcf1cb58e8cfc8106670157c2743074e6c96 100755 --- a/docs/en/setup/backend/dynamic-config.md +++ b/docs/en/setup/backend/dynamic-config.md @@ -40,7 +40,7 @@ Supported configurations are as follows: |configuration-discovery.default.agentConfigurations| The ConfigurationDiscovery settings. | See [`configuration-discovery.md`](https://github.com/apache/skywalking-java/blob/20fb8c81b3da76ba6628d34c12d23d3d45c973ef/docs/en/setup/service-agent/java-agent/configuration-discovery.md). | ## Group Configuration -Single Configuration is a config key that corresponds to a group sub config items. A sub config item is a key value pair. The logic structure is: +Group Configuration is a config key that corresponds to a group sub config items. A sub config item is a key value pair. The logic structure is: ``` {configKey}: |{subItemkey1}:{subItemValue1} |{subItemkey2}:{subItemValue2} @@ -56,6 +56,10 @@ For example: ``` Supported configurations are as follows: +| Config Key | SubItem Key Description | Value Description | Value Format Example | +|:----:|:----:|:----:|:----:| +|core.default.endpoint-name-grouping-openapi|The serviceName relevant to openAPI definition file. eg. `serviceA`. If the serviceName relevant to multiple files should add subItems for each files, and each subItem key should split serviceName and fileName with `.` eg. `serviceA.API-file1`,`serviceA.API-file2` |The openAPI definitions file contents(yaml format) for create endpoint name grouping rules.|Same as [`productAPI-v2.yaml`](endpoint-grouping-rules.md)| + ## Dynamic Configuration Implementations - [Dynamic Configuration Service, DCS](./dynamic-config-service.md) - [Zookeeper Implementation](./dynamic-config-zookeeper.md) diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index 681ec9e7fc73bb04b84d83582dc3a1d83f06f10a..1802b332d81b1badc4215387d620dcf4a1d4b2df 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -285,6 +285,10 @@ Here are some use cases: | `GET:/products/123` | serviceB | default | default | `${PATH}:<${METHOD}>` | true | `/products/{id}:` | | `/products/123:` | serviceB | default | `${PATH}:<${METHOD}>` | default | true | `GET:/products/{id}` | +### Initialize and update the OpenAPI definitions dynamically +Use [Dynamic Configuration](dynamic-config) to initialize and update OpenAPI definitions, the endpoint grouping rules from OpenAPI +will re-create by new config. + ## Endpoint name grouping by custom configuration Currently, a user could set up grouping rules through the static YAML file named `endpoint-name-grouping.yml`, diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index edd70849c287c840349d2e24119103d39f087aa3..87fd5a3e32b84f9aff7dbca388f889941f702765 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -48,7 +48,7 @@ import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogServ import org.apache.skywalking.oap.server.core.config.NamingControl; import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; import org.apache.skywalking.oap.server.core.config.group.EndpointNameGroupingRuleWatcher; -import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4Openapi; +import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointNameGroupingRule4OpenapiWatcher; import org.apache.skywalking.oap.server.core.logging.LoggingConfigWatcher; import org.apache.skywalking.oap.server.core.management.ui.template.UITemplateInitializer; import org.apache.skywalking.oap.server.core.management.ui.template.UITemplateManagementService; @@ -123,6 +123,7 @@ public class CoreModuleProvider extends ModuleProvider { private EndpointNameGroupingRuleWatcher endpointNameGroupingRuleWatcher; private OALEngineLoaderService oalEngineLoaderService; private LoggingConfigWatcher loggingConfigWatcher; + private EndpointNameGroupingRule4OpenapiWatcher endpointNameGroupingRule4OpenapiWatcher; public CoreModuleProvider() { super(); @@ -164,8 +165,8 @@ public class CoreModuleProvider extends ModuleProvider { this, endpointNameGrouping); if (moduleConfig.isEnableEndpointNameGroupingByOpenapi()) { - endpointNameGrouping.setEndpointGroupingRule4Openapi( - new EndpointGroupingRuleReader4Openapi("openapi-definitions").read()); + endpointNameGroupingRule4OpenapiWatcher = new EndpointNameGroupingRule4OpenapiWatcher( + this, endpointNameGrouping); } } catch (FileNotFoundException e) { throw new ModuleStartException(e.getMessage(), e); @@ -354,6 +355,9 @@ public class CoreModuleProvider extends ModuleProvider { dynamicConfigurationService.registerConfigChangeWatcher(apdexThresholdConfig); dynamicConfigurationService.registerConfigChangeWatcher(endpointNameGroupingRuleWatcher); dynamicConfigurationService.registerConfigChangeWatcher(loggingConfigWatcher); + if (moduleConfig.isEnableEndpointNameGroupingByOpenapi()) { + dynamicConfigurationService.registerConfigChangeWatcher(endpointNameGroupingRule4OpenapiWatcher); + } } @Override diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java index ab945e7aea7afe17035cc4d92a02ed14f3d2e0b9..366f3feeb51aff46b1f57054d5bf93060ceb15b2 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java @@ -22,6 +22,8 @@ import java.io.File; import java.io.FileNotFoundException; import java.io.FileReader; import java.io.Reader; +import java.io.StringReader; +import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.List; @@ -32,8 +34,7 @@ import org.yaml.snakeyaml.Yaml; import org.yaml.snakeyaml.constructor.SafeConstructor; public class EndpointGroupingRuleReader4Openapi { - - private final String openapiDefPath; + private final Map/*openapiData*/> serviceOpenapiDefMap; private final static String DEFAULT_ENDPOINT_NAME_FORMAT = "${METHOD}:${PATH}"; private final static String DEFAULT_ENDPOINT_NAME_MATCH_RULE = "${METHOD}:${PATH}"; private final Map requestMethodsMap = new HashMap() { @@ -49,51 +50,87 @@ public class EndpointGroupingRuleReader4Openapi { } }; - public EndpointGroupingRuleReader4Openapi(final String openapiDefPath) { + public EndpointGroupingRuleReader4Openapi(final String openapiDefPath) throws FileNotFoundException { + this.serviceOpenapiDefMap = this.parseFromDir(openapiDefPath); + } - this.openapiDefPath = openapiDefPath; + public EndpointGroupingRuleReader4Openapi(final Map openapiDefsConf) { + this.serviceOpenapiDefMap = this.parseFromDynamicConf(openapiDefsConf); } - public EndpointGroupingRule4Openapi read() throws FileNotFoundException { + public EndpointGroupingRule4Openapi read() { EndpointGroupingRule4Openapi endpointGroupingRule = new EndpointGroupingRule4Openapi(); - - List fileList = ResourceUtils.getDirectoryFilesRecursive(openapiDefPath, 1); - for (File file : fileList) { - if (!file.getName().endsWith(".yaml")) { - continue; - } - Reader reader = new FileReader(file); - Yaml yaml = new Yaml(new SafeConstructor()); - Map openapiData = yaml.load(reader); - if (openapiData != null) { - String serviceName = getServiceName(openapiData, file); + serviceOpenapiDefMap.forEach((serviceName, openapiDefs) -> { + openapiDefs.forEach(openapiData -> { LinkedHashMap> paths = - (LinkedHashMap>) openapiData.get("paths"); - + (LinkedHashMap>) openapiData.get( + "paths"); if (paths != null) { paths.forEach((pathString, pathItem) -> { pathItem.keySet().forEach(key -> { String requestMethod = requestMethodsMap.get(key); if (!StringUtil.isEmpty(requestMethod)) { - String endpointGroupName = formatEndPointName(pathString, requestMethod, openapiData); - String groupRegex = getGroupRegex(pathString, requestMethod, openapiData); + String endpointGroupName = formatEndPointName( + pathString, requestMethod, openapiData); + String groupRegex = getGroupRegex( + pathString, requestMethod, openapiData); if (isTemplatePath(pathString)) { - endpointGroupingRule.addGroupedRule(serviceName, endpointGroupName, groupRegex); + endpointGroupingRule.addGroupedRule( + serviceName, endpointGroupName, groupRegex); } else { - endpointGroupingRule.addDirectLookup(serviceName, groupRegex, endpointGroupName); + endpointGroupingRule.addDirectLookup( + serviceName, groupRegex, endpointGroupName); } } }); }); } - } - } + }); + }); endpointGroupingRule.sortRulesAll(); return endpointGroupingRule; } - private String getServiceName(Map openapiData, File file) { + private Map> parseFromDir(String openapiDefPath) throws FileNotFoundException { + Map> serviceOpenapiDefMap = new HashMap<>(); + List fileList = ResourceUtils.getDirectoryFilesRecursive(openapiDefPath, 1); + for (File file : fileList) { + if (!file.getName().endsWith(".yaml")) { + continue; + } + Reader reader = new FileReader(file); + Yaml yaml = new Yaml(new SafeConstructor()); + Map openapiData = yaml.load(reader); + if (openapiData != null) { + serviceOpenapiDefMap.computeIfAbsent(getServiceName(openapiDefPath, file, openapiData), k -> new ArrayList<>()).add(openapiData); + } + } + + return serviceOpenapiDefMap; + } + private Map> parseFromDynamicConf(final Map openapiDefsConf) { + Map> serviceOpenapiDefMap = new HashMap<>(); + openapiDefsConf.forEach((itemName, openapiDefs) -> { + String serviceName = itemName; + //service map to multiple openapiDefs + String[] itemNameInfo = itemName.split("\\."); + if (itemNameInfo.length > 1) { + serviceName = itemNameInfo[0]; + } + Reader reader = new StringReader(openapiDefs); + Yaml yaml = new Yaml(new SafeConstructor()); + Map openapiData = yaml.load(reader); + if (openapiData != null) { + serviceOpenapiDefMap.computeIfAbsent(getServiceName(serviceName, openapiData), k -> new ArrayList<>()) + .add(openapiData); + } + }); + + return serviceOpenapiDefMap; + } + + private String getServiceName(String openapiDefPath, File file, Map openapiData) { String serviceName = (String) openapiData.get("x-sw-service-name"); if (StringUtil.isEmpty(serviceName)) { File directory = new File(file.getParent()); @@ -107,6 +144,15 @@ public class EndpointGroupingRuleReader4Openapi { return serviceName; } + private String getServiceName(String defaultServiceName, Map openapiData) { + String serviceName = (String) openapiData.get("x-sw-service-name"); + if (StringUtil.isEmpty(serviceName)) { + serviceName = defaultServiceName; + } + + return serviceName; + } + private boolean isTemplatePath(String pathString) { return pathString.matches("(.*)\\{(.+?)}(.*)"); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointNameGroupingRule4OpenapiWatcher.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointNameGroupingRule4OpenapiWatcher.java new file mode 100644 index 0000000000000000000000000000000000000000..f562b5352165c4c0ac1e43c7f6971002ca872e45 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointNameGroupingRule4OpenapiWatcher.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.openapi; + +import java.io.FileNotFoundException; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.configuration.api.GroupConfigChangeWatcher; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; + +@Slf4j +public class EndpointNameGroupingRule4OpenapiWatcher extends GroupConfigChangeWatcher { + private final EndpointNameGrouping grouping; + private final Map openapiDefs; + + public EndpointNameGroupingRule4OpenapiWatcher(ModuleProvider provider, + EndpointNameGrouping grouping) throws FileNotFoundException { + super(CoreModule.NAME, provider, "endpoint-name-grouping-openapi"); + this.grouping = grouping; + this.openapiDefs = new ConcurrentHashMap<>(); + this.grouping.setEndpointGroupingRule4Openapi( + new EndpointGroupingRuleReader4Openapi("openapi-definitions").read()); + } + + @Override + public Map groupItems() { + return openapiDefs; + } + + @Override + public void notifyGroup(final Map groupItems) { + groupItems.forEach((groupItemName, event) -> { + if (EventType.DELETE.equals(event.getEventType())) { + this.openapiDefs.remove(groupItemName); + log.info("EndpointNameGroupingRule4OpenapiWatcher removed groupItem: {}", groupItemName); + } else { + this.openapiDefs.put(groupItemName, event.getNewValue()); + log.info("EndpointNameGroupingRule4OpenapiWatcher modified groupItem: {}", groupItemName); + } + }); + this.grouping.setEndpointGroupingRule4Openapi(new EndpointGroupingRuleReader4Openapi(openapiDefs).read()); + } +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointNameGroupingRule4OpenapiWatcherTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointNameGroupingRule4OpenapiWatcherTest.java new file mode 100644 index 0000000000000000000000000000000000000000..605f9b32004306dacb1ac01fee446800756ac6fa --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointNameGroupingRule4OpenapiWatcherTest.java @@ -0,0 +1,261 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.openapi; + +import java.io.FileNotFoundException; +import java.util.HashMap; +import java.util.Map; +import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.junit.Assert; +import org.junit.Test; + +public class EndpointNameGroupingRule4OpenapiWatcherTest { + @Test + public void testWatcher() throws FileNotFoundException { + EndpointNameGrouping endpointNameGrouping = new EndpointNameGrouping(); + + EndpointNameGroupingRule4OpenapiWatcher watcher = new EndpointNameGroupingRule4OpenapiWatcher( + new ModuleProvider() { + @Override + public String name() { + return "test"; + } + + @Override + public Class module() { + return CoreModule.class; + } + + @Override + public ModuleConfig createConfigBeanIfAbsent() { + return null; + } + + @Override + public void prepare() throws ServiceNotProvidedException { + + } + + @Override + public void start() throws ServiceNotProvidedException { + + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException { + + } + + @Override + public String[] requiredModules() { + return new String[0]; + } + }, endpointNameGrouping); + Assert.assertEquals("GET:/products/{id}", endpointNameGrouping.format("serviceA", "GET:/products/123")); + + Map groupItems = new HashMap<>(); + groupItems.put( + "serviceA.productAPI-v1", + new ConfigChangeWatcher + .ConfigChangeEvent( + "openapi: 3.0.0\n" + + "\n" + + "info:\n" + + " description: OpenAPI definition for SkyWalking test.\n" + + " version: v1\n" + + " title: Product API\n" + + "\n" + + "tags:\n" + + " - name: product\n" + + " description: product\n" + + " - name: relatedProducts\n" + + " description: Related Products\n" + + "\n" + + "paths:\n" + + " /products:\n" + + " get:\n" + + " tags:\n" + + " - product\n" + + " summary: Get all products list\n" + + " description: Get all products list.\n" + + " operationId: getProducts\n" + + " responses:\n" + + " \"200\":\n" + + " description: Success\n" + + " content:\n" + + " application/json:\n" + + " schema:\n" + + " type: array\n" + + " items:\n" + + " $ref: \"#/components/schemas/Product\"\n" + + " /products/{order-id}:\n" + //modified from /products/{id} + " get:\n" + + " tags:\n" + + " - product\n" + + " summary: Get product details\n" + + " description: Get product details with the given id.\n" + + " operationId: getProduct\n" + + " parameters:\n" + + " - name: id\n" + + " in: path\n" + + " description: Product id\n" + + " required: true\n" + + " schema:\n" + + " type: integer\n" + + " format: int64\n" + + " responses:\n" + + " \"200\":\n" + + " description: successful operation\n" + + " content:\n" + + " application/json:\n" + + " schema:\n" + + " $ref: \"#/components/schemas/ProductDetails\"\n" + + " \"400\":\n" + + " description: Invalid product id\n" + + " post:\n" + + " tags:\n" + + " - product\n" + + " summary: Update product details\n" + + " description: Update product details with the given id.\n" + + " operationId: updateProduct\n" + + " parameters:\n" + + " - name: id\n" + + " in: path\n" + + " description: Product id\n" + + " required: true\n" + + " schema:\n" + + " type: integer\n" + + " format: int64\n" + + " - name: name\n" + + " in: query\n" + + " description: Product name\n" + + " required: true\n" + + " schema:\n" + + " type: string\n" + + " responses:\n" + + " \"200\":\n" + + " description: successful operation\n" + + " delete:\n" + + " tags:\n" + + " - product\n" + + " summary: Delete product details\n" + + " description: Delete product details with the given id.\n" + + " operationId: deleteProduct\n" + + " parameters:\n" + + " - name: id\n" + + " in: path\n" + + " description: Product id\n" + + " required: true\n" + + " schema:\n" + + " type: integer\n" + + " format: int64\n" + + " responses:\n" + + " \"200\":\n" + + " description: successful operation\n" + + " /products/{id}/relatedProducts:\n" + + " get:\n" + + " tags:\n" + + " - relatedProducts\n" + + " summary: Get related products\n" + + " description: Get related products with the given product id.\n" + + " operationId: getRelatedProducts\n" + + " parameters:\n" + + " - name: id\n" + + " in: path\n" + + " description: Product id\n" + + " required: true\n" + + " schema:\n" + + " type: integer\n" + + " format: int64\n" + + " responses:\n" + + " \"200\":\n" + + " description: successful operation\n" + + " content:\n" + + " application/json:\n" + + " schema:\n" + + " $ref: \"#/components/schemas/RelatedProducts\"\n" + + " \"400\":\n" + + " description: Invalid product id\n" + + "\n" + + "components:\n" + + " schemas:\n" + + " Product:\n" + + " type: object\n" + + " description: Product id and name\n" + + " properties:\n" + + " id:\n" + + " type: integer\n" + + " format: int64\n" + + " description: Product id\n" + + " name:\n" + + " type: string\n" + + " description: Product name\n" + + " required:\n" + + " - id\n" + + " - name\n" + + " ProductDetails:\n" + + " type: object\n" + + " description: Product details\n" + + " properties:\n" + + " id:\n" + + " type: integer\n" + + " format: int64\n" + + " description: Product id\n" + + " name:\n" + + " type: string\n" + + " description: Product name\n" + + " description:\n" + + " type: string\n" + + " description: Product description\n" + + " required:\n" + + " - id\n" + + " - name\n" + + " RelatedProducts:\n" + + " type: object\n" + + " description: Related Products\n" + + " properties:\n" + + " id:\n" + + " type: integer\n" + + " format: int32\n" + + " description: Product id\n" + + " relatedProducts:\n" + + " type: array\n" + + " description: List of related products\n" + + " items:\n" + + " $ref: \"#/components/schemas/Product\"", + ConfigChangeWatcher.EventType.MODIFY + ) + ); + + watcher.notifyGroup(groupItems); + Assert.assertEquals("GET:/products/{order-id}", endpointNameGrouping.format("serviceA", "GET:/products/123")); + + groupItems.put("serviceA.productAPI-v1", new ConfigChangeWatcher.ConfigChangeEvent("", ConfigChangeWatcher.EventType.DELETE)); + watcher.notifyGroup(groupItems); + + Assert.assertEquals("GET:/products/123", endpointNameGrouping.format("serviceA", "GET:/products/123")); + + } +}