Support dynamic configurations for openAPI endpoint name grouping rule (#7746)

CHANGES.md

@@ -68,6 +68,7 @@ Release Notes.
 * Fix `LogHandler` of `kafka-fetcher-plugin` cannot recognize namespace.
 * Improve the speed of writing TiDB by batching the SQL execution.
 * Fix wrong service name when IP is node IP in `k8s-mesh`.
+* Support dynamic configurations for openAPI endpoint name grouping rule.
 
 #### UI
 

docs/en/setup/backend/dynamic-config.md

@@ -40,7 +40,7 @@ Supported configurations are as follows:
 |configuration-discovery.default.agentConfigurations| The ConfigurationDiscovery settings. | See [`configuration-discovery.md`](https://github.com/apache/skywalking-java/blob/20fb8c81b3da76ba6628d34c12d23d3d45c973ef/docs/en/setup/service-agent/java-agent/configuration-discovery.md). |
 
 ## Group Configuration
-Single Configuration is a config key that corresponds to a group sub config items. A sub config item is a key value pair. The logic structure is:
+Group Configuration is a config key that corresponds to a group sub config items. A sub config item is a key value pair. The logic structure is:
 ```
 {configKey}: |{subItemkey1}:{subItemValue1}
              |{subItemkey2}:{subItemValue2}
@@ -56,6 +56,10 @@ For example:
 ```
 Supported configurations are as follows:
 
+| Config Key | SubItem Key Description |  Value Description | Value Format Example |
+|:----:|:----:|:----:|:----:|
+|core.default.endpoint-name-grouping-openapi|The serviceName relevant to openAPI definition file. eg. `serviceA`. If the serviceName relevant to multiple files should add subItems for each files, and each subItem key should split serviceName and fileName with `.` eg. `serviceA.API-file1`,`serviceA.API-file2` |The openAPI definitions file contents(yaml format) for create endpoint name grouping rules.|Same as [`productAPI-v2.yaml`](endpoint-grouping-rules.md)|
+
 ## Dynamic Configuration Implementations
 - [Dynamic Configuration Service, DCS](./dynamic-config-service.md)
 - [Zookeeper Implementation](./dynamic-config-zookeeper.md)

docs/en/setup/backend/endpoint-grouping-rules.md

@@ -285,6 +285,10 @@ Here are some use cases:
    | `GET:/products/123` | serviceB | default | default | `${PATH}:<${METHOD}>` | true | `/products/{id}:<GET>` |
    | `/products/123:<GET>` | serviceB | default | `${PATH}:<${METHOD}>` | default | true | `GET:/products/{id}` |
 
+### Initialize and update the OpenAPI definitions dynamically
+Use [Dynamic Configuration](dynamic-config) to initialize and update OpenAPI definitions, the endpoint grouping rules from OpenAPI
+will re-create by new config.
+
 
 ## Endpoint name grouping by custom configuration
 Currently, a user could set up grouping rules through the static YAML file named `endpoint-name-grouping.yml`,

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java

@@ -48,7 +48,7 @@ import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogServ
 import org.apache.skywalking.oap.server.core.config.NamingControl;
 import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping;
 import org.apache.skywalking.oap.server.core.config.group.EndpointNameGroupingRuleWatcher;
-import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4Openapi;
+import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointNameGroupingRule4OpenapiWatcher;
 import org.apache.skywalking.oap.server.core.logging.LoggingConfigWatcher;
 import org.apache.skywalking.oap.server.core.management.ui.template.UITemplateInitializer;
 import org.apache.skywalking.oap.server.core.management.ui.template.UITemplateManagementService;
@@ -123,6 +123,7 @@ public class CoreModuleProvider extends ModuleProvider {
     private EndpointNameGroupingRuleWatcher endpointNameGroupingRuleWatcher;
     private OALEngineLoaderService oalEngineLoaderService;
     private LoggingConfigWatcher loggingConfigWatcher;
+    private EndpointNameGroupingRule4OpenapiWatcher endpointNameGroupingRule4OpenapiWatcher;
 
     public CoreModuleProvider() {
         super();
@@ -164,8 +165,8 @@ public class CoreModuleProvider extends ModuleProvider {
                 this, endpointNameGrouping);
 
             if (moduleConfig.isEnableEndpointNameGroupingByOpenapi()) {
-                endpointNameGrouping.setEndpointGroupingRule4Openapi(
-                    new EndpointGroupingRuleReader4Openapi("openapi-definitions").read());
+                endpointNameGroupingRule4OpenapiWatcher = new EndpointNameGroupingRule4OpenapiWatcher(
+                    this, endpointNameGrouping);
             }
         } catch (FileNotFoundException e) {
             throw new ModuleStartException(e.getMessage(), e);
@@ -354,6 +355,9 @@ public class CoreModuleProvider extends ModuleProvider {
         dynamicConfigurationService.registerConfigChangeWatcher(apdexThresholdConfig);
         dynamicConfigurationService.registerConfigChangeWatcher(endpointNameGroupingRuleWatcher);
         dynamicConfigurationService.registerConfigChangeWatcher(loggingConfigWatcher);
+        if (moduleConfig.isEnableEndpointNameGroupingByOpenapi()) {
+            dynamicConfigurationService.registerConfigChangeWatcher(endpointNameGroupingRule4OpenapiWatcher);
+        }
     }
 
     @Override

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java

@@ -22,6 +22,8 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
 import java.io.Reader;
+import java.io.StringReader;
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.List;
@@ -32,8 +34,7 @@ import org.yaml.snakeyaml.Yaml;
 import org.yaml.snakeyaml.constructor.SafeConstructor;
 
 public class EndpointGroupingRuleReader4Openapi {
-
-    private final String openapiDefPath;
+    private final Map<String, /*serviceName*/ List<Map>/*openapiData*/> serviceOpenapiDefMap;
     private final static String DEFAULT_ENDPOINT_NAME_FORMAT = "${METHOD}:${PATH}";
     private final static String DEFAULT_ENDPOINT_NAME_MATCH_RULE = "${METHOD}:${PATH}";
     private final Map<String, String> requestMethodsMap = new HashMap<String, String>() {
@@ -49,51 +50,87 @@ public class EndpointGroupingRuleReader4Openapi {
         }
     };
 
-    public EndpointGroupingRuleReader4Openapi(final String openapiDefPath) {
+    public EndpointGroupingRuleReader4Openapi(final String openapiDefPath) throws FileNotFoundException {
+        this.serviceOpenapiDefMap = this.parseFromDir(openapiDefPath);
+    }
 
-        this.openapiDefPath = openapiDefPath;
+    public EndpointGroupingRuleReader4Openapi(final Map<String, String> openapiDefsConf) {
+        this.serviceOpenapiDefMap = this.parseFromDynamicConf(openapiDefsConf);
     }
 
-    public EndpointGroupingRule4Openapi read() throws FileNotFoundException {
+    public EndpointGroupingRule4Openapi read() {
         EndpointGroupingRule4Openapi endpointGroupingRule = new EndpointGroupingRule4Openapi();
-
-        List<File> fileList = ResourceUtils.getDirectoryFilesRecursive(openapiDefPath, 1);
-        for (File file : fileList) {
-            if (!file.getName().endsWith(".yaml")) {
-                continue;
-            }
-            Reader reader = new FileReader(file);
-            Yaml yaml = new Yaml(new SafeConstructor());
-            Map openapiData = yaml.load(reader);
-            if (openapiData != null) {
-                String serviceName = getServiceName(openapiData, file);
+        serviceOpenapiDefMap.forEach((serviceName, openapiDefs) -> {
+            openapiDefs.forEach(openapiData -> {
                 LinkedHashMap<String, LinkedHashMap<String, LinkedHashMap>> paths =
-                    (LinkedHashMap<String, LinkedHashMap<String, LinkedHashMap>>) openapiData.get("paths");
-
+                    (LinkedHashMap<String, LinkedHashMap<String, LinkedHashMap>>) openapiData.get(
+                        "paths");
                 if (paths != null) {
                     paths.forEach((pathString, pathItem) -> {
                         pathItem.keySet().forEach(key -> {
                             String requestMethod = requestMethodsMap.get(key);
                             if (!StringUtil.isEmpty(requestMethod)) {
-                                String endpointGroupName = formatEndPointName(pathString, requestMethod, openapiData);
-                                String groupRegex = getGroupRegex(pathString, requestMethod, openapiData);
+                                String endpointGroupName = formatEndPointName(
+                                    pathString, requestMethod, openapiData);
+                                String groupRegex = getGroupRegex(
+                                    pathString, requestMethod, openapiData);
                                 if (isTemplatePath(pathString)) {
-                                    endpointGroupingRule.addGroupedRule(serviceName, endpointGroupName, groupRegex);
+                                    endpointGroupingRule.addGroupedRule(
+                                        serviceName, endpointGroupName, groupRegex);
                                 } else {
-                                    endpointGroupingRule.addDirectLookup(serviceName, groupRegex, endpointGroupName);
+                                    endpointGroupingRule.addDirectLookup(
+                                        serviceName, groupRegex, endpointGroupName);
                                 }
                             }
                         });
                     });
                 }
-            }
-        }
+            });
+        });
         endpointGroupingRule.sortRulesAll();
         return endpointGroupingRule;
     }
 
-    private String getServiceName(Map openapiData, File file) {
+    private Map<String, List<Map>> parseFromDir(String openapiDefPath) throws FileNotFoundException {
+        Map<String, List<Map>> serviceOpenapiDefMap = new HashMap<>();
+        List<File> fileList = ResourceUtils.getDirectoryFilesRecursive(openapiDefPath, 1);
+        for (File file : fileList) {
+            if (!file.getName().endsWith(".yaml")) {
+                continue;
+            }
+            Reader reader = new FileReader(file);
+            Yaml yaml = new Yaml(new SafeConstructor());
+            Map openapiData = yaml.load(reader);
+            if (openapiData != null) {
+                serviceOpenapiDefMap.computeIfAbsent(getServiceName(openapiDefPath, file, openapiData), k -> new ArrayList<>()).add(openapiData);
+            }
+        }
+
+        return serviceOpenapiDefMap;
+    }
 
+    private Map<String, List<Map>> parseFromDynamicConf(final Map<String, String> openapiDefsConf) {
+        Map<String, List<Map>> serviceOpenapiDefMap = new HashMap<>();
+        openapiDefsConf.forEach((itemName, openapiDefs) -> {
+            String serviceName = itemName;
+            //service map to multiple openapiDefs
+            String[] itemNameInfo = itemName.split("\\.");
+            if (itemNameInfo.length > 1) {
+                serviceName = itemNameInfo[0];
+            }
+            Reader reader = new StringReader(openapiDefs);
+            Yaml yaml = new Yaml(new SafeConstructor());
+            Map openapiData = yaml.load(reader);
+            if (openapiData != null) {
+                serviceOpenapiDefMap.computeIfAbsent(getServiceName(serviceName, openapiData), k -> new ArrayList<>())
+                                    .add(openapiData);
+            }
+        });
+
+        return serviceOpenapiDefMap;
+    }
+
+    private String getServiceName(String openapiDefPath, File file, Map openapiData) {
         String serviceName = (String) openapiData.get("x-sw-service-name");
         if (StringUtil.isEmpty(serviceName)) {
             File directory = new File(file.getParent());
@@ -107,6 +144,15 @@ public class EndpointGroupingRuleReader4Openapi {
         return serviceName;
     }
 
+    private String getServiceName(String defaultServiceName, Map openapiData) {
+        String serviceName = (String) openapiData.get("x-sw-service-name");
+        if (StringUtil.isEmpty(serviceName)) {
+            serviceName = defaultServiceName;
+        }
+
+        return serviceName;
+    }
+
     private boolean isTemplatePath(String pathString) {
         return pathString.matches("(.*)\\{(.+?)}(.*)");
     }

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointNameGroupingRule4OpenapiWatcher.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.config.group.openapi;
+
+import java.io.FileNotFoundException;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import lombok.extern.slf4j.Slf4j;
+import org.apache.skywalking.oap.server.configuration.api.GroupConfigChangeWatcher;
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping;
+import org.apache.skywalking.oap.server.library.module.ModuleProvider;
+
+@Slf4j
+public class EndpointNameGroupingRule4OpenapiWatcher extends GroupConfigChangeWatcher {
+    private final EndpointNameGrouping grouping;
+    private final Map<String, String> openapiDefs;
+
+    public EndpointNameGroupingRule4OpenapiWatcher(ModuleProvider provider,
+                                                   EndpointNameGrouping grouping) throws FileNotFoundException {
+        super(CoreModule.NAME, provider, "endpoint-name-grouping-openapi");
+        this.grouping = grouping;
+        this.openapiDefs = new ConcurrentHashMap<>();
+        this.grouping.setEndpointGroupingRule4Openapi(
+            new EndpointGroupingRuleReader4Openapi("openapi-definitions").read());
+    }
+
+    @Override
+    public Map<String, String> groupItems() {
+        return openapiDefs;
+    }
+
+    @Override
+    public void notifyGroup(final Map<String, ConfigChangeEvent> groupItems) {
+        groupItems.forEach((groupItemName, event) -> {
+            if (EventType.DELETE.equals(event.getEventType())) {
+                this.openapiDefs.remove(groupItemName);
+                log.info("EndpointNameGroupingRule4OpenapiWatcher removed groupItem: {}", groupItemName);
+            } else {
+                this.openapiDefs.put(groupItemName, event.getNewValue());
+                log.info("EndpointNameGroupingRule4OpenapiWatcher modified groupItem: {}", groupItemName);
+            }
+        });
+        this.grouping.setEndpointGroupingRule4Openapi(new EndpointGroupingRuleReader4Openapi(openapiDefs).read());
+    }
+}

oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointNameGroupingRule4OpenapiWatcherTest.java

+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package org.apache.skywalking.oap.server.core.config.group.openapi;
+
+import java.io.FileNotFoundException;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.skywalking.oap.server.configuration.api.ConfigChangeWatcher;
+import org.apache.skywalking.oap.server.core.CoreModule;
+import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping;
+import org.apache.skywalking.oap.server.library.module.ModuleConfig;
+import org.apache.skywalking.oap.server.library.module.ModuleDefine;
+import org.apache.skywalking.oap.server.library.module.ModuleProvider;
+import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class EndpointNameGroupingRule4OpenapiWatcherTest {
+    @Test
+    public void testWatcher() throws FileNotFoundException {
+        EndpointNameGrouping endpointNameGrouping = new EndpointNameGrouping();
+
+        EndpointNameGroupingRule4OpenapiWatcher watcher = new EndpointNameGroupingRule4OpenapiWatcher(
+            new ModuleProvider() {
+                @Override
+                public String name() {
+                    return "test";
+                }
+
+                @Override
+                public Class<? extends ModuleDefine> module() {
+                    return CoreModule.class;
+                }
+
+                @Override
+                public ModuleConfig createConfigBeanIfAbsent() {
+                    return null;
+                }
+
+                @Override
+                public void prepare() throws ServiceNotProvidedException {
+
+                }
+
+                @Override
+                public void start() throws ServiceNotProvidedException {
+
+                }
+
+                @Override
+                public void notifyAfterCompleted() throws ServiceNotProvidedException {
+
+                }
+
+                @Override
+                public String[] requiredModules() {
+                    return new String[0];
+                }
+            }, endpointNameGrouping);
+        Assert.assertEquals("GET:/products/{id}", endpointNameGrouping.format("serviceA", "GET:/products/123"));
+
+        Map<String, ConfigChangeWatcher.ConfigChangeEvent> groupItems = new HashMap<>();
+        groupItems.put(
+            "serviceA.productAPI-v1",
+            new ConfigChangeWatcher
+                .ConfigChangeEvent(
+                "openapi: 3.0.0\n" +
+                    "\n" +
+                    "info:\n" +
+                    "  description: OpenAPI definition for SkyWalking test.\n" +
+                    "  version: v1\n" +
+                    "  title: Product API\n" +
+                    "\n" +
+                    "tags:\n" +
+                    "  - name: product\n" +
+                    "    description: product\n" +
+                    "  - name: relatedProducts\n" +
+                    "    description: Related Products\n" +
+                    "\n" +
+                    "paths:\n" +
+                    "  /products:\n" +
+                    "    get:\n" +
+                    "      tags:\n" +
+                    "        - product\n" +
+                    "      summary: Get all products list\n" +
+                    "      description: Get all products list.\n" +
+                    "      operationId: getProducts\n" +
+                    "      responses:\n" +
+                    "        \"200\":\n" +
+                    "          description: Success\n" +
+                    "          content:\n" +
+                    "            application/json:\n" +
+                    "              schema:\n" +
+                    "                type: array\n" +
+                    "                items:\n" +
+                    "                  $ref: \"#/components/schemas/Product\"\n" +
+                    "  /products/{order-id}:\n" + //modified from /products/{id}
+                    "    get:\n" +
+                    "      tags:\n" +
+                    "        - product\n" +
+                    "      summary: Get product details\n" +
+                    "      description: Get product details with the given id.\n" +
+                    "      operationId: getProduct\n" +
+                    "      parameters:\n" +
+                    "        - name: id\n" +
+                    "          in: path\n" +
+                    "          description: Product id\n" +
+                    "          required: true\n" +
+                    "          schema:\n" +
+                    "            type: integer\n" +
+                    "            format: int64\n" +
+                    "      responses:\n" +
+                    "        \"200\":\n" +
+                    "          description: successful operation\n" +
+                    "          content:\n" +
+                    "            application/json:\n" +
+                    "              schema:\n" +
+                    "                $ref: \"#/components/schemas/ProductDetails\"\n" +
+                    "        \"400\":\n" +
+                    "          description: Invalid product id\n" +
+                    "    post:\n" +
+                    "      tags:\n" +
+                    "        - product\n" +
+                    "      summary: Update product details\n" +
+                    "      description: Update product details with the given id.\n" +
+                    "      operationId: updateProduct\n" +
+                    "      parameters:\n" +
+                    "        - name: id\n" +
+                    "          in: path\n" +
+                    "          description: Product id\n" +
+                    "          required: true\n" +
+                    "          schema:\n" +
+                    "            type: integer\n" +
+                    "            format: int64\n" +
+                    "        - name: name\n" +
+                    "          in: query\n" +
+                    "          description: Product name\n" +
+                    "          required: true\n" +
+                    "          schema:\n" +
+                    "            type: string\n" +
+                    "      responses:\n" +
+                    "        \"200\":\n" +
+                    "          description: successful operation\n" +
+                    "    delete:\n" +
+                    "      tags:\n" +
+                    "        - product\n" +
+                    "      summary: Delete product details\n" +
+                    "      description: Delete product details with the given id.\n" +
+                    "      operationId: deleteProduct\n" +
+                    "      parameters:\n" +
+                    "        - name: id\n" +
+                    "          in: path\n" +
+                    "          description: Product id\n" +
+                    "          required: true\n" +
+                    "          schema:\n" +
+                    "            type: integer\n" +
+                    "            format: int64\n" +
+                    "      responses:\n" +
+                    "        \"200\":\n" +
+                    "          description: successful operation\n" +
+                    "  /products/{id}/relatedProducts:\n" +
+                    "    get:\n" +
+                    "      tags:\n" +
+                    "        - relatedProducts\n" +
+                    "      summary: Get related products\n" +
+                    "      description: Get related products with the given product id.\n" +
+                    "      operationId: getRelatedProducts\n" +
+                    "      parameters:\n" +
+                    "        - name: id\n" +
+                    "          in: path\n" +
+                    "          description: Product id\n" +
+                    "          required: true\n" +
+                    "          schema:\n" +
+                    "            type: integer\n" +
+                    "            format: int64\n" +
+                    "      responses:\n" +
+                    "        \"200\":\n" +
+                    "          description: successful operation\n" +
+                    "          content:\n" +
+                    "            application/json:\n" +
+                    "              schema:\n" +
+                    "                $ref: \"#/components/schemas/RelatedProducts\"\n" +
+                    "        \"400\":\n" +
+                    "          description: Invalid product id\n" +
+                    "\n" +
+                    "components:\n" +
+                    "  schemas:\n" +
+                    "    Product:\n" +
+                    "      type: object\n" +
+                    "      description: Product id and name\n" +
+                    "      properties:\n" +
+                    "        id:\n" +
+                    "          type: integer\n" +
+                    "          format: int64\n" +
+                    "          description: Product id\n" +
+                    "        name:\n" +
+                    "          type: string\n" +
+                    "          description: Product name\n" +
+                    "      required:\n" +
+                    "        - id\n" +
+                    "        - name\n" +
+                    "    ProductDetails:\n" +
+                    "      type: object\n" +
+                    "      description: Product details\n" +
+                    "      properties:\n" +
+                    "        id:\n" +
+                    "          type: integer\n" +
+                    "          format: int64\n" +
+                    "          description: Product id\n" +
+                    "        name:\n" +
+                    "          type: string\n" +
+                    "          description: Product name\n" +
+                    "        description:\n" +
+                    "          type: string\n" +
+                    "          description: Product description\n" +
+                    "      required:\n" +
+                    "        - id\n" +
+                    "        - name\n" +
+                    "    RelatedProducts:\n" +
+                    "      type: object\n" +
+                    "      description: Related Products\n" +
+                    "      properties:\n" +
+                    "        id:\n" +
+                    "          type: integer\n" +
+                    "          format: int32\n" +
+                    "          description: Product id\n" +
+                    "        relatedProducts:\n" +
+                    "          type: array\n" +
+                    "          description: List of related products\n" +
+                    "          items:\n" +
+                    "            $ref: \"#/components/schemas/Product\"",
+                ConfigChangeWatcher.EventType.MODIFY
+            )
+        );
+
+        watcher.notifyGroup(groupItems);
+        Assert.assertEquals("GET:/products/{order-id}", endpointNameGrouping.format("serviceA", "GET:/products/123"));
+
+        groupItems.put("serviceA.productAPI-v1", new ConfigChangeWatcher.ConfigChangeEvent("", ConfigChangeWatcher.EventType.DELETE));
+        watcher.notifyGroup(groupItems);
+
+        Assert.assertEquals("GET:/products/123", endpointNameGrouping.format("serviceA", "GET:/products/123"));
+
+    }
+}