diff --git a/CHANGES.md b/CHANGES.md index 6a81392e187b9692763432a36bcbb6ecaf75154a..4a734f7ed289b61430693b569e33f2683b93261d 100644 --- a/CHANGES.md +++ b/CHANGES.md @@ -38,6 +38,7 @@ Release Notes. * Upgrade embed tomcat caused by CVE-2020-13935. * Upgrade commons-lang3 to avoid potential NPE in some JDK versions. * OAL supports generating metrics from events. +* Support endpoint name grouping by OpenAPI definitions. #### UI diff --git a/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java b/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java index ab30cdb45149cf941ae906a13276f480ede96463..9f22dafd2a64145799e25b7d2414b559a811e9e5 100644 --- a/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java +++ b/apm-commons/apm-util/src/main/java/org/apache/skywalking/apm/util/StringFormatGroup.java @@ -19,6 +19,7 @@ package org.apache.skywalking.apm.util; import java.util.ArrayList; +import java.util.Comparator; import java.util.List; import java.util.regex.Pattern; import lombok.Getter; @@ -68,6 +69,10 @@ public class StringFormatGroup { return new FormatResult(false, string, string); } + public void sortRules(Comparator comparator) { + rules.sort(comparator); + } + @Getter @RequiredArgsConstructor public static class FormatResult { @@ -78,7 +83,7 @@ public class StringFormatGroup { @Getter @ToString - private static class PatternRule { + public static class PatternRule { private final String name; private final Pattern pattern; diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 5becb12ee5cd5567743a2b1b0b082901d5a0ba34..33f406a2c9c8d1eb9714da85a8ce8b14602c5efc 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -42,6 +42,7 @@ core|default|role|Option values, `Mixed/Receiver/Aggregator`. **Receiver** mode | - | - | maxSizeOfAnalyzeProfileSnapshot|The max number of snapshots analyzed by OAP| - | 12000 | | - | - | syncThreads|The number of threads used to synchronously refresh the metrics data to the storage.| SW_CORE_SYNC_THREADS | 2 | | - | - | maxSyncOperationNum|The maximum number of processes supported for each synchronous storage operation. When the number of the flush data is greater than this value, it will be assigned to multiple cores for execution.| SW_CORE_MAX_SYNC_OPERATION_NUM | 50000 | +| - | - | enableEndpointNameGroupingByOpenapi |Turn it on then automatically grouping endpoint by the given OpenAPI definitions.| SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI | true | |cluster|standalone| - | standalone is not suitable for one node running, no available configuration.| - | - | | - | zookeeper|nameSpace|The namespace, represented by root path, isolates the configurations in the zookeeper.|SW_NAMESPACE| `/`, root path| | - | - | hostPort|hosts and ports of Zookeeper Cluster|SW_CLUSTER_ZK_HOST_PORT| localhost:2181| diff --git a/docs/en/setup/backend/endpoint-grouping-rules.md b/docs/en/setup/backend/endpoint-grouping-rules.md index dd623beacb35476656ea1afbfbe769d79f3e852a..e4eda52b51157911cb4262856c184ad85e1ce698 100644 --- a/docs/en/setup/backend/endpoint-grouping-rules.md +++ b/docs/en/setup/backend/endpoint-grouping-rules.md @@ -1,15 +1,294 @@ # Group Parameterized Endpoints -In most cases, the endpoint should be detected automatically through the language agents, service mesh observability solution, +In most cases, the endpoint should be detected automatically through the language agents, service mesh observability solution, or configuration of meter system. -There are some special cases, especially when people use REST style URI, the application codes put the parameter in the endpoint name, +There are some special cases, especially when people use REST style URI, the application codes put the parameter in the endpoint name, such as putting order id in the URI, like `/prod/ORDER123` and `/prod/ORDER123`. But logically, people expect they could have an endpoint name like `prod/{order-id}`. This is the feature of parameterized endpoint grouping designed for. +If the incoming endpoint name hit the rules, SkyWalking will grouping the endpoint by rules. + +SkyWalking provides 2 ways to support endpoint grouping: +1. Endpoint name grouping by OpenAPI definitions. +2. Endpoint name grouping by custom configuration. + +The 2 grouping features can work together in sequence. +## Endpoint name grouping by OpenAPI definitions +The OpenAPI definitions are the documents based on The [OpenAPI Specification (OAS)](https://www.openapis.org/) which used to define a standard, language-agnostic interface for HTTP APIs. +SkyWalking now support `OAS v2.0+`, could parse the documents `(yaml)` and build the grouping rules from them automatically. + + +### How to use +1. Add some `Specification Extensions` for SkyWalking config in the OpenAPI definition documents, otherwise, all configs are default:
+ `${METHOD}` is a reserved placeholder which represents the HTTP method eg. `POST/GET...`
+ `${PATH}` is a reserved placeholder which represents the path eg. `/products/{id}`. + + | Extension Name | Required | Description | Default Value | + |-----|-----|-----|-----| + | x-sw-service-name | false | The service name which these endpoints belong to | The directory name which the OpenAPI definition documents belong to | + | x-sw-endpoint-name-match-rule | false | The rule used to match the endpoint.| `${METHOD}:${PATH}` | + | x-sw-endpoint-name-format | false | The endpoint name after grouping.| `${METHOD}:${PATH}` | + + These extensions are under `OpenAPI Object`. For example, the document below has a full custom config: + +``` yaml +openapi: 3.0.0 +x-sw-service-name: serviceB +x-sw-endpoint-name-match-rule: "${METHOD}:${PATH}" +x-sw-endpoint-name-format: "${METHOD}:${PATH}" + +info: + description: OpenAPI definition for SkyWalking test. + version: v2 + title: Product API + ... +``` + + We highly recommend using the default config, the custom config (`x-sw-endpoint-name-match-rule/x-sw-endpoint-name-format`) would be considered as part of the match rules (regex pattern). + We provide some cases in `org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest`, you could validate your custom config as well. + +2. All OpenAPI definition documents should be located in the `openapi-definitions` directory, with at most two levels directories. Recommend using the service name as the subDirectory name then you are not required to set `x-sw-service-name`. For example: + ``` +├── openapi-definitions +│   ├── serviceA +│   │   ├── customerAPI-v1.yaml +│   │   └── productAPI-v1.yaml +│   └── serviceB +│   └── productAPI-v2.yaml +``` +3. The feature is enabled by default, you can disable it by setting the `Core Module` configuration `${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:false}` + +### Rules match priority +We recommend designing the API path as clear as possible. If the API path is fuzzy and an endpoint name might match multiple paths, SkyWalking would follow the match priority to select one as below orders: +1. The exact path matched first. + Eg. `/products or /products/inventory` +2. The path which has the less variables. + Eg. `/products/{var1}/{var2} and /products/{var1}/abc`, endpoint name `/products/123/abc` will match the second one. +3. If the paths have the same number of variables, match the longest path, and the vars are considered to be `1`. + Eg. `/products/abc/{var1} and products/{var12345}/ef`, endpoint name `/products/abc/ef` will match the first one, because `length("abc") = 3` is larger than `length("ef") = 2`. +### Examples +If we have an OpenAPI definition doc `productAPI-v2.yaml` in directory `serviceB` like this: +```yaml + +openapi: 3.0.0 + +info: + description: OpenAPI definition for SkyWalking test. + version: v2 + title: Product API + +tags: + - name: product + description: product + - name: relatedProducts + description: Related Products + +paths: + /products: + get: + tags: + - product + summary: Get all products list + description: Get all products list. + operationId: getProducts + responses: + "200": + description: Success + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Product" + /products/{region}/{country}: + get: + tags: + - product + summary: Get products regional + description: Get products regional with the given id. + operationId: getProductRegional + parameters: + - name: region + in: path + description: Products region + required: true + schema: + type: string + - name: country + in: path + description: Products country + required: true + schema: + type: string + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/Product" + "400": + description: Invalid parameters supplied + /products/{id}: + get: + tags: + - product + summary: Get product details + description: Get product details with the given id. + operationId: getProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ProductDetails" + "400": + description: Invalid product id + post: + tags: + - product + summary: Update product details + description: Update product details with the given id. + operationId: updateProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + - name: name + in: query + description: Product name + required: true + schema: + type: string + responses: + "200": + description: successful operation + delete: + tags: + - product + summary: Delete product details + description: Delete product details with the given id. + operationId: deleteProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + /products/{id}/relatedProducts: + get: + tags: + - relatedProducts + summary: Get related products + description: Get related products with the given product id. + operationId: getRelatedProducts + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/RelatedProducts" + "400": + description: Invalid product id + +components: + schemas: + Product: + type: object + description: Product id and name + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + required: + - id + - name + ProductDetails: + type: object + description: Product details + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + description: + type: string + description: Product description + required: + - id + - name + RelatedProducts: + type: object + description: Related Products + properties: + id: + type: integer + format: int32 + description: Product id + relatedProducts: + type: array + description: List of related products + items: + $ref: "#/components/schemas/Product" + + +``` + +Here are some cases: + + | Incoming Endpiont | Incoming Service | x-sw-service-name | x-sw-endpoint-name-match-rule | x-sw-endpoint-name-format | Matched | Grouping Result | + |-----|-----|-----|-----|-----|-----|-----| + | `GET:/products` | serviceB | default | default | default | true | `GET:/products` | + | `GET:/products/123` | serviceB | default | default | default | true | `GET:/products{id}` | + | `GET:/products/asia/cn` | serviceB | default | default | default | true | `GET:/products/{region}/{country}` | + | `GET:/products/123/abc/efg` | serviceB | default | default | default | false | `GET:/products/123/abc/efg` | + | `:/products/123` | serviceB | default | default | default | false | `:/products/123`| + | `GET:/products/123` | serviceC | default | default | default | false | `GET:/products/123` | + | `GET:/products/123` | serviceC | serviceC | default | default | true | `GET:/products/123` | + | `:/products/123` | serviceB | default | `<${METHOD}>:${PATH}` | `<${METHOD}>:${PATH}` | true | `:/products/{id}` | + | `GET:/products/123` | serviceB | default | default | `${PATH}:<${METHOD}>` | true | `/products/{id}:` | + | `/products/123:` | serviceB | default | `${PATH}:<${METHOD}>` | default | true | `GET:/products/{id}` | + + +## Endpoint name grouping by custom configuration Current, user could set up grouping rules through the static YAML file, named `endpoint-name-grouping.yml`, or use [Dynamic Configuration](dynamic-config.md) to initial and update the endpoint grouping rule. -## Configuration Format +### Configuration Format No matter in static local file or dynamic configuration value, they are sharing the same YAML format. ```yaml @@ -20,4 +299,4 @@ grouping: # Logic name when the regex expression matched. - endpoint-name: /prod/{id} regex: \/prod\/.+ -``` \ No newline at end of file +``` diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index 1bb16ee25a72817a9cf5c872ccdedcff7bbbc5b2..4c39a68b8e460f6f847dd65046380022d5095bcc 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -108,6 +108,8 @@ core: syncThreads: ${SW_CORE_SYNC_THREADS:2} # The maximum number of processes supported for each synchronous storage operation. When the number of the flush data is greater than this value, it will be assigned to multiple cores for execution. maxSyncOperationNum: ${SW_CORE_MAX_SYNC_OPERATION_NUM:50000} + # Turn it on then automatically grouping endpoint by the given OpenAPI definitions. + enableEndpointNameGroupingByOpenapi: ${SW_CORE_ENABLE_ENDPOINT_NAME_GROUPING_BY_OPAENAPI:true} storage: selector: ${SW_STORAGE:h2} elasticsearch: diff --git a/oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA/productAPI-v1.yaml b/oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA/productAPI-v1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..89e85ec01987dadf38dd82e14cf9cb40b8828def --- /dev/null +++ b/oap-server/server-bootstrap/src/main/resources/openapi-definitions/serviceA/productAPI-v1.yaml @@ -0,0 +1,215 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# +# Add some `Specification Extensions` for SkyWalking config in the OpenAPI definition documents, otherwise, all configs are default: +# `${METHOD}` is a reserved placeholder which represents the HTTP method eg. `POST/GET...` +# `${PATH}` is a reserved placeholder which represents the path eg. `/products/{id}`. +# +# | Extension Name | Required | Description | Default Value | +# |-----|-----|-----|-----| +# | x-sw-service-name | false | The service name to which these endpoints belong | The directory name which the definition documents belong to| +# | x-sw-endpoint-name-match-rule | false | The rule used to match the endpoint.| `${METHOD}:${PATH}` | +# | x-sw-endpoint-name-format | false | The endpoint name after grouping.| `${METHOD}:${PATH}` | +# +# These extensions are under `OpenAPI Object`. For example, the document below has a full custom config: +# +#``` yaml +#openapi: 3.0.0 +#x-sw-service-name: serviceB +#x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" +#x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" +# +#info: +# description: OpenAPI definition for SkyWalking test. +# version: v2 +# title: Product API +# ... +#``` +# +# We highly recommend using the default config, the custom config (`x-sw-endpoint-name-match-rule/x-sw-endpoint-name-format`) would be considered as part of the match rules (regex pattern). +# We provide some cases in `org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4OpenapiTest`, you could validate your custom config as well. + + +################################################### +# This is only a example using the default config.# +################################################### + + +#openapi: 3.0.0 +# +#info: +# description: OpenAPI definition for SkyWalking test. +# version: v1 +# title: Product API +# +#tags: +# - name: product +# description: product +# - name: relatedProducts +# description: Related Products +# +#paths: +# /products: +# get: +# tags: +# - product +# summary: Get all products list +# description: Get all products list. +# operationId: getProducts +# responses: +# "200": +# description: Success +# content: +# application/json: +# schema: +# type: array +# items: +# $ref: "#/components/schemas/Product" +# /products/{id}: +# get: +# tags: +# - product +# summary: Get product details +# description: Get product details with the given id. +# operationId: getProduct +# parameters: +# - name: id +# in: path +# description: Product id +# required: true +# schema: +# type: integer +# format: int64 +# responses: +# "200": +# description: successful operation +# content: +# application/json: +# schema: +# $ref: "#/components/schemas/ProductDetails" +# "400": +# description: Invalid product id +# post: +# tags: +# - product +# summary: Update product details +# description: Update product details with the given id. +# operationId: updateProduct +# parameters: +# - name: id +# in: path +# description: Product id +# required: true +# schema: +# type: integer +# format: int64 +# - name: name +# in: query +# description: Product name +# required: true +# schema: +# type: string +# responses: +# "200": +# description: successful operation +# delete: +# tags: +# - product +# summary: Delete product details +# description: Delete product details with the given id. +# operationId: deleteProduct +# parameters: +# - name: id +# in: path +# description: Product id +# required: true +# schema: +# type: integer +# format: int64 +# responses: +# "200": +# description: successful operation +# /products/{id}/relatedProducts: +# get: +# tags: +# - relatedProducts +# summary: Get related products +# description: Get related products with the given product id. +# operationId: getRelatedProducts +# parameters: +# - name: id +# in: path +# description: Product id +# required: true +# schema: +# type: integer +# format: int64 +# responses: +# "200": +# description: successful operation +# content: +# application/json: +# schema: +# $ref: "#/components/schemas/RelatedProducts" +# "400": +# description: Invalid product id +# +#components: +# schemas: +# Product: +# type: object +# description: Product id and name +# properties: +# id: +# type: integer +# format: int64 +# description: Product id +# name: +# type: string +# description: Product name +# required: +# - id +# - name +# ProductDetails: +# type: object +# description: Product details +# properties: +# id: +# type: integer +# format: int64 +# description: Product id +# name: +# type: string +# description: Product name +# description: +# type: string +# description: Product description +# required: +# - id +# - name +# RelatedProducts: +# type: object +# description: Related Products +# properties: +# id: +# type: integer +# format: int32 +# description: Product id +# relatedProducts: +# type: array +# description: List of related products +# items: +# $ref: "#/components/schemas/Product" \ No newline at end of file diff --git a/oap-server/server-core/pom.xml b/oap-server/server-core/pom.xml index 098d2f8e9e83962ec2157a513241faa6a46bdb8d..1135336b56d066b3808db76407193dbc3f867552 100644 --- a/oap-server/server-core/pom.xml +++ b/oap-server/server-core/pom.xml @@ -82,7 +82,6 @@ io.vavr vavr - org.apache.skywalking server-testing @@ -94,6 +93,11 @@ grpc-testing test + + org.openjdk.jmh + jmh-generator-annprocess + test + diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java index 22f75a9df85f1a3887bbff6073c617d52b6ce270..bed117dfe22cac41b3360d8f62872d3514fb366f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java @@ -158,6 +158,10 @@ public class CoreModuleConfig extends ModuleConfig { @Setter private int maxSyncOperationNum = 50000; + @Getter + @Setter + private boolean enableEndpointNameGroupingByOpenapi = false; + public CoreModuleConfig() { this.downsampling = new ArrayList<>(); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index 38929892f913f1928209697ef10dd9e84ab17c4f..9e99bd1f09ea955c5660c7d76c0e42c49e58e300 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -46,6 +46,7 @@ import org.apache.skywalking.oap.server.core.config.ConfigService; import org.apache.skywalking.oap.server.core.config.DownSamplingConfigService; import org.apache.skywalking.oap.server.core.config.IComponentLibraryCatalogService; import org.apache.skywalking.oap.server.core.config.NamingControl; +import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRuleReader4Openapi; import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; import org.apache.skywalking.oap.server.core.config.group.EndpointNameGroupingRuleWatcher; import org.apache.skywalking.oap.server.core.management.ui.template.UITemplateInitializer; @@ -159,6 +160,11 @@ public class CoreModuleProvider extends ModuleProvider { try { endpointNameGroupingRuleWatcher = new EndpointNameGroupingRuleWatcher( this, endpointNameGrouping); + + if (moduleConfig.isEnableEndpointNameGroupingByOpenapi()) { + endpointNameGrouping.setEndpointGroupingRule4Openapi( + new EndpointGroupingRuleReader4Openapi("openapi-definitions").read()); + } } catch (FileNotFoundException e) { throw new ModuleStartException(e.getMessage(), e); } diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java index da952d67d2c2952115d096a154e01a3aae7786af..8c460a1f056a3c386bb1760034589a67decfcf3f 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/EndpointNameGrouping.java @@ -21,20 +21,48 @@ package org.apache.skywalking.oap.server.core.config.group; import lombok.Setter; import lombok.extern.slf4j.Slf4j; import org.apache.skywalking.apm.util.StringFormatGroup; +import org.apache.skywalking.oap.server.core.config.group.openapi.EndpointGroupingRule4Openapi; @Slf4j public class EndpointNameGrouping { @Setter private volatile EndpointGroupingRule endpointGroupingRule; + @Setter + private volatile EndpointGroupingRule4Openapi endpointGroupingRule4Openapi; public String format(String serviceName, String endpointName) { - if (endpointGroupingRule == null) { - return endpointName; + String formattedName = endpointName; + if (endpointGroupingRule4Openapi != null) { + formattedName = formatByOpenapi(serviceName, formattedName); + } + + if (endpointGroupingRule != null) { + formattedName = formatByCustom(serviceName, formattedName); } + + return formattedName; + } + + private String formatByCustom(String serviceName, String endpointName) { final StringFormatGroup.FormatResult formatResult = endpointGroupingRule.format(serviceName, endpointName); if (log.isDebugEnabled() || log.isTraceEnabled()) { if (formatResult.isMatch()) { - log.debug("Endpoint {} of Service {} has been renamed in group {}", + log.debug("Endpoint {} of Service {} has been renamed in group {} by endpointGroupingRule", + endpointName, serviceName, formatResult.getName() + ); + } else { + log.trace("Endpoint {} of Service {} keeps unchanged.", endpointName, serviceName); + } + } + return formatResult.getName(); + } + + private String formatByOpenapi(String serviceName, String endpointName) { + final StringFormatGroup.FormatResult formatResult = endpointGroupingRule4Openapi.format( + serviceName, endpointName); + if (log.isDebugEnabled() || log.isTraceEnabled()) { + if (formatResult.isMatch()) { + log.debug("Endpoint {} of Service {} has been renamed in group {} by endpointGroupingRule4Openapi", endpointName, serviceName, formatResult.getName() ); } else { diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java new file mode 100644 index 0000000000000000000000000000000000000000..ea3b0ebb6fd528f4aff951da051e5dfe760791e4 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRule4Openapi.java @@ -0,0 +1,117 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.openapi; + +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import lombok.Getter; +import org.apache.skywalking.apm.util.StringFormatGroup; + +public class EndpointGroupingRule4Openapi { + private final Map> directLookup = new HashMap<>(); + @Getter + private final Map> groupedRules = new HashMap<>(); + + void addDirectLookup(String serviceName, String endpointName, String endpointGroupName) { + Map endpointNameLookup = directLookup.computeIfAbsent(serviceName, name -> new HashMap<>()); + endpointNameLookup.put(endpointName, endpointGroupName); + } + + void addGroupedRule(String serviceName, String endpointGroupName, String ruleRegex) { + String rulesGroupkey = getGroupedRulesKey(ruleRegex); + Map rules = groupedRules.computeIfAbsent(serviceName, name -> new HashMap<>()); + StringFormatGroup formatGroup = rules.computeIfAbsent(rulesGroupkey, name -> new StringFormatGroup()); + formatGroup.addRule(endpointGroupName, ruleRegex); + } + + public StringFormatGroup.FormatResult format(String service, String endpointName) { + Map endpointNameLookup = directLookup.get(service); + if (endpointNameLookup != null && endpointNameLookup.get(endpointName) != null) { + return new StringFormatGroup.FormatResult(true, endpointNameLookup.get(endpointName), endpointName); + } + + Map rules = groupedRules.get(service); + if (rules != null) { + final StringFormatGroup stringFormatGroup = rules.get(getGroupedRulesKey(endpointName)); + if (stringFormatGroup != null) { + return stringFormatGroup.format(endpointName); + } + } + + return new StringFormatGroup.FormatResult(false, endpointName, endpointName); + } + + void sortRulesAll() { + groupedRules.entrySet().forEach(rules -> { + sortRulesByService(rules.getKey()); + }); + } + + void sortRulesByService(String serviceName) { + Map rules = groupedRules.get(serviceName); + if (rules != null) { + rules.entrySet().forEach(stringFormatGroup -> { + stringFormatGroup.getValue() + .sortRules(new EndpointGroupingRule4Openapi.EndpointGroupingRulesComparator()); + }); + } + } + + String getGroupedRulesKey(String string) { + String[] ss = string.split("/"); + if (ss.length == 1) { //eg. POST:/ + return ss[0] + "/"; + } + if (ss.length > 1) { + return ss[0] + "/" + ss[1]; + } + return "/"; + } + + static class EndpointGroupingRulesComparator implements Comparator { + private static final String VAR_PATTERN = "\\(\\[\\^\\/\\]\\+\\)"; + + @Override + public int compare(final StringFormatGroup.PatternRule rule1, final StringFormatGroup.PatternRule rule2) { + + String pattern1 = rule1.getPattern().pattern(); + String pattern2 = rule2.getPattern().pattern(); + + if (getPatternVarsCount(pattern1) < getPatternVarsCount(pattern2)) { + return -1; + } else if (getPatternVarsCount(pattern1) > getPatternVarsCount(pattern2)) { + return 1; + } + + int length1 = getPatternLength(pattern1); + int length2 = getPatternLength(pattern2); + return length2 - length1; + } + + private int getPatternVarsCount(String pattern) { + return ",".concat(pattern).concat(",").split(VAR_PATTERN).length - 1; + } + + private int getPatternLength(String pattern) { + return pattern.replaceAll(VAR_PATTERN, "#").length(); + } + + } +} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java new file mode 100644 index 0000000000000000000000000000000000000000..ab945e7aea7afe17035cc4d92a02ed14f3d2e0b9 --- /dev/null +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4Openapi.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.openapi; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.Reader; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.library.util.ResourceUtils; +import org.yaml.snakeyaml.Yaml; +import org.yaml.snakeyaml.constructor.SafeConstructor; + +public class EndpointGroupingRuleReader4Openapi { + + private final String openapiDefPath; + private final static String DEFAULT_ENDPOINT_NAME_FORMAT = "${METHOD}:${PATH}"; + private final static String DEFAULT_ENDPOINT_NAME_MATCH_RULE = "${METHOD}:${PATH}"; + private final Map requestMethodsMap = new HashMap() { + { + put("get", "GET"); + put("post", "POST"); + put("put", "PUT"); + put("delete", "DELETE"); + put("trace", "TRACE"); + put("options", "OPTIONS"); + put("head", "HEAD"); + put("patch", "PATCH"); + } + }; + + public EndpointGroupingRuleReader4Openapi(final String openapiDefPath) { + + this.openapiDefPath = openapiDefPath; + } + + public EndpointGroupingRule4Openapi read() throws FileNotFoundException { + EndpointGroupingRule4Openapi endpointGroupingRule = new EndpointGroupingRule4Openapi(); + + List fileList = ResourceUtils.getDirectoryFilesRecursive(openapiDefPath, 1); + for (File file : fileList) { + if (!file.getName().endsWith(".yaml")) { + continue; + } + Reader reader = new FileReader(file); + Yaml yaml = new Yaml(new SafeConstructor()); + Map openapiData = yaml.load(reader); + if (openapiData != null) { + String serviceName = getServiceName(openapiData, file); + LinkedHashMap> paths = + (LinkedHashMap>) openapiData.get("paths"); + + if (paths != null) { + paths.forEach((pathString, pathItem) -> { + pathItem.keySet().forEach(key -> { + String requestMethod = requestMethodsMap.get(key); + if (!StringUtil.isEmpty(requestMethod)) { + String endpointGroupName = formatEndPointName(pathString, requestMethod, openapiData); + String groupRegex = getGroupRegex(pathString, requestMethod, openapiData); + if (isTemplatePath(pathString)) { + endpointGroupingRule.addGroupedRule(serviceName, endpointGroupName, groupRegex); + } else { + endpointGroupingRule.addDirectLookup(serviceName, groupRegex, endpointGroupName); + } + } + }); + }); + } + } + } + endpointGroupingRule.sortRulesAll(); + return endpointGroupingRule; + } + + private String getServiceName(Map openapiData, File file) { + + String serviceName = (String) openapiData.get("x-sw-service-name"); + if (StringUtil.isEmpty(serviceName)) { + File directory = new File(file.getParent()); + if (openapiDefPath.equals(directory.getName())) { + throw new IllegalArgumentException( + "OpenAPI definition file: " + file.getAbsolutePath() + " found in root directory, but doesn't include x-sw-service-name extensive definition in the file."); + } + serviceName = directory.getName(); + } + + return serviceName; + } + + private boolean isTemplatePath(String pathString) { + return pathString.matches("(.*)\\{(.+?)}(.*)"); + } + + private String getGroupRegex(String pathString, String requstMathod, Map openapiData) { + String endPointNameMatchRuleTemplate = (String) openapiData.get("x-sw-endpoint-name-match-rule"); + String endPointNameMatchRule = replaceTemplateVars(DEFAULT_ENDPOINT_NAME_MATCH_RULE, pathString, requstMathod); + + if (!StringUtil.isEmpty(endPointNameMatchRuleTemplate)) { + endPointNameMatchRule = replaceTemplateVars(endPointNameMatchRuleTemplate, pathString, requstMathod); + } + + if (isTemplatePath(endPointNameMatchRule)) { + return endPointNameMatchRule.replaceAll("\\{(.+?)}", "([^/]+)"); + } + return endPointNameMatchRule; + } + + private String formatEndPointName(String pathString, String requstMethod, Map openapiData) { + String endPointNameFormat = (String) openapiData.get("x-sw-endpoint-name-format"); + + if (!StringUtil.isEmpty(endPointNameFormat)) { + return replaceTemplateVars(endPointNameFormat, pathString, requstMethod); + } + + return replaceTemplateVars(DEFAULT_ENDPOINT_NAME_FORMAT, pathString, requstMethod); + } + + private String replaceTemplateVars(String template, String pathString, String requstMathod) { + return template.replaceAll("\\$\\{METHOD}", requstMathod) + .replaceAll("\\$\\{PATH}", pathString); + } + +} diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java new file mode 100644 index 0000000000000000000000000000000000000000..d2ffdf3dea0f07ac60d204664b83cb9694af328f --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingBenchmark4Openapi.java @@ -0,0 +1,172 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.openapi; + +import java.io.FileNotFoundException; +import lombok.SneakyThrows; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.profile.GCProfiler; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +@BenchmarkMode({Mode.Throughput}) +@Threads(4) +public class EndpointGroupingBenchmark4Openapi { + + @State(Scope.Benchmark) + public static class FormatClassPaths20 { + private EndpointGroupingRule4Openapi rule; + + @SneakyThrows + public FormatClassPaths20() { + rule = new EndpointGroupingRule4Openapi(); + for (int i = 0; i <= 3; i++) { + rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + i, "GET:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + i, "POST:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + i, "GET:/products2/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + i, "POST:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + i, "GET:/products3/([^/]+)/" + i); + } + } + + public void format(String serviceName, String endpointName) { + rule.format(serviceName, endpointName); + } + } + + @State(Scope.Benchmark) + public static class FormatClassPaths50 { + private EndpointGroupingRule4Openapi rule; + + @SneakyThrows + public FormatClassPaths50() { + rule = new EndpointGroupingRule4Openapi(); + for (int i = 0; i <= 9; i++) { + rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + i, "GET:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + i, "POST:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + i, "GET:/products2/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + i, "POST:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + i, "GET:/products3/([^/]+)/" + i); + } + } + + public void format(String serviceName, String endpointName) { + rule.format(serviceName, endpointName); + } + } + + @State(Scope.Benchmark) + public static class FormatClassPaths200 { + private EndpointGroupingRule4Openapi rule; + + @SneakyThrows + public FormatClassPaths200() { + rule = new EndpointGroupingRule4Openapi(); + for (int i = 0; i <= 39; i++) { + rule.addGroupedRule("serviceA", "GET:/products1/{id}/" + i, "GET:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products1/{id}/" + i, "POST:/products1/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products2/{id}/" + i, "GET:/products2/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "POST:/products3/{id}/" + i, "POST:/products3/([^/]+)/" + i); + rule.addGroupedRule("serviceA", "GET:/products3/{id}/" + i, "GET:/products3/([^/]+)/" + i); + } + } + + public void format(String serviceName, String endpointName) { + rule.format(serviceName, endpointName); + } + + } + + @Benchmark + public void formatEndpointNameMatchedPaths20(FormatClassPaths20 formatClass) { + formatClass.format("serviceA", "GET:/products1/123"); + } + + @Benchmark + public void formatEndpointNameMatchedPaths50(FormatClassPaths50 formatClass) { + formatClass.format("serviceA", "GET:/products1/123"); + } + + @Benchmark + public void formatEndpointNameMatchedPaths200(FormatClassPaths200 formatClass) { + formatClass.format("serviceA", "GET:/products1/123"); + } + + public static void main(String[] args) throws RunnerException, FileNotFoundException { + + Options opt = new OptionsBuilder() + .include(EndpointGroupingBenchmark4Openapi.class.getName()) + .addProfiler(GCProfiler.class) + .jvmArgsAppend("-Xmx512m", "-Xms512m") + .forks(1) + .build(); + + new Runner(opt).run(); + } +} + +/* +* The test is assumed each endpoint need to run all match within it's rules group. +* +# JMH version: 1.21 +# VM version: JDK 1.8.0_271, Java HotSpot(TM) 64-Bit Server VM, 25.271-b09 +# VM invoker: /Library/Java/JavaVirtualMachines/jdk1.8.0_271.jdk/Contents/Home/jre/bin/java +# VM options: -javaagent:/Applications/IntelliJ IDEA CE.app/Contents/lib/idea_rt.jar=51431:/Applications/IntelliJ IDEA CE.app/Contents/bin -Dfile.encoding=UTF-8 -Xmx512m -Xms512m +# Warmup: 5 iterations, 10 s each +# Measurement: 5 iterations, 10 s each +# Timeout: 10 min per iteration +# Threads: 4 threads, will synchronize iterations +# Benchmark mode: Throughput, ops/time + +Benchmark Mode Cnt Score Error Units +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20 thrpt 5 4180207.544 ± 833644.395 ops/s +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.alloc.rate thrpt 5 4524.954 ± 903.291 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.alloc.rate.norm thrpt 5 1192.000 ± 0.001 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Eden_Space thrpt 5 4550.511 ± 916.117 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Eden_Space.norm thrpt 5 1198.713 ± 10.572 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Survivor_Space thrpt 5 0.493 ± 0.118 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.130 ± 0.039 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.count thrpt 5 1410.000 counts +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths20:·gc.time thrpt 5 783.000 ms +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200 thrpt 5 600313.461 ± 58702.201 ops/s +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.alloc.rate thrpt 5 4260.484 ± 415.215 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.alloc.rate.norm thrpt 5 7816.000 ± 0.001 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Eden_Space thrpt 5 4285.685 ± 407.822 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Eden_Space.norm thrpt 5 7862.339 ± 46.737 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Survivor_Space thrpt 5 0.444 ± 0.061 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.815 ± 0.062 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.count thrpt 5 1328.000 counts +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths200:·gc.time thrpt 5 729.000 ms +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50 thrpt 5 2001647.224 ± 139386.146 ops/s +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.alloc.rate thrpt 5 4173.062 ± 291.166 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.alloc.rate.norm thrpt 5 2296.000 ± 0.001 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Eden_Space thrpt 5 4198.202 ± 271.551 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Eden_Space.norm thrpt 5 2309.878 ± 14.994 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Survivor_Space thrpt 5 0.393 ± 0.171 MB/sec +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.churn.PS_Survivor_Space.norm thrpt 5 0.216 ± 0.086 B/op +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.count thrpt 5 1301.000 counts +EndpointGroupingBenchmark4Openapi.formatEndpointNameMatchedPaths50:·gc.time thrpt 5 715.000 ms + */ \ No newline at end of file diff --git a/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java new file mode 100644 index 0000000000000000000000000000000000000000..b66df94c8cfe7f98e5f52919c24e7cab4570d135 --- /dev/null +++ b/oap-server/server-core/src/test/java/org/apache/skywalking/oap/server/core/config/group/openapi/EndpointGroupingRuleReader4OpenapiTest.java @@ -0,0 +1,97 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.core.config.group.openapi; + +import java.io.IOException; +import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping; +import org.junit.Assert; +import org.junit.Test; + +public class EndpointGroupingRuleReader4OpenapiTest { + + @Test + public void testReadingRule() throws IOException { + + EndpointGroupingRuleReader4Openapi reader = new EndpointGroupingRuleReader4Openapi("openapi-definitions"); + EndpointGroupingRule4Openapi rule = reader.read(); + EndpointNameGrouping nameGrouping = new EndpointNameGrouping(); + nameGrouping.setEndpointGroupingRule4Openapi(rule); + + //default x-sw-service-name x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format + // test direct lookup + String endpointName = nameGrouping.format("serviceA", "GET:/products"); + Assert.assertEquals("GET:/products", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/123"); + Assert.assertEquals("GET:/products/{id}", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/123/abc/ef"); + Assert.assertEquals("GET:/products/123/abc/ef", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/123/relatedProducts"); + Assert.assertEquals("GET:/products/{id}/relatedProducts", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/1/relatedProducts"); + Assert.assertEquals("GET:/products/{id}/relatedProducts", endpointName); + + //test custom x-sw-service-name same x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format + endpointName = nameGrouping.format("serviceA-1", "POST:/customer"); + Assert.assertEquals("POST:/customer", endpointName); + + endpointName = nameGrouping.format("serviceA-1", ":/customers/1"); + Assert.assertEquals(":/customers/{id}", endpointName); + + //test different x-sw-endpoint-name-match-rule and x-sw-endpoint-name-format + endpointName = nameGrouping.format("serviceB", "GET:/products"); + Assert.assertEquals("/products:", endpointName); + + endpointName = nameGrouping.format("serviceB", "GET:/products/asia/cn"); + Assert.assertEquals("/products/{region}/{country}:", endpointName); + + //test match priority, not match /products/{region}/{country}: + endpointName = nameGrouping.format("serviceB", "GET:/products/12/relatedProducts"); + Assert.assertEquals("/products/{id}/relatedProducts:", endpointName); + + //test not match, return the origin + endpointName = nameGrouping.format("serviceA", "GET:/products/"); + Assert.assertNotEquals("GET:/products", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/123/"); + Assert.assertEquals("GET:/products/123/", endpointName); + + endpointName = nameGrouping.format("serviceC", "GET:/products/123"); + Assert.assertEquals("GET:/products/123", endpointName); + + endpointName = nameGrouping.format("serviceA", "GET:/products/1/ratings/123"); + Assert.assertEquals("GET:/products/1/ratings/123", endpointName); + + endpointName = nameGrouping.format("serviceA-1", ":/customers/1/123"); + Assert.assertEquals(":/customers/1/123", endpointName); + + endpointName = nameGrouping.format("serviceB", "/products/:"); + Assert.assertEquals("/products/:", endpointName); + + endpointName = nameGrouping.format("serviceB", "{GET}:/products"); + Assert.assertEquals("{GET}:/products", endpointName); + + endpointName = nameGrouping.format("serviceB", "/products/1/2/3:"); + Assert.assertEquals("/products/1/2/3:", endpointName); + + } +} diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/customerAPI-v1.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/customerAPI-v1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..20dc3f0db1a75bbc23f86861c90b042689240fe5 --- /dev/null +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/customerAPI-v1.yaml @@ -0,0 +1,172 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +openapi: 3.0.0 +x-sw-service-name: serviceA-1 +x-sw-endpoint-name-match-rule: "<${METHOD}>:${PATH}" +x-sw-endpoint-name-format: "<${METHOD}>:${PATH}" +info: + description: OpenAPI definition for SkyWalking test. + version: v1 + title: Customer API + +tags: + - name: customer + description: customer + +paths: + /customers: + get: + tags: + - customer + summary: Get all customers list + description: Get all customers list. + operationId: getCustomers + responses: + "200": + description: Success + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Customer" + /customers/{id}: + get: + tags: + - customer + summary: Get customer details + description: Get customer details with the given id. + operationId: getCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/CustomerDetails" + "400": + description: Invalid customer id + post: + tags: + - customer + summary: Update customer details + description: Update customer details with the given id. + operationId: updateCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + - name: name + in: query + description: Customer name + required: true + schema: + type: string + responses: + "200": + description: successful operation + delete: + tags: + - customer + summary: Delete customer details + description: Delete customer details with the given id. + operationId: deleteCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + + /customer/{region}/{country}: + get: + tags: + - customer + summary: Get customers regional + description: Get customers regional with the given id. + operationId: getCustomersRegional + parameters: + - name: region + in: path + description: Customers region + required: true + schema: + type: string + - name: country + in: path + description: Customers country + required: true + schema: + type: string + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/Customer" + "400": + description: Invalid parameters supplied +components: + schemas: + Customer: + type: object + description: Customer id and name + properties: + id: + type: integer + format: int64 + description: Customer id + name: + type: string + description: Customer name + required: + - id + - name + CustomerDetails: + type: object + description: Customer details + properties: + id: + type: integer + format: int64 + description: Customer id + name: + type: string + description: Customer name + description: + type: string + description: Customer description + required: + - id + - name diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/productAPI-v1.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/productAPI-v1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2a6d5722ad3f4b2b6b547188c93732e5e4df5613 --- /dev/null +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/productAPI-v1.yaml @@ -0,0 +1,180 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +openapi: 3.0.0 + +info: + description: OpenAPI definition for SkyWalking test. + version: v1 + title: Product API + +tags: + - name: product + description: product + - name: relatedProducts + description: Related Products + +paths: + /products: + get: + tags: + - product + summary: Get all products list + description: Get all products list. + operationId: getProducts + responses: + "200": + description: Success + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Product" + /products/{id}: + get: + tags: + - product + summary: Get product details + description: Get product details with the given id. + operationId: getProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ProductDetails" + "400": + description: Invalid product id + post: + tags: + - product + summary: Update product details + description: Update product details with the given id. + operationId: updateProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + - name: name + in: query + description: Product name + required: true + schema: + type: string + responses: + "200": + description: successful operation + delete: + tags: + - product + summary: Delete product details + description: Delete product details with the given id. + operationId: deleteProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + /products/{id}/relatedProducts: + get: + tags: + - relatedProducts + summary: Get related products + description: Get related products with the given product id. + operationId: getRelatedProducts + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/RelatedProducts" + "400": + description: Invalid product id + +components: + schemas: + Product: + type: object + description: Product id and name + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + required: + - id + - name + ProductDetails: + type: object + description: Product details + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + description: + type: string + description: Product description + required: + - id + - name + RelatedProducts: + type: object + description: Related Products + properties: + id: + type: integer + format: int32 + description: Product id + relatedProducts: + type: array + description: List of related products + items: + $ref: "#/components/schemas/Product" diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/serviceA-1/customerAPI-a-1-v1.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/serviceA-1/customerAPI-a-1-v1.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d4a60a532d88632152154bca370e23bdab5e19ed --- /dev/null +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceA/serviceA-1/customerAPI-a-1-v1.yaml @@ -0,0 +1,171 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +openapi: 3.0.0 +x-sw-service-name: serviceA + +info: + description: OpenAPI definition for SkyWalking test. + version: v1 + title: Customer API + +tags: + - name: customer + description: customer + +paths: + /customers: + get: + tags: + - customer + summary: Get all customers list + description: Get all customers list. + operationId: getCustomers + responses: + "200": + description: Success + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Customer" + /customers/{id}: + get: + tags: + - customer + summary: Get customer details + description: Get customer details with the given id. + operationId: getCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/CustomerDetails" + "400": + description: Invalid customer id + post: + tags: + - customer + summary: Update customer details + description: Update customer details with the given id. + operationId: updateCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + - name: name + in: query + description: Customer name + required: true + schema: + type: string + responses: + "200": + description: successful operation + delete: + tags: + - customer + summary: Delete customer details + description: Delete customer details with the given id. + operationId: deleteCustomer + parameters: + - name: id + in: path + description: Customer id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + + /customer/{region}/{country}: + get: + tags: + - customer + summary: Get customers regional + description: Get customers regional with the given id. + operationId: getCustomersRegional + parameters: + - name: region + in: path + description: Customers region + required: true + schema: + type: string + - name: country + in: path + description: Customers country + required: true + schema: + type: string + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/Customer" + "400": + description: Invalid parameters supplied +components: + schemas: + Customer: + type: object + description: Customer id and name + properties: + id: + type: integer + format: int64 + description: Customer id + name: + type: string + description: Customer name + required: + - id + - name + CustomerDetails: + type: object + description: Customer details + properties: + id: + type: integer + format: int64 + description: Customer id + name: + type: string + description: Customer name + description: + type: string + description: Customer description + required: + - id + - name diff --git a/oap-server/server-core/src/test/resources/openapi-definitions/serviceB/productAPI-v2.yaml b/oap-server/server-core/src/test/resources/openapi-definitions/serviceB/productAPI-v2.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b30a8b36654f49c705be84bd05b77bef2bb4a936 --- /dev/null +++ b/oap-server/server-core/src/test/resources/openapi-definitions/serviceB/productAPI-v2.yaml @@ -0,0 +1,210 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +openapi: 3.0.0 +x-sw-endpoint-name-format: "${PATH}:<${METHOD}>" + +info: + description: OpenAPI definition for SkyWalking test. + version: v2 + title: Product API + +tags: + - name: product + description: product + - name: relatedProducts + description: Related Products + +paths: + /products: + get: + tags: + - product + summary: Get all products list + description: Get all products list. + operationId: getProducts + responses: + "200": + description: Success + content: + application/json: + schema: + type: array + items: + $ref: "#/components/schemas/Product" + /products/{region}/{country}: + get: + tags: + - product + summary: Get products regional + description: Get products regional with the given id. + operationId: getProductRegional + parameters: + - name: region + in: path + description: Products region + required: true + schema: + type: string + - name: country + in: path + description: Products country + required: true + schema: + type: string + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/Product" + "400": + description: Invalid parameters supplied + /products/{id}: + get: + tags: + - product + summary: Get product details + description: Get product details with the given id. + operationId: getProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/ProductDetails" + "400": + description: Invalid product id + post: + tags: + - product + summary: Update product details + description: Update product details with the given id. + operationId: updateProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + - name: name + in: query + description: Product name + required: true + schema: + type: string + responses: + "200": + description: successful operation + delete: + tags: + - product + summary: Delete product details + description: Delete product details with the given id. + operationId: deleteProduct + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + /products/{id}/relatedProducts: + get: + tags: + - relatedProducts + summary: Get related products + description: Get related products with the given product id. + operationId: getRelatedProducts + parameters: + - name: id + in: path + description: Product id + required: true + schema: + type: integer + format: int64 + responses: + "200": + description: successful operation + content: + application/json: + schema: + $ref: "#/components/schemas/RelatedProducts" + "400": + description: Invalid product id + +components: + schemas: + Product: + type: object + description: Product id and name + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + required: + - id + - name + ProductDetails: + type: object + description: Product details + properties: + id: + type: integer + format: int64 + description: Product id + name: + type: string + description: Product name + description: + type: string + description: Product description + required: + - id + - name + RelatedProducts: + type: object + description: Related Products + properties: + id: + type: integer + format: int32 + description: Product id + relatedProducts: + type: array + description: List of related products + items: + $ref: "#/components/schemas/Product" diff --git a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java index 1381090a218ebb87086002abacfaaee9d138f49b..93b26207a7d08db2995200aefbf7d524ee0530f0 100644 --- a/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java +++ b/oap-server/server-library/library-util/src/main/java/org/apache/skywalking/oap/server/library/util/ResourceUtils.java @@ -24,8 +24,10 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.Reader; import java.net.URL; +import java.util.ArrayList; import java.util.Arrays; import java.util.HashSet; +import java.util.List; import java.util.Objects; import java.util.Set; @@ -58,12 +60,51 @@ public class ResourceUtils { throw new FileNotFoundException("path not found: " + parentPath); } final Set nameSet = new HashSet<>(Arrays.asList(fileNames)); - final File[] listFiles = Objects.requireNonNull(new File(url.getPath()) - .listFiles((dir, name) -> nameSet.contains(name)), "No files in " + parentPath); + final File[] listFiles = Objects.requireNonNull( + new File(url.getPath()) + .listFiles((dir, name) -> nameSet.contains(name)), "No files in " + parentPath); if (listFiles.length == 0) { throw new FileNotFoundException("files not found:" + nameSet); } return listFiles; } + + /** + * @param directoryPath the directory path + * @param maxDepth the max directory depth to get the files, the given directory is 0 as the tree root + * @return all normal files which in this directory and subDirectory according to the maxDepth + * @throws FileNotFoundException the directory not exist in the given path + */ + public static List getDirectoryFilesRecursive(String directoryPath, + int maxDepth) throws FileNotFoundException { + + URL url = ResourceUtils.class.getClassLoader().getResource(directoryPath); + if (url == null) { + throw new FileNotFoundException("path not found: " + directoryPath); + } + List fileList = new ArrayList<>(); + return getDirectoryFilesRecursive(url.getPath(), fileList, maxDepth); + } + + private static List getDirectoryFilesRecursive(String directoryPath, List fileList, int maxDepth) { + if (maxDepth < 0) { + return fileList; + } + maxDepth--; + File file = new File(directoryPath); + if (file.isDirectory()) { + File[] subFiles = file.listFiles(); + if (subFiles != null) { + for (File subFile : subFiles) { + if (subFile.isDirectory()) { + getDirectoryFilesRecursive(subFile.getPath(), fileList, maxDepth); + } else { + fileList.add(subFile); + } + } + } + } + return fileList; + } }