From 54fb6e7285d695b1bd467edb6df0c3a3254adc12 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=90=B4=E6=99=9F=20Wu=20Sheng?= Date: Tue, 31 Mar 2020 09:04:10 +0800 Subject: [PATCH] Set the endpoint name length to 150, to keep safe and avoid over-length. (#4595) * Set the endpoint name length to 150, to keep safe and avoid over-length. * Set endpointNameMaxLength in the application.yml * Fix a mischange. * Correct the endpoint name length control logic Co-authored-by: kezhenxu94 --- .../apm/agent/core/conf/Config.java | 20 ++++++++--------- .../dictionary/NetworkAddressDictionary.java | 4 ++-- apm-sniffer/config/agent.config | 5 +++-- dist-material/application.yml | 4 +++- .../src/main/resources/application.yml | 4 +++- .../oap/server/core/CoreModule.java | 22 ++++++++++++++++++- .../oap/server/core/CoreModuleConfig.java | 16 ++++++++------ .../oap/server/core/CoreModuleProvider.java | 3 +++ .../oap/server/core/source/Endpoint.java | 7 +++++- .../server/core/source/EndpointRelation.java | 17 +++++++++----- 10 files changed, 72 insertions(+), 30 deletions(-) diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java index 726aa4af4c..71fc41c369 100755 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java @@ -18,14 +18,13 @@ package org.apache.skywalking.apm.agent.core.conf; +import java.util.HashMap; +import java.util.Map; import org.apache.skywalking.apm.agent.core.context.trace.TraceSegment; import org.apache.skywalking.apm.agent.core.logging.core.LogLevel; import org.apache.skywalking.apm.agent.core.logging.core.LogOutput; import org.apache.skywalking.apm.agent.core.logging.core.WriterFactory; -import java.util.HashMap; -import java.util.Map; - /** * This is the core config in sniffer agent. */ @@ -102,9 +101,12 @@ public class Config { public static long FORCE_RECONNECTION_PERIOD = 1; /** - * Limit the length of the operationName to prevent errors when inserting elasticsearch - **/ - public static int OPERATION_NAME_THRESHOLD = 500; + * Limit the length of the operationName to prevent the overlength issue in the storage. + * + *

NOTICE

+ * In the current practice, we don't recommend the length over 190. + */ + public static int OPERATION_NAME_THRESHOLD = 150; } public static class Collector { @@ -173,11 +175,9 @@ public class Config { public static class Dictionary { /** - * The buffer size of application codes and peer + * The buffer size of the registered network addresses. */ - public static int SERVICE_CODE_BUFFER_SIZE = 10 * 10000; - - public static int ENDPOINT_NAME_BUFFER_SIZE = 1000 * 10000; + public static int NETWORK_ADDRESS_BUFFER_SIZE = 10 * 10000; } public static class Logging { diff --git a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/dictionary/NetworkAddressDictionary.java b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/dictionary/NetworkAddressDictionary.java index 99f187dc85..ac30860b60 100755 --- a/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/dictionary/NetworkAddressDictionary.java +++ b/apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/dictionary/NetworkAddressDictionary.java @@ -26,7 +26,7 @@ import org.apache.skywalking.apm.network.register.v2.NetAddressMapping; import org.apache.skywalking.apm.network.register.v2.NetAddresses; import org.apache.skywalking.apm.network.register.v2.RegisterGrpc; -import static org.apache.skywalking.apm.agent.core.conf.Config.Dictionary.SERVICE_CODE_BUFFER_SIZE; +import static org.apache.skywalking.apm.agent.core.conf.Config.Dictionary.NETWORK_ADDRESS_BUFFER_SIZE; /** * Map of network address id to network literal address, which is from the collector side. @@ -41,7 +41,7 @@ public enum NetworkAddressDictionary { if (applicationId != null) { return new Found(applicationId); } else { - if (serviceDictionary.size() + unRegisterServices.size() < SERVICE_CODE_BUFFER_SIZE) { + if (serviceDictionary.size() + unRegisterServices.size() < NETWORK_ADDRESS_BUFFER_SIZE) { unRegisterServices.add(networkAddress); } return new NotFound(); diff --git a/apm-sniffer/config/agent.config b/apm-sniffer/config/agent.config index ee6dc5a4d6..23cb92a474 100644 --- a/apm-sniffer/config/agent.config +++ b/apm-sniffer/config/agent.config @@ -29,7 +29,7 @@ agent.service_name=${SW_AGENT_NAME:Your_ApplicationName} # The max amount of spans in a single segment. # Through this config item, SkyWalking keep your application memory cost estimated. -# agent.span_limit_per_segment=${SW_AGENT_SPAN_LIMIT:300} +# agent.span_limit_per_segment=${SW_AGENT_SPAN_LIMIT:150} # Ignore the segments if their operation names end with these suffix. # agent.ignore_suffix=${SW_AGENT_IGNORE_SUFFIX:.jpg,.jpeg,.js,.css,.png,.bmp,.gif,.ico,.mp3,.mp4,.html,.svg} @@ -39,7 +39,8 @@ agent.service_name=${SW_AGENT_NAME:Your_ApplicationName} # agent.is_open_debugging_class = ${SW_AGENT_OPEN_DEBUG:true} # The operationName max length -# agent.operation_name_threshold=${SW_AGENT_OPERATION_NAME_THRESHOLD:500} +# Notice, in the current practice, we don't recommend the length over 190. +# agent.operation_name_threshold=${SW_AGENT_OPERATION_NAME_THRESHOLD:150} # If true, skywalking agent will enable profile when user create a new profile task. Otherwise disable profile. # profile.active=${SW_AGENT_PROFILE_ACTIVE:true} diff --git a/dist-material/application.yml b/dist-material/application.yml index f715df9c1a..5e38bfcf7a 100644 --- a/dist-material/application.yml +++ b/dist-material/application.yml @@ -85,7 +85,9 @@ core: # and it will cause more load for memory, network of OAP and storage. # But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves. activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false} - + # The max length of the endpoint name. + # In the current practice, we don't recommend the length over 190. + endpointNameMaxLength: ${SW_CORE_ENDPOINT_NAME_MAX_LENGTH:150} storage: selector: ${SW_STORAGE:h2} elasticsearch: diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index 7c796257ac..c794452798 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -84,7 +84,9 @@ core: # and it will cause more load for memory, network of OAP and storage. # But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves. activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false} - + # The max length of the endpoint name. + # In the current practice, we don't recommend the length over 190. + endpointNameMaxLength: ${SW_CORE_ENDPOINT_NAME_MAX_LENGTH:150} storage: selector: ${SW_STORAGE:h2} elasticsearch: diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java index 81878e6cea..0dcd04572a 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java @@ -57,8 +57,8 @@ import org.apache.skywalking.oap.server.library.module.ModuleDefine; * Core module definition. Define all open services to other modules. */ public class CoreModule extends ModuleDefine { - public static final String NAME = "core"; + private static int ENDPOINT_NAME_MAX_LENGTH = 150; public CoreModule() { super(NAME); @@ -87,6 +87,26 @@ public class CoreModule extends ModuleDefine { return classes.toArray(new Class[] {}); } + /** + * Format endpoint name by using the length config in the core module. This is a global rule, every place including + * endpoint as the {@link org.apache.skywalking.oap.server.core.source.Source} should follow this for any core + * module implementation. + * + * @param endpointName raw data, literal string. + * @return the string, which length less than or equals {@link #ENDPOINT_NAME_MAX_LENGTH}; + */ + public static String formatEndpointName(String endpointName) { + if (endpointName.length() > ENDPOINT_NAME_MAX_LENGTH) { + return endpointName.substring(0, ENDPOINT_NAME_MAX_LENGTH); + } else { + return endpointName; + } + } + + public static void setEndpointNameMaxLength(final int endpointNameMaxLength) { + ENDPOINT_NAME_MAX_LENGTH = endpointNameMaxLength; + } + private void addProfileService(List classes) { classes.add(ProfileTaskMutationService.class); classes.add(ProfileTaskQueryService.class); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java index e59ae44490..96b8fb5654 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java @@ -88,7 +88,6 @@ public class CoreModuleConfig extends ModuleConfig { */ @Setter private int remoteTimeout = 20; - /** * Following are cache settings for inventory(s) */ @@ -96,31 +95,34 @@ public class CoreModuleConfig extends ModuleConfig { private long maxSizeOfServiceInstanceInventory = 1_000_000L; private long maxSizeOfEndpointInventory = 1_000_000L; private long maxSizeOfNetworkInventory = 1_000_000L; - /** * Following are cache setting for none stream(s) */ private long maxSizeOfProfileTask = 10_000L; - /** * Analyze profile snapshots paging size. */ private int maxPageSizeOfQueryProfileSnapshot = 500; - /** * Analyze profile snapshots max size. */ private int maxSizeOfAnalyzeProfileSnapshot = 12000; - /** * Extra model column are the column defined by {@link ScopeDefaultColumn.DefinedByField#requireDynamicActive()} == - * true. These columns of model are not required logically in aggregation or further query, and it will cause more load for - * memory, network of OAP and storage. + * true. These columns of model are not required logically in aggregation or further query, and it will cause more + * load for memory, network of OAP and storage. * * But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party * tool, such as Kibana->ES, to query the data by themselves. */ private boolean activeExtraModelColumns = false; + /** + * The max length of the endpoint name. + * + *

NOTICE

+ * In the current practice, we don't recommend the length over 190. + */ + private int endpointNameMaxLength = 150; public CoreModuleConfig() { this.downsampling = new ArrayList<>(); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java index 547c57d6c1..43d90c8f93 100755 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java @@ -139,6 +139,9 @@ public class CoreModuleProvider extends ModuleProvider { if (moduleConfig.isActiveExtraModelColumns()) { DefaultScopeDefine.activeExtraModelColumns(); } + if (moduleConfig.getEndpointNameMaxLength() > 0) { + CoreModule.setEndpointNameMaxLength(moduleConfig.getEndpointNameMaxLength()); + } StreamAnnotationListener streamAnnotationListener = new StreamAnnotationListener(getManager()); diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java index 821f768c54..9ce9186fc8 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java @@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.source; import lombok.Getter; import lombok.Setter; +import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic; import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT; @@ -42,9 +43,13 @@ public class Endpoint extends Source { } @Getter - @Setter @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true) private String name; + + public void setName(final String name) { + this.name = CoreModule.formatEndpointName(name); + } + @Getter @Setter @ScopeDefaultColumn.DefinedByField(columnName = "service_id") diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java index 4536929c40..473120dffe 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java @@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.source; import lombok.Getter; import lombok.Setter; +import org.apache.skywalking.oap.server.core.CoreModule; import org.apache.skywalking.oap.server.core.analysis.manual.RelationDefineUtil; import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT_RELATION; @@ -44,9 +45,13 @@ public class EndpointRelation extends Source { } @Getter - @Setter @ScopeDefaultColumn.DefinedByField(columnName = "source_endpoint_name") private String endpoint; + + public void setEndpoint(final String endpoint) { + this.endpoint = CoreModule.formatEndpointName(endpoint); + } + @Getter @Setter @ScopeDefaultColumn.DefinedByField(columnName = "service_id") @@ -61,17 +66,20 @@ public class EndpointRelation extends Source { @Getter @Setter private String serviceInstanceName; - @Getter - @Setter @ScopeDefaultColumn.DefinedByField(columnName = "child_endpoint_name") private String childEndpoint; + + public void setChildEndpoint(final String childEndpoint) { + this.childEndpoint = CoreModule.formatEndpointName(childEndpoint); + } + @Getter @Setter @ScopeDefaultColumn.DefinedByField(columnName = "child_service_id") private int childServiceId; - @Getter @Setter + @Getter @ScopeDefaultColumn.DefinedByField(columnName = "child_service_name", requireDynamicActive = true) private String childServiceName; @Getter @@ -80,7 +88,6 @@ public class EndpointRelation extends Source { @Getter @Setter private String childServiceInstanceName; - @Getter @Setter private int componentId; -- GitLab