Set the endpoint name length to 150, to keep safe and avoid over-length. (#4595)

* Set the endpoint name length to 150, to keep safe and avoid over-length.

* Set endpointNameMaxLength in the application.yml

* Fix a mischange.

* Correct the endpoint name length control logic
Co-authored-by: Nkezhenxu94 <kezhenxu94@apache.org>

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/conf/Config.java

@@ -18,14 +18,13 @@
 
 package org.apache.skywalking.apm.agent.core.conf;
 
+import java.util.HashMap;
+import java.util.Map;
 import org.apache.skywalking.apm.agent.core.context.trace.TraceSegment;
 import org.apache.skywalking.apm.agent.core.logging.core.LogLevel;
 import org.apache.skywalking.apm.agent.core.logging.core.LogOutput;
 import org.apache.skywalking.apm.agent.core.logging.core.WriterFactory;
 
-import java.util.HashMap;
-import java.util.Map;
-
 /**
  * This is the core config in sniffer agent.
  */
@@ -102,9 +101,12 @@ public class Config {
         public static long FORCE_RECONNECTION_PERIOD = 1;
 
         /**
-         * Limit the length of the operationName to prevent errors when inserting elasticsearch
-         **/
-        public static int OPERATION_NAME_THRESHOLD = 500;
+         * Limit the length of the operationName to prevent the overlength issue in the storage.
+         *
+         * <p>NOTICE</p>
+         * In the current practice, we don't recommend the length over 190.
+         */
+        public static int OPERATION_NAME_THRESHOLD = 150;
     }
 
     public static class Collector {
@@ -173,11 +175,9 @@ public class Config {
 
     public static class Dictionary {
         /**
-         * The buffer size of application codes and peer
+         * The buffer size of the registered network addresses.
          */
-        public static int SERVICE_CODE_BUFFER_SIZE = 10 * 10000;
-
-        public static int ENDPOINT_NAME_BUFFER_SIZE = 1000 * 10000;
+        public static int NETWORK_ADDRESS_BUFFER_SIZE = 10 * 10000;
     }
 
     public static class Logging {

apm-sniffer/apm-agent-core/src/main/java/org/apache/skywalking/apm/agent/core/dictionary/NetworkAddressDictionary.java

@@ -26,7 +26,7 @@ import org.apache.skywalking.apm.network.register.v2.NetAddressMapping;
 import org.apache.skywalking.apm.network.register.v2.NetAddresses;
 import org.apache.skywalking.apm.network.register.v2.RegisterGrpc;
 
-import static org.apache.skywalking.apm.agent.core.conf.Config.Dictionary.SERVICE_CODE_BUFFER_SIZE;
+import static org.apache.skywalking.apm.agent.core.conf.Config.Dictionary.NETWORK_ADDRESS_BUFFER_SIZE;
 
 /**
  * Map of network address id to network literal address, which is from the collector side.
@@ -41,7 +41,7 @@ public enum NetworkAddressDictionary {
         if (applicationId != null) {
             return new Found(applicationId);
         } else {
-            if (serviceDictionary.size() + unRegisterServices.size() < SERVICE_CODE_BUFFER_SIZE) {
+            if (serviceDictionary.size() + unRegisterServices.size() < NETWORK_ADDRESS_BUFFER_SIZE) {
                 unRegisterServices.add(networkAddress);
             }
             return new NotFound();

apm-sniffer/config/agent.config

@@ -29,7 +29,7 @@ agent.service_name=${SW_AGENT_NAME:Your_ApplicationName}
 
 # The max amount of spans in a single segment.
 # Through this config item, SkyWalking keep your application memory cost estimated.
-# agent.span_limit_per_segment=${SW_AGENT_SPAN_LIMIT:300}
+# agent.span_limit_per_segment=${SW_AGENT_SPAN_LIMIT:150}
 
 # Ignore the segments if their operation names end with these suffix.
 # agent.ignore_suffix=${SW_AGENT_IGNORE_SUFFIX:.jpg,.jpeg,.js,.css,.png,.bmp,.gif,.ico,.mp3,.mp4,.html,.svg}
@@ -39,7 +39,8 @@ agent.service_name=${SW_AGENT_NAME:Your_ApplicationName}
 # agent.is_open_debugging_class = ${SW_AGENT_OPEN_DEBUG:true}
 
 # The operationName max length
-# agent.operation_name_threshold=${SW_AGENT_OPERATION_NAME_THRESHOLD:500}
+# Notice, in the current practice, we don't recommend the length over 190.
+# agent.operation_name_threshold=${SW_AGENT_OPERATION_NAME_THRESHOLD:150}
 
 # If true, skywalking agent will enable profile when user create a new profile task. Otherwise disable profile.
 # profile.active=${SW_AGENT_PROFILE_ACTIVE:true}

dist-material/application.yml

@@ -85,7 +85,9 @@ core:
     # and it will cause more load for memory, network of OAP and storage.
     # But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves.
     activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false}
-
+    # The max length of the endpoint name.
+    # In the current practice, we don't recommend the length over 190.
+    endpointNameMaxLength: ${SW_CORE_ENDPOINT_NAME_MAX_LENGTH:150}
 storage:
   selector: ${SW_STORAGE:h2}
   elasticsearch:

oap-server/server-bootstrap/src/main/resources/application.yml

@@ -84,7 +84,9 @@ core:
     # and it will cause more load for memory, network of OAP and storage.
     # But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party tool, such as Kibana->ES, to query the data by themselves.
     activeExtraModelColumns: ${SW_CORE_ACTIVE_EXTRA_MODEL_COLUMNS:false}
-
+    # The max length of the endpoint name.
+    # In the current practice, we don't recommend the length over 190.
+    endpointNameMaxLength: ${SW_CORE_ENDPOINT_NAME_MAX_LENGTH:150}
 storage:
   selector: ${SW_STORAGE:h2}
   elasticsearch:

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModule.java

@@ -57,8 +57,8 @@ import org.apache.skywalking.oap.server.library.module.ModuleDefine;
  * Core module definition. Define all open services to other modules.
  */
 public class CoreModule extends ModuleDefine {
-
     public static final String NAME = "core";
+    private static int ENDPOINT_NAME_MAX_LENGTH = 150;
 
     public CoreModule() {
         super(NAME);
@@ -87,6 +87,26 @@ public class CoreModule extends ModuleDefine {
         return classes.toArray(new Class[] {});
     }
 
+    /**
+     * Format endpoint name by using the length config in the core module. This is a global rule, every place including
+     * endpoint as the {@link org.apache.skywalking.oap.server.core.source.Source} should follow this for any core
+     * module implementation.
+     *
+     * @param endpointName raw data, literal string.
+     * @return the string, which length less than or equals {@link #ENDPOINT_NAME_MAX_LENGTH};
+     */
+    public static String formatEndpointName(String endpointName) {
+        if (endpointName.length() > ENDPOINT_NAME_MAX_LENGTH) {
+            return endpointName.substring(0, ENDPOINT_NAME_MAX_LENGTH);
+        } else {
+            return endpointName;
+        }
+    }
+
+    public static void setEndpointNameMaxLength(final int endpointNameMaxLength) {
+        ENDPOINT_NAME_MAX_LENGTH = endpointNameMaxLength;
+    }
+
     private void addProfileService(List<Class> classes) {
         classes.add(ProfileTaskMutationService.class);
         classes.add(ProfileTaskQueryService.class);

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleConfig.java

@@ -88,7 +88,6 @@ public class CoreModuleConfig extends ModuleConfig {
      */
     @Setter
     private int remoteTimeout = 20;
-
     /**
      * Following are cache settings for inventory(s)
      */
@@ -96,31 +95,34 @@ public class CoreModuleConfig extends ModuleConfig {
     private long maxSizeOfServiceInstanceInventory = 1_000_000L;
     private long maxSizeOfEndpointInventory = 1_000_000L;
     private long maxSizeOfNetworkInventory = 1_000_000L;
-
     /**
      * Following are cache setting for none stream(s)
      */
     private long maxSizeOfProfileTask = 10_000L;
-
     /**
      * Analyze profile snapshots paging size.
      */
     private int maxPageSizeOfQueryProfileSnapshot = 500;
-
     /**
      * Analyze profile snapshots max size.
      */
     private int maxSizeOfAnalyzeProfileSnapshot = 12000;
-
     /**
      * Extra model column are the column defined by {@link ScopeDefaultColumn.DefinedByField#requireDynamicActive()} ==
-     * true. These columns of model are not required logically in aggregation or further query, and it will cause more load for
-     * memory, network of OAP and storage.
+     * true. These columns of model are not required logically in aggregation or further query, and it will cause more
+     * load for memory, network of OAP and storage.
      *
      * But, being activated, user could see the name in the storage entities, which make users easier to use 3rd party
      * tool, such as Kibana->ES, to query the data by themselves.
      */
     private boolean activeExtraModelColumns = false;
+    /**
+     * The max length of the endpoint name.
+     *
+     * <p>NOTICE</p>
+     * In the current practice, we don't recommend the length over 190.
+     */
+    private int endpointNameMaxLength = 150;
 
     public CoreModuleConfig() {
         this.downsampling = new ArrayList<>();

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/CoreModuleProvider.java

@@ -139,6 +139,9 @@ public class CoreModuleProvider extends ModuleProvider {
         if (moduleConfig.isActiveExtraModelColumns()) {
             DefaultScopeDefine.activeExtraModelColumns();
         }
+        if (moduleConfig.getEndpointNameMaxLength() > 0) {
+            CoreModule.setEndpointNameMaxLength(moduleConfig.getEndpointNameMaxLength());
+        }
 
         StreamAnnotationListener streamAnnotationListener = new StreamAnnotationListener(getManager());
 

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/Endpoint.java

@@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.source;
 
 import lombok.Getter;
 import lombok.Setter;
+import org.apache.skywalking.oap.server.core.CoreModule;
 import org.apache.skywalking.oap.server.core.analysis.manual.endpoint.EndpointTraffic;
 
 import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT;
@@ -42,9 +43,13 @@ public class Endpoint extends Source {
     }
 
     @Getter
-    @Setter
     @ScopeDefaultColumn.DefinedByField(columnName = "name", requireDynamicActive = true)
     private String name;
+
+    public void setName(final String name) {
+        this.name = CoreModule.formatEndpointName(name);
+    }
+
     @Getter
     @Setter
     @ScopeDefaultColumn.DefinedByField(columnName = "service_id")

oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/EndpointRelation.java

@@ -20,6 +20,7 @@ package org.apache.skywalking.oap.server.core.source;
 
 import lombok.Getter;
 import lombok.Setter;
+import org.apache.skywalking.oap.server.core.CoreModule;
 import org.apache.skywalking.oap.server.core.analysis.manual.RelationDefineUtil;
 
 import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.ENDPOINT_RELATION;
@@ -44,9 +45,13 @@ public class EndpointRelation extends Source {
     }
 
     @Getter
-    @Setter
     @ScopeDefaultColumn.DefinedByField(columnName = "source_endpoint_name")
     private String endpoint;
+
+    public void setEndpoint(final String endpoint) {
+        this.endpoint = CoreModule.formatEndpointName(endpoint);
+    }
+
     @Getter
     @Setter
     @ScopeDefaultColumn.DefinedByField(columnName = "service_id")
@@ -61,17 +66,20 @@ public class EndpointRelation extends Source {
     @Getter
     @Setter
     private String serviceInstanceName;
-
     @Getter
-    @Setter
     @ScopeDefaultColumn.DefinedByField(columnName = "child_endpoint_name")
     private String childEndpoint;
+
+    public void setChildEndpoint(final String childEndpoint) {
+        this.childEndpoint = CoreModule.formatEndpointName(childEndpoint);
+    }
+
     @Getter
     @Setter
     @ScopeDefaultColumn.DefinedByField(columnName = "child_service_id")
     private int childServiceId;
-    @Getter
     @Setter
+    @Getter
     @ScopeDefaultColumn.DefinedByField(columnName = "child_service_name", requireDynamicActive = true)
     private String childServiceName;
     @Getter
@@ -80,7 +88,6 @@ public class EndpointRelation extends Source {
     @Getter
     @Setter
     private String childServiceInstanceName;
-
     @Getter
     @Setter
     private int componentId;